From 90dc188dc8030cb69152772d106f03d1bd123d75 Mon Sep 17 00:00:00 2001 From: Zetaphor Date: Sat, 12 Apr 2025 12:54:58 -0500 Subject: [PATCH] Remove cruft --- src/main.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main.py b/src/main.py index 34864a4..2072957 100644 --- a/src/main.py +++ b/src/main.py @@ -14,14 +14,13 @@ app = FastAPI() logger = Logger() db = Database() -domain_exclusions = DomainExclusions() # Initialize with default config path +domain_exclusions = DomainExclusions() @app.websocket("/ws") async def websocket_endpoint(websocket: WebSocket): await websocket.accept() logger.info("New WebSocket connection established") - # Create crawler instance outside the loop for reuse async with AsyncWebCrawler() as crawler: try: while True: @@ -42,10 +41,8 @@ async def websocket_endpoint(websocket: WebSocket): }) continue - # Crawl the URL to get title and content try: result = await crawler.arun(url=url) - # Get the first result from the container and access metadata crawl_result = result[0] title = crawl_result.metadata.get('title') or url.split("/")[-1] content = crawl_result.markdown @@ -54,7 +51,6 @@ async def websocket_endpoint(websocket: WebSocket): title = url.split("/")[-1] content = str(data) - # Store received data with crawled information db.add_history( url=url, title=title,