diff --git a/.env b/.env new file mode 100644 index 0000000..063eceb --- /dev/null +++ b/.env @@ -0,0 +1 @@ +CRAWL_INTERVAL=10 \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 6ec2997..31e3ac1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,4 +10,5 @@ dependencies = [ "uvicorn[standard]", "crawl4ai", "browser-history", + "python-dotenv" ] diff --git a/src/main.py b/src/main.py index e68a7db..77f46f7 100644 --- a/src/main.py +++ b/src/main.py @@ -10,9 +10,14 @@ from base_crawler import BaseCrawler import asyncio from contextlib import asynccontextmanager from browser_history import get_history +from dotenv import load_dotenv # Create logs directory if it doesn't exist os.makedirs('logs', exist_ok=True) +# Load environment variables +load_dotenv() +CRAWL_INTERVAL = int(os.getenv('CRAWL_INTERVAL', 30)) # Default to 30 seconds if not set + @asynccontextmanager async def lifespan(app: FastAPI): # Startup @@ -62,7 +67,7 @@ class HistoryCrawler(BaseCrawler): if success: self.logger.info(f"Processed historical URL: {url}") - await asyncio.sleep(30) # Wait 30 seconds before next crawl + await asyncio.sleep(CRAWL_INTERVAL) # Use environment variable for interval @app.websocket("/ws") async def websocket_endpoint(websocket: WebSocket): diff --git a/uv.lock b/uv.lock index 69e3d53..8b01975 100644 --- a/uv.lock +++ b/uv.lock @@ -197,6 +197,7 @@ dependencies = [ { name = "browser-history" }, { name = "crawl4ai" }, { name = "fastapi" }, + { name = "python-dotenv" }, { name = "uvicorn", extra = ["standard"] }, ] @@ -205,6 +206,7 @@ requires-dist = [ { name = "browser-history" }, { name = "crawl4ai" }, { name = "fastapi" }, + { name = "python-dotenv" }, { name = "uvicorn", extras = ["standard"] }, ]