Move history crawl interval to env

.env

@@ -0,0 +1 @@
+CRAWL_INTERVAL=10

pyproject.toml

@@ -10,4 +10,5 @@ dependencies = [
     "uvicorn[standard]",
     "crawl4ai",
     "browser-history",
+    "python-dotenv"
 ]

src/main.py

@@ -10,9 +10,14 @@ from base_crawler import BaseCrawler
 import asyncio
 from contextlib import asynccontextmanager
 from browser_history import get_history
+from dotenv import load_dotenv
 # Create logs directory if it doesn't exist
 os.makedirs('logs', exist_ok=True)
 
+# Load environment variables
+load_dotenv()
+CRAWL_INTERVAL = int(os.getenv('CRAWL_INTERVAL', 30))  # Default to 30 seconds if not set
+
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     # Startup
@@ -62,7 +67,7 @@ class HistoryCrawler(BaseCrawler):
             if success:
                 self.logger.info(f"Processed historical URL: {url}")
 
-            await asyncio.sleep(30)  # Wait 30 seconds before next crawl
+            await asyncio.sleep(CRAWL_INTERVAL)  # Use environment variable for interval
 
 @app.websocket("/ws")
 async def websocket_endpoint(websocket: WebSocket):

uv.lock

@@ -197,6 +197,7 @@ dependencies = [
     { name = "browser-history" },
     { name = "crawl4ai" },
     { name = "fastapi" },
+    { name = "python-dotenv" },
     { name = "uvicorn", extra = ["standard"] },
 ]
 
@@ -205,6 +206,7 @@ requires-dist = [
     { name = "browser-history" },
     { name = "crawl4ai" },
     { name = "fastapi" },
+    { name = "python-dotenv" },
     { name = "uvicorn", extras = ["standard"] },
 ]