Enhance crawler with seed list and SQL utilities

Add seedList module for URL initialization, comprehensive SQL utilities for database analysis, and update project configuration.
This commit is contained in:
antanst
2025-06-16 12:29:33 +03:00
parent 51f94c90b2
commit 330b596497
37 changed files with 742 additions and 682 deletions

View File

@@ -0,0 +1,30 @@
BEGIN;
WITH matching_urls AS (
SELECT url, host
FROM snapshots
WHERE url ~ '^gemini://[^/]+/$'
AND timestamp < (NOW() - INTERVAL '1 week')
ORDER BY random()
LIMIT 500
)
INSERT INTO urls (url, host)
SELECT url, host
FROM matching_urls
ON CONFLICT DO NOTHING;
-- WITH matching_urls AS (
-- SELECT url, host
-- FROM snapshots
-- WHERE url ~ '^gemini://[^/]+/$'
-- AND timestamp < (NOW() - INTERVAL '1 week')
-- ORDER BY random()
-- LIMIT 500
-- )
-- DELETE FROM snapshots
-- WHERE url IN (
-- SELECT url
-- FROM matching_urls
-- );
COMMIT;