Improve crawler performance and logging
- Optimize job scheduler to use NumOfWorkers for URL limits - Clean up verbose logging in worker processing - Update log messages for better clarity
This commit is contained in:
6
misc/sql/fetch-snapshot-history.sql
Normal file
6
misc/sql/fetch-snapshot-history.sql
Normal file
@@ -0,0 +1,6 @@
|
||||
select count(*) from snapshots
|
||||
where last_crawled < now() - interval '30 days'
|
||||
and error IS NULL
|
||||
and gemtext IS NOT NULL
|
||||
and mimetype='text/gemini'
|
||||
and url ~ '^gemini://[^/]+/?$';
|
||||
Reference in New Issue
Block a user