Add robots.txt checking

Still needs periodic cache refresh
2024-10-23 14:24:10 +03:00
parent c49a69728a
commit 1e54df741d
7 changed files with 112 additions and 42 deletions
--- a/db/initdb.sql
+++ b/db/initdb.sql
@@ -42,4 +42,5 @@ CREATE INDEX idx_lang ON snapshots (lang);
 CREATE INDEX idx_response_code ON snapshots (response_code);
 CREATE INDEX idx_error ON snapshots (error);
 CREATE INDEX idx_host ON snapshots (host);
+-- Add the unprocessed snapshots index here! check db
 CREATE INDEX idx_response_code_error_nulls ON snapshots (response_code, error) WHERE response_code IS NULL AND error IS NULL;