Add robots.txt checking

Still needs periodic cache refresh
2024-10-23 14:24:10 +03:00
parent c49a69728a
commit ef3f009709
12 changed files with 225 additions and 156 deletions
--- a/db/stats.sql
+++ b/db/stats.sql
@@ -1,5 +1,5 @@
 SELECT
    COUNT(CASE WHEN response_code IS NOT NULL AND error IS NULL THEN 1 END) AS "Visited",
-    COUNT(CASE WHEN response_code IS NULL THEN 1 END) AS "Pending",
+    COUNT(CASE WHEN response_code IS NULL     AND error IS NULL THEN 1 END) AS "Pending",
    COUNT(CASE WHEN error IS NOT NULL THEN 1 END) AS "Errors"
 FROM snapshots;