Add robots.txt checking

Still needs periodic cache refresh
2024-10-23 14:24:10 +03:00
parent c49a69728a
commit ef3f009709
12 changed files with 225 additions and 156 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -3,6 +3,7 @@
 **/*~
 /cmd
 /db/initdb.sql
+/db/*sh
 /run*.sh
 /gemini-grc
 /snaps