.

2024-11-18 16:28:45 +02:00
parent f0452ff9f7
commit 825c7e3391
34 changed files with 624 additions and 426 deletions
--- a/gemini/robots.go
+++ b/gemini/robots.go
@@ -5,7 +5,7 @@ import (
 	"strings"
 )

-// Takes robots.txt content and a host, and
+// ParseRobotsTxt takes robots.txt content and a host, and
 // returns a list of full URLs that shouldn't
 // be visited.
 // TODO Also take into account the user agent?