Better error handling, many fixes all around

gemini/robotmatch.go

@@ -57,8 +57,8 @@ func populateBlacklist(key string) (entries []string) {
 // RobotMatch checks if the snapshot URL matches
 // a robots.txt allow rule.
 func RobotMatch(url URL) bool {
-	logging.LogDebug("Checking robots.txt cache for %s", url.String())
 	key := strings.ToLower(fmt.Sprintf("%s:%d", url.Hostname, url.Port))
+	logging.LogDebug("Checking robots.txt cache for %s", key)
 	var disallowedURLs []string
 	cacheEntries, ok := RobotsCache.Load(key)
 	if !ok {