Lots of features, first version that reliably crawls Geminispace.

- [x] Concurrent downloading with workers
- [x] Concurrent connection limit per host
- [x] URL Blacklist
- [x] Save image/* and text/* files
- [x] Configuration via environment variables
- [x] Storing snapshots in PostgreSQL
- [x] Proper response header & body UTF-8 and format validation
.

.

gemini/blacklist.go

@@ -0,0 +1,18 @@
+package gemini
+
+import "strings"
+
+var Blacklist *[]string
+
+func InBlacklist(s *Snapshot) bool {
+	if Blacklist == nil {
+		data := ReadLines("blacklist.txt")
+		Blacklist = &data
+	}
+	for _, l := range *Blacklist {
+		if strings.HasPrefix(s.URL.String(), l) {
+			return true
+		}
+	}
+	return false
+}