Lots of features, first version that reliably crawls Geminispace.

- [x] Concurrent downloading with workers
- [x] Concurrent connection limit per host
- [x] URL Blacklist
- [x] Save image/* and text/* files
- [x] Configuration via environment variables
- [x] Storing snapshots in PostgreSQL
- [x] Proper response header & body UTF-8 and format validation
.

.

gemini/connectionPool.go

@@ -0,0 +1,32 @@
+package gemini
+
+import (
+	"gemini-grc/logging"
+)
+
+var IpPool IpAddressPool = IpAddressPool{IPs: make(map[string]int)}
+
+func AddIPsToPool(IPs []string) {
+	IpPool.Lock.Lock()
+	for _, ip := range IPs {
+		logging.LogDebug("Adding %s to pool", ip)
+		IpPool.IPs[ip]++
+	}
+	IpPool.Lock.Unlock()
+}
+
+func RemoveIPsFromPool(IPs []string) {
+	IpPool.Lock.Lock()
+	for _, ip := range IPs {
+		_, ok := IpPool.IPs[ip]
+		if ok {
+			logging.LogDebug("Removing %s from pool", ip)
+			if IpPool.IPs[ip] == 1 {
+				delete(IpPool.IPs, ip)
+			} else {
+				IpPool.IPs[ip]--
+			}
+		}
+	}
+	IpPool.Lock.Unlock()
+}