Add robots.txt checking

Still needs periodic cache refresh

gemini/persistence.go

@@ -57,6 +57,34 @@ func SaveSnapshotToDB(tx *sqlx.Tx, s *Snapshot) error {
 	return nil
 }
 
+func SaveLinksToDBinBatches(tx *sqlx.Tx, snapshots []*Snapshot) error {
+	// Approximately 5,957 rows maximum (65535/11 parameters), use 5000 to be safe
+	const batchSize = 5000
+
+	query := `
+        INSERT INTO snapshots (uid, url, host, timestamp, mimetype, data, gemtext, links, lang, response_code, error)
+        VALUES (:uid, :url, :host, :timestamp, :mimetype, :data, :gemtext, :links, :lang, :response_code, :error)
+        ON CONFLICT (uid) DO NOTHING
+    `
+
+	for i := 0; i < len(snapshots); i += batchSize {
+		end := i + batchSize
+		if end > len(snapshots) {
+			end = len(snapshots)
+		}
+
+		batch := snapshots[i:end]
+
+		_, err := tx.NamedExec(query, batch)
+		if err != nil {
+			logging.LogError("Error batch inserting snapshots: %w", err)
+			return fmt.Errorf("DB error: %w", err)
+		}
+	}
+
+	return nil
+}
+
 func SaveLinksToDB(tx *sqlx.Tx, snapshots []*Snapshot) error {
 	query := `
         INSERT INTO snapshots (uid, url, host, timestamp, mimetype, data, gemtext, links, lang, response_code, error)