Fix snapshot overwrite logic to preserve successful responses

- Prevent overwriting snapshots that have valid response codes
- Ensure URL is removed from queue when snapshot update is skipped
- Add last_crawled timestamp tracking for better crawl scheduling
- Remove SkipIdenticalContent flag, simplify content deduplication logic
- Update database schema with last_crawled column and indexes

db/db_queries.go

@@ -67,38 +67,10 @@ LIMIT $1
 `
 	// New query - always insert a new snapshot without conflict handling
 	SQL_INSERT_SNAPSHOT = `
-        INSERT INTO snapshots (url, host, timestamp, mimetype, data, gemtext, links, lang, response_code, error, header)
-        VALUES (:url, :host, :timestamp, :mimetype, :data, :gemtext, :links, :lang, :response_code, :error, :header)
+        INSERT INTO snapshots (url, host, timestamp, mimetype, data, gemtext, links, lang, response_code, error, header, last_crawled)
+        VALUES (:url, :host, :timestamp, :mimetype, :data, :gemtext, :links, :lang, :response_code, :error, :header, :last_crawled)
         RETURNING id
     `
-	// Keep for backward compatibility, but should be phased out
-	SQL_INSERT_SNAPSHOT_IF_NEW = `
-        INSERT INTO snapshots (url, host, timestamp, mimetype, data, gemtext, links, lang, response_code, error, header)
-        VALUES (:url, :host, :timestamp, :mimetype, :data, :gemtext, :links, :lang, :response_code, :error, :header)
-        ON CONFLICT DO NOTHING
-    `
-	// Update to match the SQL_INSERT_SNAPSHOT - we no longer want to upsert, just insert new versions
-	SQL_UPSERT_SNAPSHOT = `
-        INSERT INTO snapshots (url, host, timestamp, mimetype, data, gemtext, links, lang, response_code, error, header)
-        VALUES (:url, :host, :timestamp, :mimetype, :data, :gemtext, :links, :lang, :response_code, :error, :header)
-        RETURNING id
-    `
-	SQL_UPDATE_SNAPSHOT = `UPDATE snapshots
-SET url = :url,
-host = :host,
-timestamp = :timestamp,
-mimetype = :mimetype,
-data = :data,
-gemtext = :gemtext,
-links = :links,
-lang = :lang,
-response_code = :response_code,
-error = :error,
-header = :header,
-last_crawled = CURRENT_TIMESTAMP
-WHERE id = :id
-RETURNING id
-`
 	SQL_INSERT_URL = `
         INSERT INTO urls (url, host, timestamp)
         VALUES (:url, :host, :timestamp)
@@ -115,7 +87,6 @@ RETURNING id
 	SQL_DELETE_URL = `
         DELETE FROM urls WHERE url=$1
     `
-	// New queries for retrieving snapshots
 	SQL_GET_LATEST_SNAPSHOT = `
         SELECT * FROM snapshots
         WHERE url = $1
@@ -140,9 +111,65 @@ RETURNING id
         AND timestamp BETWEEN $2 AND $3
         ORDER BY timestamp DESC
     `
-	// New query to record crawl attempt when content is identical (no new snapshot needed)
-	SQL_RECORD_CRAWL_ATTEMPT = `
-        INSERT INTO snapshots (url, host, mimetype, response_code, error)
-        VALUES ($1, $2, $3, $4, $5)
+	// Update last_crawled timestamp for the most recent snapshot of a URL
+	SQL_UPDATE_LAST_CRAWLED = `
+        UPDATE snapshots 
+        SET last_crawled = CURRENT_TIMESTAMP 
+        WHERE id = (
+            SELECT id FROM snapshots 
+            WHERE url = $1 
+            ORDER BY timestamp DESC 
+            LIMIT 1
+        )
+    `
+	// SQL_FETCH_SNAPSHOTS_FROM_HISTORY Fetches URLs from snapshots for re-crawling based on last_crawled timestamp
+	// This query finds root domain URLs that haven't been crawled recently and selects
+	// one URL per host for diversity. Uses CTEs to:
+	// 1. Find latest crawl attempt per URL (via MAX(last_crawled))
+	// 2. Filter to URLs with actual content and successful responses (20-29)
+	// 3. Select URLs where latest crawl is older than cutoff date
+	// 4. Rank randomly within each host and pick one URL per host
+	// Parameters: $1 = cutoff_date, $2 = limit
+	SQL_FETCH_SNAPSHOTS_FROM_HISTORY = `
+		WITH latest_attempts AS (
+			SELECT 
+				url,
+				host,
+				COALESCE(MAX(last_crawled), '1970-01-01'::timestamp) as latest_attempt
+			FROM snapshots
+			WHERE url ~ '^gemini://[^/]+/?$' AND mimetype = 'text/gemini'
+			GROUP BY url, host
+		),
+		root_urls_with_content AS (
+			SELECT DISTINCT
+				la.url,
+				la.host,
+				la.latest_attempt
+			FROM latest_attempts la
+			JOIN snapshots s ON s.url = la.url 
+			WHERE (s.gemtext IS NOT NULL OR s.data IS NOT NULL)
+				AND s.response_code BETWEEN 20 AND 29
+		),
+		eligible_urls AS (
+			SELECT 
+				url,
+				host,
+				latest_attempt
+			FROM root_urls_with_content
+			WHERE latest_attempt < $1
+		),
+		ranked_urls AS (
+			SELECT
+				url,
+				host,
+				latest_attempt,
+				ROW_NUMBER() OVER (PARTITION BY host ORDER BY RANDOM()) as rank
+			FROM eligible_urls
+		)
+		SELECT url, host
+		FROM ranked_urls
+		WHERE rank = 1
+		ORDER BY RANDOM()
+		LIMIT $2
     `
 )