feat: Batch insert links to improve database performance

gemini/persistence.go

@@ -57,16 +57,16 @@ func SaveSnapshotToDB(tx *sqlx.Tx, s *Snapshot) error {
 	return nil
 }
 
-func SaveLinkToDB(tx *sqlx.Tx, s *Snapshot) error {
+func SaveLinksToDB(tx *sqlx.Tx, snapshots []*Snapshot) error {
 	query := `
         INSERT INTO snapshots (uid, url, host, timestamp, mimetype, data, gemtext, links, lang, response_code, error)
         VALUES (:uid, :url, :host, :timestamp, :mimetype, :data, :gemtext, :links, :lang, :response_code, :error)
         ON CONFLICT (uid) DO NOTHING
     `
-	_, err := tx.NamedExec(query, s)
+	_, err := tx.NamedExec(query, snapshots)
 	if err != nil {
-		logging.LogError("[%s] [%s] Error upserting snapshot: %w", s.URL, s.MimeType.String, err)
-		return fmt.Errorf("DB error: %w", err) // Return the error instead of panicking
+		logging.LogError("Error batch inserting snapshots: %w", err)
+		return fmt.Errorf("DB error: %w", err)
 	}
 	return nil
 }

gemini/worker.go

@@ -84,16 +84,28 @@ func workOnSnapshot(id int, tx *sqlx.Tx, s *Snapshot) (err error) {
 		return fmt.Errorf("[%d] DB Error: %w", id, err)
 	}
 
-	// Store links
+	// Store links in batch
 	if s.Links != nil {
+		var batchSnapshots []*Snapshot
+		timestamp := null.TimeFrom(time.Now())
+		
 		for _, link := range *s.Links {
-			newSnapshot := Snapshot{UID: uid.UID(), URL: link, Host: link.Hostname, Timestamp: null.TimeFrom(time.Now())}
 			if shouldPersistURL(tx, link) {
-				logging.LogDebug("[%d] Saving link %s", id, link)
-				err = SaveLinkToDB(tx, &newSnapshot)
-				if err != nil {
-					return fmt.Errorf("[%d] DB Error: %w", id, err)
+				newSnapshot := &Snapshot{
+					UID: uid.UID(),
+					URL: link,
+					Host: link.Hostname,
+					Timestamp: timestamp,
 				}
+				batchSnapshots = append(batchSnapshots, newSnapshot)
+			}
+		}
+		
+		if len(batchSnapshots) > 0 {
+			logging.LogDebug("[%d] Batch saving %d links", id, len(batchSnapshots))
+			err = SaveLinksToDB(tx, batchSnapshots)
+			if err != nil {
+				return fmt.Errorf("[%d] DB Error: %w", id, err)
 			}
 		}
 	}