diff --git a/gemini/persistence.go b/gemini/persistence.go index fea48ce..d10abd5 100644 --- a/gemini/persistence.go +++ b/gemini/persistence.go @@ -57,16 +57,16 @@ func SaveSnapshotToDB(tx *sqlx.Tx, s *Snapshot) error { return nil } -func SaveLinkToDB(tx *sqlx.Tx, s *Snapshot) error { +func SaveLinksToDB(tx *sqlx.Tx, snapshots []*Snapshot) error { query := ` INSERT INTO snapshots (uid, url, host, timestamp, mimetype, data, gemtext, links, lang, response_code, error) VALUES (:uid, :url, :host, :timestamp, :mimetype, :data, :gemtext, :links, :lang, :response_code, :error) ON CONFLICT (uid) DO NOTHING ` - _, err := tx.NamedExec(query, s) + _, err := tx.NamedExec(query, snapshots) if err != nil { - logging.LogError("[%s] [%s] Error upserting snapshot: %w", s.URL, s.MimeType.String, err) - return fmt.Errorf("DB error: %w", err) // Return the error instead of panicking + logging.LogError("Error batch inserting snapshots: %w", err) + return fmt.Errorf("DB error: %w", err) } return nil } diff --git a/gemini/worker.go b/gemini/worker.go index 81807fc..20125c9 100644 --- a/gemini/worker.go +++ b/gemini/worker.go @@ -84,16 +84,28 @@ func workOnSnapshot(id int, tx *sqlx.Tx, s *Snapshot) (err error) { return fmt.Errorf("[%d] DB Error: %w", id, err) } - // Store links + // Store links in batch if s.Links != nil { + var batchSnapshots []*Snapshot + timestamp := null.TimeFrom(time.Now()) + for _, link := range *s.Links { - newSnapshot := Snapshot{UID: uid.UID(), URL: link, Host: link.Hostname, Timestamp: null.TimeFrom(time.Now())} if shouldPersistURL(tx, link) { - logging.LogDebug("[%d] Saving link %s", id, link) - err = SaveLinkToDB(tx, &newSnapshot) - if err != nil { - return fmt.Errorf("[%d] DB Error: %w", id, err) + newSnapshot := &Snapshot{ + UID: uid.UID(), + URL: link, + Host: link.Hostname, + Timestamp: timestamp, } + batchSnapshots = append(batchSnapshots, newSnapshot) + } + } + + if len(batchSnapshots) > 0 { + logging.LogDebug("[%d] Batch saving %d links", id, len(batchSnapshots)) + err = SaveLinksToDB(tx, batchSnapshots) + if err != nil { + return fmt.Errorf("[%d] DB Error: %w", id, err) } } }