antanst/gemini-grc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Batch insert links to improve database performance

Browse Source
This commit is contained in:
antanst (aider)
2024-10-23 09:05:50 +03:00
parent a2a6bd200a
commit 17ef03d621
2 changed files with 22 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
gemini/persistence.go
View File

@@ -57,16 +57,16 @@ func SaveSnapshotToDB(tx *sqlx.Tx, s *Snapshot) error {
return nil return nil
} }
func SaveLinkToDB(tx *sqlx.Tx, s *Snapshot) error { func SaveLinksToDB(tx *sqlx.Tx, snapshots []*Snapshot) error {
query := ` query := `
INSERT INTO snapshots (uid, url, host, timestamp, mimetype, data, gemtext, links, lang, response_code, error) INSERT INTO snapshots (uid, url, host, timestamp, mimetype, data, gemtext, links, lang, response_code, error)
VALUES (:uid, :url, :host, :timestamp, :mimetype, :data, :gemtext, :links, :lang, :response_code, :error) VALUES (:uid, :url, :host, :timestamp, :mimetype, :data, :gemtext, :links, :lang, :response_code, :error)
ON CONFLICT (uid) DO NOTHING ON CONFLICT (uid) DO NOTHING
` `
_, err := tx.NamedExec(query, s) _, err := tx.NamedExec(query, snapshots)
if err != nil { if err != nil {
logging.LogError("[%s] [%s] Error upserting snapshot: %w", s.URL, s.MimeType.String, err) logging.LogError("Error batch inserting snapshots: %w", err)
return fmt.Errorf("DB error: %w", err) // Return the error instead of panicking return fmt.Errorf("DB error: %w", err)
} }
return nil return nil
} }

24
gemini/worker.go
View File

@@ -84,16 +84,28 @@ func workOnSnapshot(id int, tx *sqlx.Tx, s *Snapshot) (err error) {
return fmt.Errorf("[%d] DB Error: %w", id, err) return fmt.Errorf("[%d] DB Error: %w", id, err)
} }
// Store links // Store links in batch
if s.Links != nil { if s.Links != nil {
var batchSnapshots []*Snapshot
timestamp := null.TimeFrom(time.Now())
for _, link := range *s.Links { for _, link := range *s.Links {
newSnapshot := Snapshot{UID: uid.UID(), URL: link, Host: link.Hostname, Timestamp: null.TimeFrom(time.Now())}
if shouldPersistURL(tx, link) { if shouldPersistURL(tx, link) {
logging.LogDebug("[%d] Saving link %s", id, link) newSnapshot := &Snapshot{
err = SaveLinkToDB(tx, &newSnapshot) UID: uid.UID(),
if err != nil { URL: link,
return fmt.Errorf("[%d] DB Error: %w", id, err) Host: link.Hostname,
Timestamp: timestamp,
} }
batchSnapshots = append(batchSnapshots, newSnapshot)
}
}
if len(batchSnapshots) > 0 {
logging.LogDebug("[%d] Batch saving %d links", id, len(batchSnapshots))
err = SaveLinksToDB(tx, batchSnapshots)
if err != nil {
return fmt.Errorf("[%d] DB Error: %w", id, err)
} }
} }
} }