antanst/gemini-grc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix infinite recrawl loop with skip-identical-content

Browse Source 
Add last_crawled timestamp tracking to fix fetchSnapshotsFromHistory()
infinite loop when SkipIdenticalContent=true. Now tracks actual crawl
attempts separately from content changes via database DEFAULT timestamps.
This commit is contained in:
antanst
2025-06-17 10:41:17 +03:00
parent 8b498a2603
commit 98d3ed6707
5 changed files with 555 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
common/worker.go
View File

@@ -202,7 +202,12 @@ func WorkOnUrl(ctx context.Context, tx *sqlx.Tx, url string) (err error) {
return err
}
if skipIdentical {
contextlog.LogInfoWithContext(ctx, logging.GetSlogger(), "Content identical to existing snapshot, skipping")
contextlog.LogInfoWithContext(ctx, logging.GetSlogger(), "Content identical to existing snapshot, recording crawl attempt")
// Record the crawl attempt to track that we processed this URL
err = gemdb.Database.RecordCrawlAttempt(ctx, tx, s)
if err != nil {
return err
}
return removeURL(ctx, tx, s.URL.String())
}