Update last_crawled timestamp when skipping duplicate content and improve error handling

2025-06-18 12:02:55 +03:00
parent 349968d019
commit 55bb0d96d0
2 changed files with 5 additions and 1 deletions
--- a/common/worker.go
+++ b/common/worker.go
@@ -236,6 +236,10 @@ func WorkOnUrl(ctx context.Context, tx *sqlx.Tx, url string) (err error) {
 			return saveSnapshotAndRemoveURL(ctx, tx, s)
 		} else {
 			contextlog.LogInfoWithContext(ctx, logging.GetSlogger(), "%2d %s (but old content exists, not updating)", s.ResponseCode.ValueOrZero(), s.Error.ValueOrZero())
+			err = gemdb.Database.UpdateLastCrawled(ctx, tx, s.URL.String())
+			if err != nil {
+				return err
+			}
 			return removeURL(ctx, tx, s.URL.String())
 		}
 	} else {