Enhance crawler with seed list and SQL utilities

Add seedList module for URL initialization, comprehensive SQL utilities for database analysis, and update project configuration.
This commit is contained in:
antanst
2025-06-16 12:29:33 +03:00
parent dfb050588c
commit 37d5e7cd78
37 changed files with 742 additions and 682 deletions

View File

@@ -12,7 +12,7 @@ import (
commonErrors "gemini-grc/common/errors"
"gemini-grc/config"
"gemini-grc/logging"
"git.antanst.com/antanst/logging"
"git.antanst.com/antanst/xerrors"
)

View File

@@ -18,7 +18,7 @@ import (
_url "gemini-grc/common/url"
"gemini-grc/config"
"gemini-grc/contextutil"
"gemini-grc/logging"
"git.antanst.com/antanst/logging"
"git.antanst.com/antanst/xerrors"
"github.com/guregu/null/v5"
)
@@ -30,8 +30,6 @@ func VisitWithContext(ctx context.Context, url string) (*snapshot.Snapshot, erro
// Create a gopher-specific context with the "gopher" component
gopherCtx := contextutil.ContextWithComponent(ctx, "gopher")
contextlog.LogDebugWithContext(gopherCtx, logging.GetSlogger(), "Visiting Gopher URL: %s", url)
if !config.CONFIG.GopherEnable {
contextlog.LogDebugWithContext(gopherCtx, logging.GetSlogger(), "Gopher protocol is disabled")
return nil, nil