Enhance crawler with seed list and SQL utilities
Add seedList module for URL initialization, comprehensive SQL utilities for database analysis, and update project configuration.
This commit is contained in:
@@ -18,7 +18,7 @@ import (
|
||||
_url "gemini-grc/common/url"
|
||||
"gemini-grc/config"
|
||||
"gemini-grc/contextutil"
|
||||
"gemini-grc/logging"
|
||||
"git.antanst.com/antanst/logging"
|
||||
"git.antanst.com/antanst/xerrors"
|
||||
"github.com/guregu/null/v5"
|
||||
)
|
||||
@@ -30,8 +30,6 @@ func VisitWithContext(ctx context.Context, url string) (*snapshot.Snapshot, erro
|
||||
// Create a gopher-specific context with the "gopher" component
|
||||
gopherCtx := contextutil.ContextWithComponent(ctx, "gopher")
|
||||
|
||||
contextlog.LogDebugWithContext(gopherCtx, logging.GetSlogger(), "Visiting Gopher URL: %s", url)
|
||||
|
||||
if !config.CONFIG.GopherEnable {
|
||||
contextlog.LogDebugWithContext(gopherCtx, logging.GetSlogger(), "Gopher protocol is disabled")
|
||||
return nil, nil
|
||||
|
||||
Reference in New Issue
Block a user