Enhance crawler with seed list and SQL utilities

Add seedList module for URL initialization, comprehensive SQL utilities for database analysis, and update project configuration.

hostPool/hostPool.go

@@ -8,7 +8,8 @@ import (
 
 	"gemini-grc/common/contextlog"
 	"gemini-grc/contextutil"
-	"gemini-grc/logging"
+	"git.antanst.com/antanst/logging"
+	"git.antanst.com/antanst/xerrors"
 )
 
 var hostPool = HostPool{hostnames: make(map[string]struct{})}
@@ -20,18 +21,13 @@ type HostPool struct {
 
 // RemoveHostFromPool removes a host from the pool with context awareness
 func RemoveHostFromPool(ctx context.Context, key string) {
-	// Create a hostPool-specific context
 	hostCtx := contextutil.ContextWithComponent(ctx, "hostPool")
-
-	contextlog.LogDebugWithContext(hostCtx, logging.GetSlogger(), "Removing host %s from pool", key)
-
 	hostPool.lock.Lock()
 	delete(hostPool.hostnames, key)
 	hostPool.lock.Unlock()
 
 	// Add some jitter
 	time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond)
-
 	contextlog.LogDebugWithContext(hostCtx, logging.GetSlogger(), "Host %s removed from pool", key)
 }
 
@@ -41,18 +37,18 @@ func AddHostToHostPool(ctx context.Context, key string) error {
 	// Create a hostPool-specific context
 	hostCtx := contextutil.ContextWithComponent(ctx, "hostPool")
 
-	contextlog.LogDebugWithContext(hostCtx, logging.GetSlogger(), "Adding host %s to pool", key)
-
 	// Use a ticker to periodically check if we can add the host
 	ticker := time.NewTicker(500 * time.Millisecond)
 	defer ticker.Stop()
 
+	// We continuously poll the pool,
+	// and if the host isn't already
+	// there, we add it.
 	for {
 		// Check if context is done before attempting to acquire lock
 		select {
 		case <-ctx.Done():
-			contextlog.LogDebugWithContext(hostCtx, logging.GetSlogger(), "Context canceled while waiting to add host %s", key)
-			return ctx.Err()
+			return xerrors.NewSimpleError(ctx.Err())
 		default:
 			// Continue with attempt to add host
 		}
@@ -67,15 +63,13 @@ func AddHostToHostPool(ctx context.Context, key string) error {
 		}
 		hostPool.lock.Unlock()
 
-		contextlog.LogDebugWithContext(hostCtx, logging.GetSlogger(), "Host %s busy, waiting...", key)
-
 		// Wait for next tick or context cancellation
 		select {
 		case <-ticker.C:
 			// Try again on next tick
 		case <-ctx.Done():
 			contextlog.LogDebugWithContext(hostCtx, logging.GetSlogger(), "Context canceled while waiting for host %s", key)
-			return ctx.Err()
+			return xerrors.NewSimpleError(ctx.Err())
 		}
 	}
 }