Simplify IP pool and convert it to host pool

common/errors.go

@@ -29,7 +29,7 @@ func NewErrGeminiStatusCode(code int, header string) error {
 	case code >= 60 && code < 70:
 		msg = "TLS error"
 	default:
-		msg = "unexpected Status code"
+		msg = "unexpected status code"
 	}
 	return &GeminiError{
 		Msg:    msg,
@@ -51,8 +51,6 @@ var (
 	ErrUTF8Parse    = errors.New("UTF-8 parse error")
 	ErrTextParse    = errors.New("text parse error")
 
-	ErrBlacklistMatches = errors.New("url matches blacklist")
-
 	ErrNetwork                        = errors.New("network error")
 	ErrNetworkDNS                     = errors.New("network DNS error")
 	ErrNetworkTLS                     = errors.New("network TLS error")
@@ -60,8 +58,7 @@ var (
 	ErrNetworkCannotWrite             = errors.New("network error - cannot write")
 	ErrNetworkResponseSizeExceededMax = errors.New("network error - response size exceeded maximum size")
 
-	ErrDatabase     = errors.New("database error")
+	ErrDatabase = errors.New("database error")
-	ErrDatabaseScan = errors.New("database scan error")
 )
 
 // We could have used a map for speed, but
@@ -78,8 +75,6 @@ var knownErrors = []error{ //nolint:gochecknoglobals
 	ErrGeminiResponseHeader,
 	ErrGeminiRedirect,
 
-	ErrBlacklistMatches,
-
 	ErrURLParse,
 	ErrURLDecode,
 	ErrUTF8Parse,
@@ -93,7 +88,6 @@ var knownErrors = []error{ //nolint:gochecknoglobals
 	ErrNetworkResponseSizeExceededMax,
 
 	ErrDatabase,
-	ErrDatabaseScan,
 }
 
 func IsKnownError(err error) bool {

common/errors_test.go

@@ -1,25 +1,24 @@
-package common_test
+package common
 
 import (
 	"errors"
 	"fmt"
-	"gemini-grc/common"
 	"testing"
 )
 
 func TestErrGemini(t *testing.T) {
 	t.Parallel()
-	err := common.NewErrGeminiStatusCode(50, "50 server error")
+	err := NewErrGeminiStatusCode(50, "50 server error")
-	if !errors.As(err, new(*common.GeminiError)) {
+	if !errors.As(err, new(*GeminiError)) {
 		t.Errorf("TestErrGemini fail")
 	}
 }
 
 func TestErrGeminiWrapped(t *testing.T) {
 	t.Parallel()
-	err := common.NewErrGeminiStatusCode(50, "50 server error")
+	err := NewErrGeminiStatusCode(50, "50 server error")
 	errWrapped := fmt.Errorf("%w wrapped", err)
-	if !errors.As(errWrapped, new(*common.GeminiError)) {
+	if !errors.As(errWrapped, new(*GeminiError)) {
 		t.Errorf("TestErrGeminiWrapped fail")
 	}
 }

common/gemini_url.go

@@ -26,11 +26,11 @@ func (u *URL) Scan(value interface{}) error {
 	}
 	b, ok := value.(string)
 	if !ok {
-		return fmt.Errorf("%w: expected string, got %T", ErrDatabaseScan, value)
+		return fmt.Errorf("failed to scan GeminiUrl: expected string, got %T", value)
 	}
 	parsedURL, err := ParseURLNoNormalize(b, "")
 	if err != nil {
-		err = fmt.Errorf("%w: failed to scan GeminiUrl %s: %v", ErrDatabaseScan, b, err)
+		err = fmt.Errorf("failed to scan GeminiUrl %s: %v", b, err)
 		return err
 	}
 	*u = *parsedURL
@@ -75,13 +75,6 @@ func ParseURLNoNormalize(input string, descr string) (*URL, error) {
 		return nil, fmt.Errorf("%w: Input %s GeminiError %w", ErrURLParse, input, err)
 	}
 	full := fmt.Sprintf("%s://%s:%d%s", protocol, hostname, port, urlPath)
-	// full field should also contain query params and url fragments
-	if u.RawQuery != "" {
-		full += "?" + u.RawQuery
-	}
-	if u.Fragment != "" {
-		full += "#" + u.Fragment
-	}
 	return &URL{Protocol: protocol, Hostname: hostname, Port: port, Path: urlPath, Descr: descr, Full: full}, nil
 }
 
@@ -105,13 +98,6 @@ func ParseURL(input string, descr string) (*URL, error) {
 		return nil, fmt.Errorf("%w: Input %s GeminiError %w", ErrURLParse, input, err)
 	}
 	full := fmt.Sprintf("%s://%s:%d%s", protocol, hostname, port, urlPath)
-	// full field should also contain query params and url fragments
-	if u.RawQuery != "" {
-		full += "?" + u.RawQuery
-	}
-	if u.Fragment != "" {
-		full += "#" + u.Fragment
-	}
 	return &URL{Protocol: protocol, Hostname: hostname, Port: port, Path: urlPath, Descr: descr, Full: full}, nil
 }
 

common/gemini_url_test.go

@@ -1,7 +1,6 @@
-package common_test
+package common
 
 import (
-	"gemini-grc/common"
 	"reflect"
 	"testing"
 )
@@ -9,7 +8,7 @@ import (
 func TestParseURL(t *testing.T) {
 	t.Parallel()
 	input := "gemini://caolan.uk/cgi-bin/weather.py/wxfcs/3162"
-	parsed, err := common.ParseURL(input, "")
+	parsed, err := ParseURL(input, "")
 	value, _ := parsed.Value()
 	if err != nil || !(value == "gemini://caolan.uk:1965/cgi-bin/weather.py/wxfcs/3162") {
 		t.Errorf("fail: %s", parsed)
@@ -18,7 +17,7 @@ func TestParseURL(t *testing.T) {
 
 func TestDeriveAbsoluteURL_abs_url_input(t *testing.T) {
 	t.Parallel()
-	currentURL := common.URL{
+	currentURL := URL{
 		Protocol: "gemini",
 		Hostname: "smol.gr",
 		Port:     1965,
@@ -27,11 +26,11 @@ func TestDeriveAbsoluteURL_abs_url_input(t *testing.T) {
 		Full:     "gemini://smol.gr:1965/a/b",
 	}
 	input := "gemini://a.b/c"
-	output, err := common.DeriveAbsoluteURL(currentURL, input)
+	output, err := DeriveAbsoluteURL(currentURL, input)
 	if err != nil {
 		t.Errorf("fail: %v", err)
 	}
-	expected := &common.URL{
+	expected := &URL{
 		Protocol: "gemini",
 		Hostname: "a.b",
 		Port:     1965,
@@ -47,7 +46,7 @@ func TestDeriveAbsoluteURL_abs_url_input(t *testing.T) {
 
 func TestDeriveAbsoluteURL_abs_path_input(t *testing.T) {
 	t.Parallel()
-	currentURL := common.URL{
+	currentURL := URL{
 		Protocol: "gemini",
 		Hostname: "smol.gr",
 		Port:     1965,
@@ -56,11 +55,11 @@ func TestDeriveAbsoluteURL_abs_path_input(t *testing.T) {
 		Full:     "gemini://smol.gr:1965/a/b",
 	}
 	input := "/c"
-	output, err := common.DeriveAbsoluteURL(currentURL, input)
+	output, err := DeriveAbsoluteURL(currentURL, input)
 	if err != nil {
 		t.Errorf("fail: %v", err)
 	}
-	expected := &common.URL{
+	expected := &URL{
 		Protocol: "gemini",
 		Hostname: "smol.gr",
 		Port:     1965,
@@ -76,7 +75,7 @@ func TestDeriveAbsoluteURL_abs_path_input(t *testing.T) {
 
 func TestDeriveAbsoluteURL_rel_path_input(t *testing.T) {
 	t.Parallel()
-	currentURL := common.URL{
+	currentURL := URL{
 		Protocol: "gemini",
 		Hostname: "smol.gr",
 		Port:     1965,
@@ -85,11 +84,11 @@ func TestDeriveAbsoluteURL_rel_path_input(t *testing.T) {
 		Full:     "gemini://smol.gr:1965/a/b",
 	}
 	input := "c/d"
-	output, err := common.DeriveAbsoluteURL(currentURL, input)
+	output, err := DeriveAbsoluteURL(currentURL, input)
 	if err != nil {
 		t.Errorf("fail: %v", err)
 	}
-	expected := &common.URL{
+	expected := &URL{
 		Protocol: "gemini",
 		Hostname: "smol.gr",
 		Port:     1965,
@@ -106,7 +105,7 @@ func TestDeriveAbsoluteURL_rel_path_input(t *testing.T) {
 func TestNormalizeURLSlash(t *testing.T) {
 	t.Parallel()
 	input := "gemini://uscoffings.net/retro-computing/magazines/"
-	normalized, _ := common.NormalizeURL(input)
+	normalized, _ := NormalizeURL(input)
 	output := normalized.String()
 	expected := input
 	pass := reflect.DeepEqual(output, expected)
@@ -118,7 +117,7 @@ func TestNormalizeURLSlash(t *testing.T) {
 func TestNormalizeURLNoSlash(t *testing.T) {
 	t.Parallel()
 	input := "gemini://uscoffings.net/retro-computing/magazines"
-	normalized, _ := common.NormalizeURL(input)
+	normalized, _ := NormalizeURL(input)
 	output := normalized.String()
 	expected := input
 	pass := reflect.DeepEqual(output, expected)
@@ -130,7 +129,7 @@ func TestNormalizeURLNoSlash(t *testing.T) {
 func TestNormalizeMultiSlash(t *testing.T) {
 	t.Parallel()
 	input := "gemini://uscoffings.net/retro-computing/////////a///magazines"
-	normalized, _ := common.NormalizeURL(input)
+	normalized, _ := NormalizeURL(input)
 	output := normalized.String()
 	expected := "gemini://uscoffings.net/retro-computing/a/magazines"
 	pass := reflect.DeepEqual(output, expected)
@@ -142,7 +141,7 @@ func TestNormalizeMultiSlash(t *testing.T) {
 func TestNormalizeTrailingSlash(t *testing.T) {
 	t.Parallel()
 	input := "gemini://uscoffings.net/"
-	normalized, _ := common.NormalizeURL(input)
+	normalized, _ := NormalizeURL(input)
 	output := normalized.String()
 	expected := "gemini://uscoffings.net/"
 	pass := reflect.DeepEqual(output, expected)
@@ -154,7 +153,7 @@ func TestNormalizeTrailingSlash(t *testing.T) {
 func TestNormalizeNoTrailingSlash(t *testing.T) {
 	t.Parallel()
 	input := "gemini://uscoffings.net"
-	normalized, _ := common.NormalizeURL(input)
+	normalized, _ := NormalizeURL(input)
 	output := normalized.String()
 	expected := "gemini://uscoffings.net"
 	pass := reflect.DeepEqual(output, expected)
@@ -166,7 +165,7 @@ func TestNormalizeNoTrailingSlash(t *testing.T) {
 func TestNormalizeTrailingSlashPath(t *testing.T) {
 	t.Parallel()
 	input := "gemini://uscoffings.net/a/"
-	normalized, _ := common.NormalizeURL(input)
+	normalized, _ := NormalizeURL(input)
 	output := normalized.String()
 	expected := "gemini://uscoffings.net/a/"
 	pass := reflect.DeepEqual(output, expected)
@@ -178,7 +177,7 @@ func TestNormalizeTrailingSlashPath(t *testing.T) {
 func TestNormalizeNoTrailingSlashPath(t *testing.T) {
 	t.Parallel()
 	input := "gemini://uscoffings.net/a"
-	normalized, _ := common.NormalizeURL(input)
+	normalized, _ := NormalizeURL(input)
 	output := normalized.String()
 	expected := "gemini://uscoffings.net/a"
 	pass := reflect.DeepEqual(output, expected)
@@ -190,7 +189,7 @@ func TestNormalizeNoTrailingSlashPath(t *testing.T) {
 func TestNormalizeDot(t *testing.T) {
 	t.Parallel()
 	input := "gemini://uscoffings.net/retro-computing/./././////a///magazines"
-	normalized, _ := common.NormalizeURL(input)
+	normalized, _ := NormalizeURL(input)
 	output := normalized.String()
 	expected := "gemini://uscoffings.net/retro-computing/a/magazines"
 	pass := reflect.DeepEqual(output, expected)
@@ -202,7 +201,7 @@ func TestNormalizeDot(t *testing.T) {
 func TestNormalizePort(t *testing.T) {
 	t.Parallel()
 	input := "gemini://uscoffings.net:1965/a"
-	normalized, _ := common.NormalizeURL(input)
+	normalized, _ := NormalizeURL(input)
 	output := normalized.String()
 	expected := "gemini://uscoffings.net/a"
 	pass := reflect.DeepEqual(output, expected)
@@ -214,38 +213,11 @@ func TestNormalizePort(t *testing.T) {
 func TestNormalizeURL(t *testing.T) {
 	t.Parallel()
 	input := "gemini://chat.gemini.lehmann.cx:11965/"
-	normalized, _ := common.NormalizeURL(input)
+	normalized, _ := NormalizeURL(input)
 	output := normalized.String()
 	expected := "gemini://chat.gemini.lehmann.cx:11965/"
 	pass := reflect.DeepEqual(output, expected)
 	if !pass {
 		t.Errorf("fail: %#v != %#v", output, expected)
 	}
-
-	input = "gemini://chat.gemini.lehmann.cx:11965/index?a=1&b=c"
-	normalized, _ = common.NormalizeURL(input)
-	output = normalized.String()
-	expected = "gemini://chat.gemini.lehmann.cx:11965/index?a=1&b=c"
-	pass = reflect.DeepEqual(output, expected)
-	if !pass {
-		t.Errorf("fail: %#v != %#v", output, expected)
-	}
-
-	input = "gemini://chat.gemini.lehmann.cx:11965/index#1"
-	normalized, _ = common.NormalizeURL(input)
-	output = normalized.String()
-	expected = "gemini://chat.gemini.lehmann.cx:11965/index#1"
-	pass = reflect.DeepEqual(output, expected)
-	if !pass {
-		t.Errorf("fail: %#v != %#v", output, expected)
-	}
-
-	input = "gemini://gemi.dev/cgi-bin/xkcd.cgi?1494"
-	normalized, _ = common.NormalizeURL(input)
-	output = normalized.String()
-	expected = "gemini://gemi.dev/cgi-bin/xkcd.cgi?1494"
-	pass = reflect.DeepEqual(output, expected)
-	if !pass {
-		t.Errorf("fail: %#v != %#v", output, expected)
-	}
 }

config/config.go

@@ -18,7 +18,6 @@ const (
 	EnvPanicOnUnexpectedError = "PANIC_ON_UNEXPECTED_ERROR"
 	EnvBlacklistPath          = "BLACKLIST_PATH"
 	EnvDryRun                 = "DRY_RUN"
-	EnvPrintWorkerStatus      = "PRINT_WORKER_STATUS"
 )
 
 // Config holds the application configuration loaded from environment variables.
@@ -31,7 +30,6 @@ type Config struct {
 	PanicOnUnexpectedError bool          // Panic on unexpected errors when visiting a URL
 	BlacklistPath          string        // File that has blacklisted strings of "host:port"
 	DryRun                 bool          // If false, don't write to disk
-	PrintWorkerStatus      bool          // If false, don't print worker status table
 }
 
 var CONFIG Config //nolint:gochecknoglobals
@@ -138,14 +136,6 @@ func GetConfig() *Config {
 			config.DryRun = val
 			return nil
 		},
-		EnvPrintWorkerStatus: func(v string) error {
-			val, err := parseBool(EnvPrintWorkerStatus, v)
-			if err != nil {
-				return err
-			}
-			config.PrintWorkerStatus = val
-			return nil
-		},
 	}
 
 	// Process each environment variable