Proper package in tests

common/errors.go

@@ -29,7 +29,7 @@ func NewErrGeminiStatusCode(code int, header string) error {
 	case code >= 60 && code < 70:
 		msg = "TLS error"
 	default:
-		msg = "unexpected status code"
+		msg = "unexpected Status code"
 	}
 	return &GeminiError{
 		Msg:    msg,
@@ -51,6 +51,8 @@ var (
 	ErrUTF8Parse    = errors.New("UTF-8 parse error")
 	ErrTextParse    = errors.New("text parse error")
 
+	ErrBlacklistMatches = errors.New("url matches blacklist")
+
 	ErrNetwork                        = errors.New("network error")
 	ErrNetworkDNS                     = errors.New("network DNS error")
 	ErrNetworkTLS                     = errors.New("network TLS error")
@@ -59,6 +61,7 @@ var (
 	ErrNetworkResponseSizeExceededMax = errors.New("network error - response size exceeded maximum size")
 
 	ErrDatabase     = errors.New("database error")
+	ErrDatabaseScan = errors.New("database scan error")
 )
 
 // We could have used a map for speed, but
@@ -75,6 +78,8 @@ var knownErrors = []error{ //nolint:gochecknoglobals
 	ErrGeminiResponseHeader,
 	ErrGeminiRedirect,
 
+	ErrBlacklistMatches,
+
 	ErrURLParse,
 	ErrURLDecode,
 	ErrUTF8Parse,
@@ -88,6 +93,7 @@ var knownErrors = []error{ //nolint:gochecknoglobals
 	ErrNetworkResponseSizeExceededMax,
 
 	ErrDatabase,
+	ErrDatabaseScan,
 }
 
 func IsKnownError(err error) bool {

common/errors_test.go

@@ -1,24 +1,25 @@
-package common
+package common_test
 
 import (
 	"errors"
 	"fmt"
+	"gemini-grc/common"
 	"testing"
 )
 
 func TestErrGemini(t *testing.T) {
 	t.Parallel()
-	err := NewErrGeminiStatusCode(50, "50 server error")
+	err := common.NewErrGeminiStatusCode(50, "50 server error")
-	if !errors.As(err, new(*GeminiError)) {
+	if !errors.As(err, new(*common.GeminiError)) {
 		t.Errorf("TestErrGemini fail")
 	}
 }
 
 func TestErrGeminiWrapped(t *testing.T) {
 	t.Parallel()
-	err := NewErrGeminiStatusCode(50, "50 server error")
+	err := common.NewErrGeminiStatusCode(50, "50 server error")
 	errWrapped := fmt.Errorf("%w wrapped", err)
-	if !errors.As(errWrapped, new(*GeminiError)) {
+	if !errors.As(errWrapped, new(*common.GeminiError)) {
 		t.Errorf("TestErrGeminiWrapped fail")
 	}
 }

common/gemini_url.go

@@ -26,11 +26,11 @@ func (u *URL) Scan(value interface{}) error {
 	}
 	b, ok := value.(string)
 	if !ok {
-		return fmt.Errorf("failed to scan GeminiUrl: expected string, got %T", value)
+		return fmt.Errorf("%w: expected string, got %T", ErrDatabaseScan, value)
 	}
 	parsedURL, err := ParseURLNoNormalize(b, "")
 	if err != nil {
-		err = fmt.Errorf("failed to scan GeminiUrl %s: %v", b, err)
+		err = fmt.Errorf("%w: failed to scan GeminiUrl %s: %v", ErrDatabaseScan, b, err)
 		return err
 	}
 	*u = *parsedURL
@@ -75,6 +75,13 @@ func ParseURLNoNormalize(input string, descr string) (*URL, error) {
 		return nil, fmt.Errorf("%w: Input %s GeminiError %w", ErrURLParse, input, err)
 	}
 	full := fmt.Sprintf("%s://%s:%d%s", protocol, hostname, port, urlPath)
+	// full field should also contain query params and url fragments
+	if u.RawQuery != "" {
+		full += "?" + u.RawQuery
+	}
+	if u.Fragment != "" {
+		full += "#" + u.Fragment
+	}
 	return &URL{Protocol: protocol, Hostname: hostname, Port: port, Path: urlPath, Descr: descr, Full: full}, nil
 }
 
@@ -98,6 +105,13 @@ func ParseURL(input string, descr string) (*URL, error) {
 		return nil, fmt.Errorf("%w: Input %s GeminiError %w", ErrURLParse, input, err)
 	}
 	full := fmt.Sprintf("%s://%s:%d%s", protocol, hostname, port, urlPath)
+	// full field should also contain query params and url fragments
+	if u.RawQuery != "" {
+		full += "?" + u.RawQuery
+	}
+	if u.Fragment != "" {
+		full += "#" + u.Fragment
+	}
 	return &URL{Protocol: protocol, Hostname: hostname, Port: port, Path: urlPath, Descr: descr, Full: full}, nil
 }
 

common/gemini_url_test.go

@@ -1,6 +1,7 @@
-package common
+package common_test
 
 import (
+	"gemini-grc/common"
 	"reflect"
 	"testing"
 )
@@ -8,7 +9,7 @@ import (
 func TestParseURL(t *testing.T) {
 	t.Parallel()
 	input := "gemini://caolan.uk/cgi-bin/weather.py/wxfcs/3162"
-	parsed, err := ParseURL(input, "")
+	parsed, err := common.ParseURL(input, "")
 	value, _ := parsed.Value()
 	if err != nil || !(value == "gemini://caolan.uk:1965/cgi-bin/weather.py/wxfcs/3162") {
 		t.Errorf("fail: %s", parsed)
@@ -17,7 +18,7 @@ func TestParseURL(t *testing.T) {
 
 func TestDeriveAbsoluteURL_abs_url_input(t *testing.T) {
 	t.Parallel()
-	currentURL := URL{
+	currentURL := common.URL{
 		Protocol: "gemini",
 		Hostname: "smol.gr",
 		Port:     1965,
@@ -26,11 +27,11 @@ func TestDeriveAbsoluteURL_abs_url_input(t *testing.T) {
 		Full:     "gemini://smol.gr:1965/a/b",
 	}
 	input := "gemini://a.b/c"
-	output, err := DeriveAbsoluteURL(currentURL, input)
+	output, err := common.DeriveAbsoluteURL(currentURL, input)
 	if err != nil {
 		t.Errorf("fail: %v", err)
 	}
-	expected := &URL{
+	expected := &common.URL{
 		Protocol: "gemini",
 		Hostname: "a.b",
 		Port:     1965,
@@ -46,7 +47,7 @@ func TestDeriveAbsoluteURL_abs_url_input(t *testing.T) {
 
 func TestDeriveAbsoluteURL_abs_path_input(t *testing.T) {
 	t.Parallel()
-	currentURL := URL{
+	currentURL := common.URL{
 		Protocol: "gemini",
 		Hostname: "smol.gr",
 		Port:     1965,
@@ -55,11 +56,11 @@ func TestDeriveAbsoluteURL_abs_path_input(t *testing.T) {
 		Full:     "gemini://smol.gr:1965/a/b",
 	}
 	input := "/c"
-	output, err := DeriveAbsoluteURL(currentURL, input)
+	output, err := common.DeriveAbsoluteURL(currentURL, input)
 	if err != nil {
 		t.Errorf("fail: %v", err)
 	}
-	expected := &URL{
+	expected := &common.URL{
 		Protocol: "gemini",
 		Hostname: "smol.gr",
 		Port:     1965,
@@ -75,7 +76,7 @@ func TestDeriveAbsoluteURL_abs_path_input(t *testing.T) {
 
 func TestDeriveAbsoluteURL_rel_path_input(t *testing.T) {
 	t.Parallel()
-	currentURL := URL{
+	currentURL := common.URL{
 		Protocol: "gemini",
 		Hostname: "smol.gr",
 		Port:     1965,
@@ -84,11 +85,11 @@ func TestDeriveAbsoluteURL_rel_path_input(t *testing.T) {
 		Full:     "gemini://smol.gr:1965/a/b",
 	}
 	input := "c/d"
-	output, err := DeriveAbsoluteURL(currentURL, input)
+	output, err := common.DeriveAbsoluteURL(currentURL, input)
 	if err != nil {
 		t.Errorf("fail: %v", err)
 	}
-	expected := &URL{
+	expected := &common.URL{
 		Protocol: "gemini",
 		Hostname: "smol.gr",
 		Port:     1965,
@@ -105,7 +106,7 @@ func TestDeriveAbsoluteURL_rel_path_input(t *testing.T) {
 func TestNormalizeURLSlash(t *testing.T) {
 	t.Parallel()
 	input := "gemini://uscoffings.net/retro-computing/magazines/"
-	normalized, _ := NormalizeURL(input)
+	normalized, _ := common.NormalizeURL(input)
 	output := normalized.String()
 	expected := input
 	pass := reflect.DeepEqual(output, expected)
@@ -117,7 +118,7 @@ func TestNormalizeURLSlash(t *testing.T) {
 func TestNormalizeURLNoSlash(t *testing.T) {
 	t.Parallel()
 	input := "gemini://uscoffings.net/retro-computing/magazines"
-	normalized, _ := NormalizeURL(input)
+	normalized, _ := common.NormalizeURL(input)
 	output := normalized.String()
 	expected := input
 	pass := reflect.DeepEqual(output, expected)
@@ -129,7 +130,7 @@ func TestNormalizeURLNoSlash(t *testing.T) {
 func TestNormalizeMultiSlash(t *testing.T) {
 	t.Parallel()
 	input := "gemini://uscoffings.net/retro-computing/////////a///magazines"
-	normalized, _ := NormalizeURL(input)
+	normalized, _ := common.NormalizeURL(input)
 	output := normalized.String()
 	expected := "gemini://uscoffings.net/retro-computing/a/magazines"
 	pass := reflect.DeepEqual(output, expected)
@@ -141,7 +142,7 @@ func TestNormalizeMultiSlash(t *testing.T) {
 func TestNormalizeTrailingSlash(t *testing.T) {
 	t.Parallel()
 	input := "gemini://uscoffings.net/"
-	normalized, _ := NormalizeURL(input)
+	normalized, _ := common.NormalizeURL(input)
 	output := normalized.String()
 	expected := "gemini://uscoffings.net/"
 	pass := reflect.DeepEqual(output, expected)
@@ -153,7 +154,7 @@ func TestNormalizeTrailingSlash(t *testing.T) {
 func TestNormalizeNoTrailingSlash(t *testing.T) {
 	t.Parallel()
 	input := "gemini://uscoffings.net"
-	normalized, _ := NormalizeURL(input)
+	normalized, _ := common.NormalizeURL(input)
 	output := normalized.String()
 	expected := "gemini://uscoffings.net"
 	pass := reflect.DeepEqual(output, expected)
@@ -165,7 +166,7 @@ func TestNormalizeNoTrailingSlash(t *testing.T) {
 func TestNormalizeTrailingSlashPath(t *testing.T) {
 	t.Parallel()
 	input := "gemini://uscoffings.net/a/"
-	normalized, _ := NormalizeURL(input)
+	normalized, _ := common.NormalizeURL(input)
 	output := normalized.String()
 	expected := "gemini://uscoffings.net/a/"
 	pass := reflect.DeepEqual(output, expected)
@@ -177,7 +178,7 @@ func TestNormalizeTrailingSlashPath(t *testing.T) {
 func TestNormalizeNoTrailingSlashPath(t *testing.T) {
 	t.Parallel()
 	input := "gemini://uscoffings.net/a"
-	normalized, _ := NormalizeURL(input)
+	normalized, _ := common.NormalizeURL(input)
 	output := normalized.String()
 	expected := "gemini://uscoffings.net/a"
 	pass := reflect.DeepEqual(output, expected)
@@ -189,7 +190,7 @@ func TestNormalizeNoTrailingSlashPath(t *testing.T) {
 func TestNormalizeDot(t *testing.T) {
 	t.Parallel()
 	input := "gemini://uscoffings.net/retro-computing/./././////a///magazines"
-	normalized, _ := NormalizeURL(input)
+	normalized, _ := common.NormalizeURL(input)
 	output := normalized.String()
 	expected := "gemini://uscoffings.net/retro-computing/a/magazines"
 	pass := reflect.DeepEqual(output, expected)
@@ -201,7 +202,7 @@ func TestNormalizeDot(t *testing.T) {
 func TestNormalizePort(t *testing.T) {
 	t.Parallel()
 	input := "gemini://uscoffings.net:1965/a"
-	normalized, _ := NormalizeURL(input)
+	normalized, _ := common.NormalizeURL(input)
 	output := normalized.String()
 	expected := "gemini://uscoffings.net/a"
 	pass := reflect.DeepEqual(output, expected)
@@ -213,11 +214,38 @@ func TestNormalizePort(t *testing.T) {
 func TestNormalizeURL(t *testing.T) {
 	t.Parallel()
 	input := "gemini://chat.gemini.lehmann.cx:11965/"
-	normalized, _ := NormalizeURL(input)
+	normalized, _ := common.NormalizeURL(input)
 	output := normalized.String()
 	expected := "gemini://chat.gemini.lehmann.cx:11965/"
 	pass := reflect.DeepEqual(output, expected)
 	if !pass {
 		t.Errorf("fail: %#v != %#v", output, expected)
 	}
+
+	input = "gemini://chat.gemini.lehmann.cx:11965/index?a=1&b=c"
+	normalized, _ = common.NormalizeURL(input)
+	output = normalized.String()
+	expected = "gemini://chat.gemini.lehmann.cx:11965/index?a=1&b=c"
+	pass = reflect.DeepEqual(output, expected)
+	if !pass {
+		t.Errorf("fail: %#v != %#v", output, expected)
+	}
+
+	input = "gemini://chat.gemini.lehmann.cx:11965/index#1"
+	normalized, _ = common.NormalizeURL(input)
+	output = normalized.String()
+	expected = "gemini://chat.gemini.lehmann.cx:11965/index#1"
+	pass = reflect.DeepEqual(output, expected)
+	if !pass {
+		t.Errorf("fail: %#v != %#v", output, expected)
+	}
+
+	input = "gemini://gemi.dev/cgi-bin/xkcd.cgi?1494"
+	normalized, _ = common.NormalizeURL(input)
+	output = normalized.String()
+	expected = "gemini://gemi.dev/cgi-bin/xkcd.cgi?1494"
+	pass = reflect.DeepEqual(output, expected)
+	if !pass {
+		t.Errorf("fail: %#v != %#v", output, expected)
+	}
 }

config/config.go

@@ -18,6 +18,7 @@ const (
 	EnvPanicOnUnexpectedError = "PANIC_ON_UNEXPECTED_ERROR"
 	EnvBlacklistPath          = "BLACKLIST_PATH"
 	EnvDryRun                 = "DRY_RUN"
+	EnvPrintWorkerStatus      = "PRINT_WORKER_STATUS"
 )
 
 // Config holds the application configuration loaded from environment variables.
@@ -30,6 +31,7 @@ type Config struct {
 	PanicOnUnexpectedError bool          // Panic on unexpected errors when visiting a URL
 	BlacklistPath          string        // File that has blacklisted strings of "host:port"
 	DryRun                 bool          // If false, don't write to disk
+	PrintWorkerStatus      bool          // If false, don't print worker status table
 }
 
 var CONFIG Config //nolint:gochecknoglobals
@@ -136,6 +138,14 @@ func GetConfig() *Config {
 			config.DryRun = val
 			return nil
 		},
+		EnvPrintWorkerStatus: func(v string) error {
+			val, err := parseBool(EnvPrintWorkerStatus, v)
+			if err != nil {
+				return err
+			}
+			config.PrintWorkerStatus = val
+			return nil
+		},
 	}
 
 	// Process each environment variable