antanst/gemini-grc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use go_errors library everywhere.

Browse Source
This commit is contained in:
antanst
2025-02-26 13:31:46 +02:00
parent be38104f05
commit 5b84960c5a
17 changed files with 66 additions and 353 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
common/blackList/blacklist.go
View File

@@ -7,8 +7,8 @@ import (
"strings"
"gemini-grc/config"
"gemini-grc/errors"
"gemini-grc/logging"
"github.com/antanst/go_errors"
)
var Blacklist []regexp.Regexp //nolint:gochecknoglobals
@@ -21,7 +21,7 @@ func LoadBlacklist() error {
data, err := os.ReadFile(config.CONFIG.BlacklistPath)
if err != nil {
Blacklist = []regexp.Regexp{}
return errors.NewError(fmt.Errorf("could not load Blacklist file: %w", err))
return go_errors.NewError(fmt.Errorf("could not load Blacklist file: %w", err))
}
lines := strings.Split(string(data), "\n")
@@ -32,7 +32,7 @@ func LoadBlacklist() error {
}
regex, err := regexp.Compile(line)
if err != nil {
return errors.NewError(fmt.Errorf("could not compile Blacklist line %s: %w", line, err))
return go_errors.NewError(fmt.Errorf("could not compile Blacklist line %s: %w", line, err))
}
Blacklist = append(Blacklist, *regex)

4
common/errors/errors.go
View File

@@ -3,7 +3,7 @@ package errors
import (
"fmt"
"gemini-grc/errors"
"github.com/antanst/go_errors"
)
// HostError is an error encountered while
@@ -30,7 +30,7 @@ func IsHostError(err error) bool {
return false
}
var asError *HostError
return errors.As(err, &asError)
return go_errors.As(err, &asError)
}
// Sentinel errors used for their string message primarily.

4
common/snapshot/snapshot.go
View File

@@ -5,7 +5,7 @@ import (
"gemini-grc/common/linkList"
commonUrl "gemini-grc/common/url"
"gemini-grc/errors"
"github.com/antanst/go_errors"
"github.com/guregu/null/v5"
)
@@ -27,7 +27,7 @@ type Snapshot struct {
func SnapshotFromURL(u string, normalize bool) (*Snapshot, error) {
url, err := commonUrl.ParseURL(u, "", normalize)
if err != nil {
return nil, errors.NewError(err)
return nil, go_errors.NewError(err)
}
newSnapshot := Snapshot{
URL: *url,

16
common/url/url.go
View File

@@ -9,7 +9,7 @@ import (
"strconv"
"strings"
"gemini-grc/errors"
"github.com/antanst/go_errors"
)
type URL struct {
@@ -29,7 +29,7 @@ func (u *URL) Scan(value interface{}) error {
}
b, ok := value.(string)
if !ok {
return errors.NewFatalError(fmt.Errorf("database scan error: expected string, got %T", value))
return go_errors.NewFatalError(fmt.Errorf("database scan error: expected string, got %T", value))
}
parsedURL, err := ParseURL(b, "", false)
if err != nil {
@@ -82,7 +82,7 @@ func ParseURL(input string, descr string, normalize bool) (*URL, error) {
} else {
u, err = url.Parse(input)
if err != nil {
return nil, errors.NewError(fmt.Errorf("error parsing URL: %w: %s", err, input))
return nil, go_errors.NewError(fmt.Errorf("error parsing URL: %w: %s", err, input))
}
}
protocol := u.Scheme
@@ -99,7 +99,7 @@ func ParseURL(input string, descr string, normalize bool) (*URL, error) {
}
port, err := strconv.Atoi(strPort)
if err != nil {
return nil, errors.NewError(fmt.Errorf("error parsing URL: %w: %s", err, input))
return nil, go_errors.NewError(fmt.Errorf("error parsing URL: %w: %s", err, input))
}
full := fmt.Sprintf("%s://%s:%d%s", protocol, hostname, port, urlPath)
// full field should also contain query params and url fragments
@@ -145,13 +145,13 @@ func NormalizeURL(rawURL string) (*url.URL, error) {
// Parse the URL
u, err := url.Parse(rawURL)
if err != nil {
return nil, errors.NewError(fmt.Errorf("error normalizing URL: %w: %s", err, rawURL))
return nil, go_errors.NewError(fmt.Errorf("error normalizing URL: %w: %s", err, rawURL))
}
if u.Scheme == "" {
return nil, errors.NewError(fmt.Errorf("error normalizing URL: No scheme: %s", rawURL))
return nil, go_errors.NewError(fmt.Errorf("error normalizing URL: No scheme: %s", rawURL))
}
if u.Host == "" {
return nil, errors.NewError(fmt.Errorf("error normalizing URL: No host: %s", rawURL))
return nil, go_errors.NewError(fmt.Errorf("error normalizing URL: No host: %s", rawURL))
}
// Convert scheme to lowercase
@@ -275,7 +275,7 @@ func ExtractRedirectTargetFromHeader(currentURL URL, input string) (*URL, error)
re := regexp.MustCompile(pattern)
matches := re.FindStringSubmatch(input)
if len(matches) < 2 {
return nil, errors.NewError(fmt.Errorf("error extracting redirect target from string %s", input))
return nil, go_errors.NewError(fmt.Errorf("error extracting redirect target from string %s", input))
}
newURL, err := DeriveAbsoluteURL(currentURL, matches[1])
if err != nil {

16
common/worker.go
View File

@@ -9,11 +9,11 @@ import (
"gemini-grc/common/snapshot"
url2 "gemini-grc/common/url"
_db "gemini-grc/db"
"gemini-grc/errors"
"gemini-grc/gemini"
"gemini-grc/gopher"
"gemini-grc/hostPool"
"gemini-grc/logging"
"github.com/antanst/go_errors"
"github.com/guregu/null/v5"
"github.com/jmoiron/sqlx"
)
@@ -25,12 +25,12 @@ func CrawlOneURL(db *sqlx.DB, url *string) error {
}
if !url2.IsGeminiUrl(parsedURL.String()) && !url2.IsGopherURL(parsedURL.String()) {
return errors.NewError(fmt.Errorf("error parsing URL: not a Gemini or Gopher URL: %s", parsedURL.String()))
return go_errors.NewError(fmt.Errorf("error parsing URL: not a Gemini or Gopher URL: %s", parsedURL.String()))
}
tx, err := db.Beginx()
if err != nil {
return errors.NewFatalError(err)
return go_errors.NewFatalError(err)
}
err = _db.InsertURL(tx, parsedURL.Full)
@@ -49,9 +49,9 @@ func CrawlOneURL(db *sqlx.DB, url *string) error {
// logging.LogError("Deadlock detected. Rolling back")
// time.Sleep(time.Duration(10) * time.Second)
// err := tx.Rollback()
// return errors.NewFatalError(err)
// return go_errors.NewFatalError(err)
//}
return errors.NewFatalError(err)
return go_errors.NewFatalError(err)
}
logging.LogInfo("Done")
return nil
@@ -156,7 +156,7 @@ func workOnUrl(workerID int, tx *sqlx.Tx, url string) (err error) {
isGemini := url2.IsGeminiUrl(s.URL.String())
isGopher := url2.IsGopherURL(s.URL.String())
if !isGemini && !isGopher {
return errors.NewError(fmt.Errorf("not a Gopher or Gemini URL: %s", s.URL.String()))
return go_errors.NewError(fmt.Errorf("not a Gopher or Gemini URL: %s", s.URL.String()))
}
if blackList.IsBlacklisted(s.URL.String()) {
@@ -270,14 +270,14 @@ func haveWeVisitedURL(tx *sqlx.Tx, u string) (bool, error) {
var result []bool
err := tx.Select(&result, `SELECT TRUE FROM urls WHERE url=$1`, u)
if err != nil {
return false, errors.NewFatalError(fmt.Errorf("database error: %w", err))
return false, go_errors.NewFatalError(fmt.Errorf("database error: %w", err))
}
if len(result) > 0 {
return result[0], nil
}
err = tx.Select(&result, `SELECT TRUE FROM snapshots WHERE snapshots.url=$1`, u)
if err != nil {
return false, errors.NewFatalError(fmt.Errorf("database error: %w", err))
return false, go_errors.NewFatalError(fmt.Errorf("database error: %w", err))
}
if len(result) > 0 {
return result[0], nil