From a0563074ed4b93c1a5c1e3066cec3b05bb950685 Mon Sep 17 00:00:00 2001 From: antanst Date: Fri, 1 Nov 2024 10:10:42 +0200 Subject: [PATCH] Lint fixes. --- db/populateDB.go | 43 ++++++++++++++++++++++++++----------------- gemini/files.go | 2 +- gemini/gemini.go | 22 ---------------------- gemini/network.go | 10 ++++------ gemini/processing.go | 1 + gemini/robotmatch.go | 2 +- gemini/worker.go | 4 ++-- lint.sh | 3 +++ main.go | 4 ++++ 9 files changed, 42 insertions(+), 49 deletions(-) create mode 100755 lint.sh diff --git a/db/populateDB.go b/db/populateDB.go index a9de324..42debba 100644 --- a/db/populateDB.go +++ b/db/populateDB.go @@ -1,20 +1,29 @@ -// func PopulateDB(db *sqlx.DB) { -// // Delete all rows in the snapshots table -// db.MustExec("TRUNCATE snapshots;") +package main -// // Prepare the query for inserting a snapshot with uid, url, and timestamp -// query := `INSERT INTO snapshots(uid, url, timestamp) -// VALUES ($1, $2, $3)` +import ( + "gemini-grc/uid" + "time" -// // Calculate the timestamp for 2 days ago -// timestamp := time.Now().Add(-48 * time.Hour) + "github.com/jmoiron/sqlx" +) -// db.MustExec(query, uid.UID(), "gemini://geminiprotocol.net/", timestamp) -// db.MustExec(query, uid.UID(), "gemini://warmedal.se/~antenna", timestamp) -// db.MustExec(query, uid.UID(), "gemini://skyjake.fi/~Cosmos/", timestamp) -// db.MustExec(query, uid.UID(), "gemini://gemini.circumlunar.space/capcom/", timestamp) -// db.MustExec(query, uid.UID(), "gemini://auragem.letz.dev/", timestamp) -// db.MustExec(query, uid.UID(), "gemini://gemplex.space/", timestamp) -// db.MustExec(query, uid.UID(), "gemini://kennedy.gemi.dev/", timestamp) -// db.MustExec(query, uid.UID(), "gemini://tlgs.one/", timestamp) -// } +func PopulateDB(db *sqlx.DB) { + // Delete all rows in the snapshots table + db.MustExec("TRUNCATE snapshots;") + + // Prepare the query for inserting a snapshot with uid, url, and timestamp + query := `INSERT INTO snapshots(uid, url, timestamp) + VALUES ($1, $2, $3)` + + // Calculate the timestamp for 2 days ago + timestamp := time.Now().Add(-48 * time.Hour) + + db.MustExec(query, uid.UID(), "gemini://geminiprotocol.net/", timestamp) + db.MustExec(query, uid.UID(), "gemini://warmedal.se/~antenna", timestamp) + db.MustExec(query, uid.UID(), "gemini://skyjake.fi/~Cosmos/", timestamp) + db.MustExec(query, uid.UID(), "gemini://gemini.circumlunar.space/capcom/", timestamp) + db.MustExec(query, uid.UID(), "gemini://auragem.letz.dev/", timestamp) + db.MustExec(query, uid.UID(), "gemini://gemplex.space/", timestamp) + db.MustExec(query, uid.UID(), "gemini://kennedy.gemi.dev/", timestamp) + db.MustExec(query, uid.UID(), "gemini://tlgs.one/", timestamp) +} diff --git a/gemini/files.go b/gemini/files.go index e00581c..c2e8147 100644 --- a/gemini/files.go +++ b/gemini/files.go @@ -67,7 +67,7 @@ func SaveToFile(rootPath string, s *Snapshot, done chan struct{}) { urlPath := s.URL.Path // If path is empty, add `index.gmi` as the file to save if urlPath == "" || urlPath == "." { - urlPath = fmt.Sprintf("index.gmi") + urlPath = "index.gmi" } // If path ends with '/' then add index.gmi for the // directory to be created. diff --git a/gemini/gemini.go b/gemini/gemini.go index eb24d4e..18d48b2 100644 --- a/gemini/gemini.go +++ b/gemini/gemini.go @@ -5,32 +5,10 @@ import ( "fmt" "gemini-grc/logging" "net/url" - gourl "net/url" "regexp" "strconv" - "strings" ) -func isGeminiURL(url string) bool { - _, err := gourl.Parse(url) - if err != nil { - logging.LogWarn("[%s] Invalid URL: %v", url, err) - return false - } - return strings.HasPrefix(url, "gemini://") -} - -func parseLinks(s Snapshot, queue chan string) { - for _, link := range *s.Links { - if strings.HasPrefix(link.Full, "gemini://") { - go func(link GeminiUrl) { - // fmt.Printf("LINK: %s\n", link) - queue <- link.Full - }(link) - } - } -} - func checkGeminiStatusCode(code int) error { switch { case code == 20: diff --git a/gemini/network.go b/gemini/network.go index 4efc58b..5f48e2e 100644 --- a/gemini/network.go +++ b/gemini/network.go @@ -62,10 +62,9 @@ func ConnectAndGetData(url string) ([]byte, error) { } // Make sure we always close the connection. defer func() { - err := conn.Close() - if err != nil { - // Do nothing! Connection will timeout eventually if still open somehow. - } + // No need to handle error: + // Connection will timeout eventually if still open somehow. + conn.Close() }() // Set read and write timeouts on the TCP connection. @@ -80,7 +79,7 @@ func ConnectAndGetData(url string) ([]byte, error) { // Perform the TLS handshake tlsConfig := &tls.Config{ - InsecureSkipVerify: true, // Accept all TLS certs, even if insecure. + InsecureSkipVerify: true, // Accept all TLS certs, even if insecure. ServerName: parsedUrl.Hostname(), // SNI should not include port // MinVersion: tls.VersionTLS12, // Use a minimum TLS version. Warning breaks a lot of sites. } @@ -141,7 +140,6 @@ func Visit(s *Snapshot) { if pageData.Data != nil { s.Data = null.ValueFrom(pageData.Data) } - return } // Update given snapshot with the diff --git a/gemini/processing.go b/gemini/processing.go index aa15fd4..ca26268 100644 --- a/gemini/processing.go +++ b/gemini/processing.go @@ -20,6 +20,7 @@ func EnsureValidUTF8(input []byte) (string, error) { charmap.Windows1252.NewDecoder(), // Then try Windows-1252, etc // TODO: Try more encodings? } + // First successful conversion wins for _, encoding := range encodings { reader := transform.NewReader(bytes.NewReader(inputNoNull), encoding) result, err := io.ReadAll(reader) diff --git a/gemini/robotmatch.go b/gemini/robotmatch.go index ddce449..9524162 100644 --- a/gemini/robotmatch.go +++ b/gemini/robotmatch.go @@ -57,7 +57,7 @@ func RobotMatch(s *Snapshot) bool { logging.LogDebug("Checking robots.txt cache for %s", s.URL.String()) key := fmt.Sprintf("%s:%d", s.Host, s.URL.Port) v, ok := RobotsCache.Load(key) - if ok == false { + if !ok { // First time check, populate robot cache logging.LogDebug("No robots.txt entry, populating cache for %s", s.URL.String()) disallowedURLs := populateBlacklist(key) diff --git a/gemini/worker.go b/gemini/worker.go index 8bf6acf..71ce079 100644 --- a/gemini/worker.go +++ b/gemini/worker.go @@ -58,11 +58,11 @@ func runWorker(id int, db *sqlx.DB) { logging.LogInfo("[%d] Starting %d/%d %s", id, i+1, total, s.URL) err = workOnSnapshot(id, tx, &s) if err != nil { - logging.LogError("[%d] [%s] Error %w", id, s.URL, err) + logging.LogError("[%d] [%s] Unexpected Error %w", id, s.URL, err) util.PrintStackAndPanic(err) } if s.Error.Valid { - logging.LogWarn("[%d] [%s] Error: %v", id, s.URL, fmt.Errorf(s.Error.String)) + logging.LogWarn("[%d] [%s] Worker Error: %v", id, s.URL, s.Error.String) } logging.LogDebug("[%d] Done %d/%d.", id, i, total) } diff --git a/lint.sh b/lint.sh new file mode 100755 index 0000000..b3f1b7f --- /dev/null +++ b/lint.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +golangci-lint run diff --git a/main.go b/main.go index 15d578a..f00a0e8 100644 --- a/main.go +++ b/main.go @@ -44,6 +44,10 @@ func runApp() error { } }(db) + // if len(os.Args) > 1 { + // url := os.Args[1] + // } + // os.Exit(1) go gemini.SpawnWorkers(config.CONFIG.NumOfWorkers, db) <-sigs