Lint fixes.

This commit is contained in:
2024-11-01 10:10:42 +02:00
parent ef3f009709
commit a0563074ed
9 changed files with 42 additions and 49 deletions

View File

@@ -67,7 +67,7 @@ func SaveToFile(rootPath string, s *Snapshot, done chan struct{}) {
urlPath := s.URL.Path
// If path is empty, add `index.gmi` as the file to save
if urlPath == "" || urlPath == "." {
urlPath = fmt.Sprintf("index.gmi")
urlPath = "index.gmi"
}
// If path ends with '/' then add index.gmi for the
// directory to be created.

View File

@@ -5,32 +5,10 @@ import (
"fmt"
"gemini-grc/logging"
"net/url"
gourl "net/url"
"regexp"
"strconv"
"strings"
)
func isGeminiURL(url string) bool {
_, err := gourl.Parse(url)
if err != nil {
logging.LogWarn("[%s] Invalid URL: %v", url, err)
return false
}
return strings.HasPrefix(url, "gemini://")
}
func parseLinks(s Snapshot, queue chan string) {
for _, link := range *s.Links {
if strings.HasPrefix(link.Full, "gemini://") {
go func(link GeminiUrl) {
// fmt.Printf("LINK: %s\n", link)
queue <- link.Full
}(link)
}
}
}
func checkGeminiStatusCode(code int) error {
switch {
case code == 20:

View File

@@ -62,10 +62,9 @@ func ConnectAndGetData(url string) ([]byte, error) {
}
// Make sure we always close the connection.
defer func() {
err := conn.Close()
if err != nil {
// Do nothing! Connection will timeout eventually if still open somehow.
}
// No need to handle error:
// Connection will timeout eventually if still open somehow.
conn.Close()
}()
// Set read and write timeouts on the TCP connection.
@@ -80,7 +79,7 @@ func ConnectAndGetData(url string) ([]byte, error) {
// Perform the TLS handshake
tlsConfig := &tls.Config{
InsecureSkipVerify: true, // Accept all TLS certs, even if insecure.
InsecureSkipVerify: true, // Accept all TLS certs, even if insecure.
ServerName: parsedUrl.Hostname(), // SNI should not include port
// MinVersion: tls.VersionTLS12, // Use a minimum TLS version. Warning breaks a lot of sites.
}
@@ -141,7 +140,6 @@ func Visit(s *Snapshot) {
if pageData.Data != nil {
s.Data = null.ValueFrom(pageData.Data)
}
return
}
// Update given snapshot with the

View File

@@ -20,6 +20,7 @@ func EnsureValidUTF8(input []byte) (string, error) {
charmap.Windows1252.NewDecoder(), // Then try Windows-1252, etc
// TODO: Try more encodings?
}
// First successful conversion wins
for _, encoding := range encodings {
reader := transform.NewReader(bytes.NewReader(inputNoNull), encoding)
result, err := io.ReadAll(reader)

View File

@@ -57,7 +57,7 @@ func RobotMatch(s *Snapshot) bool {
logging.LogDebug("Checking robots.txt cache for %s", s.URL.String())
key := fmt.Sprintf("%s:%d", s.Host, s.URL.Port)
v, ok := RobotsCache.Load(key)
if ok == false {
if !ok {
// First time check, populate robot cache
logging.LogDebug("No robots.txt entry, populating cache for %s", s.URL.String())
disallowedURLs := populateBlacklist(key)

View File

@@ -58,11 +58,11 @@ func runWorker(id int, db *sqlx.DB) {
logging.LogInfo("[%d] Starting %d/%d %s", id, i+1, total, s.URL)
err = workOnSnapshot(id, tx, &s)
if err != nil {
logging.LogError("[%d] [%s] Error %w", id, s.URL, err)
logging.LogError("[%d] [%s] Unexpected Error %w", id, s.URL, err)
util.PrintStackAndPanic(err)
}
if s.Error.Valid {
logging.LogWarn("[%d] [%s] Error: %v", id, s.URL, fmt.Errorf(s.Error.String))
logging.LogWarn("[%d] [%s] Worker Error: %v", id, s.URL, s.Error.String)
}
logging.LogDebug("[%d] Done %d/%d.", id, i, total)
}