Lint fixes.
This commit is contained in:
@@ -67,7 +67,7 @@ func SaveToFile(rootPath string, s *Snapshot, done chan struct{}) {
|
||||
urlPath := s.URL.Path
|
||||
// If path is empty, add `index.gmi` as the file to save
|
||||
if urlPath == "" || urlPath == "." {
|
||||
urlPath = fmt.Sprintf("index.gmi")
|
||||
urlPath = "index.gmi"
|
||||
}
|
||||
// If path ends with '/' then add index.gmi for the
|
||||
// directory to be created.
|
||||
|
||||
@@ -5,32 +5,10 @@ import (
|
||||
"fmt"
|
||||
"gemini-grc/logging"
|
||||
"net/url"
|
||||
gourl "net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func isGeminiURL(url string) bool {
|
||||
_, err := gourl.Parse(url)
|
||||
if err != nil {
|
||||
logging.LogWarn("[%s] Invalid URL: %v", url, err)
|
||||
return false
|
||||
}
|
||||
return strings.HasPrefix(url, "gemini://")
|
||||
}
|
||||
|
||||
func parseLinks(s Snapshot, queue chan string) {
|
||||
for _, link := range *s.Links {
|
||||
if strings.HasPrefix(link.Full, "gemini://") {
|
||||
go func(link GeminiUrl) {
|
||||
// fmt.Printf("LINK: %s\n", link)
|
||||
queue <- link.Full
|
||||
}(link)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func checkGeminiStatusCode(code int) error {
|
||||
switch {
|
||||
case code == 20:
|
||||
|
||||
@@ -62,10 +62,9 @@ func ConnectAndGetData(url string) ([]byte, error) {
|
||||
}
|
||||
// Make sure we always close the connection.
|
||||
defer func() {
|
||||
err := conn.Close()
|
||||
if err != nil {
|
||||
// Do nothing! Connection will timeout eventually if still open somehow.
|
||||
}
|
||||
// No need to handle error:
|
||||
// Connection will timeout eventually if still open somehow.
|
||||
conn.Close()
|
||||
}()
|
||||
|
||||
// Set read and write timeouts on the TCP connection.
|
||||
@@ -80,7 +79,7 @@ func ConnectAndGetData(url string) ([]byte, error) {
|
||||
|
||||
// Perform the TLS handshake
|
||||
tlsConfig := &tls.Config{
|
||||
InsecureSkipVerify: true, // Accept all TLS certs, even if insecure.
|
||||
InsecureSkipVerify: true, // Accept all TLS certs, even if insecure.
|
||||
ServerName: parsedUrl.Hostname(), // SNI should not include port
|
||||
// MinVersion: tls.VersionTLS12, // Use a minimum TLS version. Warning breaks a lot of sites.
|
||||
}
|
||||
@@ -141,7 +140,6 @@ func Visit(s *Snapshot) {
|
||||
if pageData.Data != nil {
|
||||
s.Data = null.ValueFrom(pageData.Data)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Update given snapshot with the
|
||||
|
||||
@@ -20,6 +20,7 @@ func EnsureValidUTF8(input []byte) (string, error) {
|
||||
charmap.Windows1252.NewDecoder(), // Then try Windows-1252, etc
|
||||
// TODO: Try more encodings?
|
||||
}
|
||||
// First successful conversion wins
|
||||
for _, encoding := range encodings {
|
||||
reader := transform.NewReader(bytes.NewReader(inputNoNull), encoding)
|
||||
result, err := io.ReadAll(reader)
|
||||
|
||||
@@ -57,7 +57,7 @@ func RobotMatch(s *Snapshot) bool {
|
||||
logging.LogDebug("Checking robots.txt cache for %s", s.URL.String())
|
||||
key := fmt.Sprintf("%s:%d", s.Host, s.URL.Port)
|
||||
v, ok := RobotsCache.Load(key)
|
||||
if ok == false {
|
||||
if !ok {
|
||||
// First time check, populate robot cache
|
||||
logging.LogDebug("No robots.txt entry, populating cache for %s", s.URL.String())
|
||||
disallowedURLs := populateBlacklist(key)
|
||||
|
||||
@@ -58,11 +58,11 @@ func runWorker(id int, db *sqlx.DB) {
|
||||
logging.LogInfo("[%d] Starting %d/%d %s", id, i+1, total, s.URL)
|
||||
err = workOnSnapshot(id, tx, &s)
|
||||
if err != nil {
|
||||
logging.LogError("[%d] [%s] Error %w", id, s.URL, err)
|
||||
logging.LogError("[%d] [%s] Unexpected Error %w", id, s.URL, err)
|
||||
util.PrintStackAndPanic(err)
|
||||
}
|
||||
if s.Error.Valid {
|
||||
logging.LogWarn("[%d] [%s] Error: %v", id, s.URL, fmt.Errorf(s.Error.String))
|
||||
logging.LogWarn("[%d] [%s] Worker Error: %v", id, s.URL, s.Error.String)
|
||||
}
|
||||
logging.LogDebug("[%d] Done %d/%d.", id, i, total)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user