diff --git a/main.go b/main.go index c9e1b28..ea03551 100644 --- a/main.go +++ b/main.go @@ -1,12 +1,13 @@ package main import ( - main2 "gemini-grc/db" "os" "os/signal" "syscall" + "gemini-grc/common" "gemini-grc/config" + "gemini-grc/db" "gemini-grc/gemini" "gemini-grc/logging" "github.com/jmoiron/sqlx" @@ -30,7 +31,7 @@ func runApp() error { signals := make(chan os.Signal, 1) signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) - db := main2.ConnectToDB() + _db := db.ConnectToDB() defer func(db *sqlx.DB) { err := db.Close() @@ -38,17 +39,20 @@ func runApp() error { // TODO properly log & hangle error panic(err) } - }(db) + }(_db) gemini.LoadBlacklist() + common.StatusChan = make(chan common.WorkerStatus, config.CONFIG.NumOfWorkers) + // If there's an argument, visit this // URL only and don't spawn other workers if len(os.Args) > 1 { url := os.Args[1] - go gemini.RunWorkerWithTx(0, db, &url) + gemini.CrawlOneURL(_db, &url) + return nil } else { - go gemini.SpawnWorkers(config.CONFIG.NumOfWorkers, db) + go gemini.SpawnWorkers(config.CONFIG.NumOfWorkers, _db) } <-signals diff --git a/util/util.go b/util/util.go index dbf36c8..b5efb82 100644 --- a/util/util.go +++ b/util/util.go @@ -5,11 +5,12 @@ import ( "encoding/json" "fmt" "math/big" + "regexp" "runtime/debug" ) func PrintStackAndPanic(err error) { - fmt.Printf("Error %s Stack trace:\n%s", err, debug.Stack()) + fmt.Printf("PANIC Error %s Stack trace:\n%s", err, debug.Stack()) panic("PANIC") } @@ -34,3 +35,10 @@ func PrettyJson(data string) string { marshalled, _ := json.MarshalIndent(data, "", " ") return fmt.Sprintf("%s\n", marshalled) } + +// GetLinesMatchingRegex returns all lines that match given regex +func GetLinesMatchingRegex(input string, pattern string) []string { + re := regexp.MustCompile(pattern) + matches := re.FindAllString(input, -1) + return matches +}