83 lines
1.8 KiB
Go
83 lines
1.8 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"os/signal"
|
|
"syscall"
|
|
|
|
"gemini-grc/common"
|
|
"gemini-grc/common/blackList"
|
|
"gemini-grc/config"
|
|
"gemini-grc/db"
|
|
"gemini-grc/logging"
|
|
"github.com/antanst/go_errors"
|
|
"github.com/jmoiron/sqlx"
|
|
"github.com/rs/zerolog"
|
|
zlog "github.com/rs/zerolog/log"
|
|
)
|
|
|
|
func main() {
|
|
config.CONFIG = *config.GetConfig()
|
|
zerolog.TimeFieldFormat = zerolog.TimeFormatUnix
|
|
zerolog.SetGlobalLevel(config.CONFIG.LogLevel)
|
|
zlog.Logger = zlog.Output(zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: "[2006-01-02 15:04:05]"})
|
|
err := runApp()
|
|
if err != nil {
|
|
var asErr *go_errors.Error
|
|
if go_errors.As(err, &asErr) {
|
|
logging.LogError("Unexpected error: %v", err)
|
|
_, _ = fmt.Fprintf(os.Stderr, "Unexpected error: %v", err)
|
|
} else {
|
|
logging.LogError("Unexpected error: %v", err)
|
|
}
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
|
|
func runApp() (err error) {
|
|
logging.LogInfo("gemcrawl %s starting up. Press Ctrl+C to exit", common.VERSION)
|
|
signals := make(chan os.Signal, 1)
|
|
signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
|
|
|
|
_db, err := db.ConnectToDB()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
defer func(db *sqlx.DB) {
|
|
_ = db.Close()
|
|
}(_db)
|
|
|
|
err = blackList.LoadBlacklist()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
common.StatusChan = make(chan common.WorkerStatus, config.CONFIG.NumOfWorkers)
|
|
common.ErrorsChan = make(chan error, config.CONFIG.NumOfWorkers)
|
|
|
|
// If there's an argument, visit this
|
|
// URL only and don't spawn other workers
|
|
if len(os.Args) > 1 {
|
|
url := os.Args[1]
|
|
err = common.CrawlOneURL(_db, &url)
|
|
return err
|
|
}
|
|
|
|
go common.SpawnWorkers(config.CONFIG.NumOfWorkers, _db)
|
|
|
|
for {
|
|
select {
|
|
case <-signals:
|
|
logging.LogWarn("Received SIGINT or SIGTERM signal, exiting")
|
|
return nil
|
|
case err := <-common.ErrorsChan:
|
|
if go_errors.IsFatal(err) {
|
|
return err
|
|
}
|
|
logging.LogError("%s", fmt.Sprintf("%v", err))
|
|
}
|
|
}
|
|
}
|