Add gemget script that downloads Gemini pages

bin/gemget/main.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"gemini-grc/common/snapshot"
+	_url "gemini-grc/common/url"
+	"gemini-grc/config"
+	"gemini-grc/errors"
+	"gemini-grc/gemini"
+	"gemini-grc/gopher"
+	"gemini-grc/logging"
+)
+
+func main() {
+	config.CONFIG = *config.GetConfig()
+	err := runApp()
+	if err != nil {
+		fmt.Printf("%v\n", err)
+		logging.LogError("%v", err)
+		os.Exit(1)
+	}
+}
+
+func runApp() error {
+	if len(os.Args) != 2 {
+		return errors.NewError(fmt.Errorf("missing URL to visit"))
+	}
+	url := os.Args[1]
+	var s *snapshot.Snapshot
+	var err error
+	if _url.IsGeminiUrl(url) {
+		s, err = gemini.Visit(url)
+	} else if _url.IsGopherURL(url) {
+		s, err = gopher.Visit(url)
+	} else {
+		return errors.NewFatalError(fmt.Errorf("not a Gemini or Gopher URL"))
+	}
+	if err != nil {
+		return err
+	}
+	_json, _ := json.MarshalIndent(s, "", "  ")
+	fmt.Printf("%s\n", _json)
+	return err
+}

gemget.sh

@@ -0,0 +1,16 @@
+#!/bin/env bash
+set -eu
+set -o pipefail
+
+# Max response size 10MiB
+LOG_LEVEL=debug \
+PRINT_WORKER_STATUS=false \
+DRY_RUN=false \
+NUM_OF_WORKERS=1 \
+WORKER_BATCH_SIZE=1 \
+BLACKLIST_PATH="$(pwd)/blacklist.txt" \
+MAX_RESPONSE_SIZE=10485760 \
+RESPONSE_TIMEOUT=10 \
+PANIC_ON_UNEXPECTED_ERROR=true \
+go run ./bin/gemget/main.go "$@"
+