DB scripts and migrations

db/migration_3/migrate3_url_port.go

@@ -0,0 +1,87 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"gemini-grc/gemini"
+	_ "github.com/jackc/pgx/v5/stdlib" // PGX driver for PostgreSQL
+	"github.com/jmoiron/sqlx"
+)
+
+// MANUALLY SET TO TRUE WHEN MIGRATION HAS BEEN APPLIED
+func checkIfDone() bool { return true }
+
+func MustSelect(tx *sqlx.Tx, dest interface{}, query string, args ...interface{}) {
+	if err := tx.Select(dest, query, args...); err != nil {
+		panic(err)
+	}
+}
+
+// Populates the `host` field
+func main() {
+	if checkIfDone() {
+		fmt.Println("Migration already applied")
+		return
+	}
+
+	db := connectToDB()
+	defer db.Close()
+
+	batchSize := 1000
+	for {
+		// Start the transaction
+		tx := db.MustBegin()
+		query := `SELECT * FROM snapshots WHERE url NOT LIKE '%1965%' LIMIT $1`
+		var snapshots []gemini.Snapshot
+		MustSelect(tx, &snapshots, query, batchSize)
+		if len(snapshots) == 0 {
+			fmt.Println("No snapshots remaining, done")
+			break
+		}
+		for i, s := range snapshots {
+			_, err := gemini.ParseURL(s.URL.String(), "")
+			if err != nil {
+				panic(fmt.Sprintf("Error parsing URL. ID %d URL %s\n", s.ID, s.URL))
+			}
+			fmt.Printf("Saving %d %d %s\n", i+1, s.ID, s.URL)
+			err = gemini.UpsertSnapshot(0, tx, &s)
+			if err != nil {
+				panic(fmt.Sprintf("Error saving %s: %s", s.URL, err))
+			}
+			tx.MustExec(`DELETE FROM snapshots WHERE id=$1`, s.ID)
+		}
+		err := tx.Commit()
+		if err != nil {
+			fmt.Println(err)
+			err := tx.Rollback()
+			if err != nil {
+				panic(err)
+			}
+		}
+	}
+}
+
+func connectToDB() *sqlx.DB {
+	connStr := fmt.Sprintf("postgres://%s:%s@%s:%s/%s",
+		os.Getenv("PG_USER"),
+		os.Getenv("PG_PASSWORD"),
+		os.Getenv("PG_HOST"),
+		os.Getenv("PG_PORT"),
+		os.Getenv("PG_DATABASE"),
+	)
+
+	// Create a connection pool
+	db, err := sqlx.Open("pgx", connStr)
+	if err != nil {
+		panic(fmt.Sprintf("Unable to connect to database with URL %s: %v\n", connStr, err))
+	}
+	db.SetMaxOpenConns(20)
+	err = db.Ping()
+	if err != nil {
+		panic(fmt.Sprintf("Unable to ping database: %v\n", err))
+	}
+
+	fmt.Println("Connected to database")
+	return db
+}