package main import ( "fmt" "os" "gemini-grc/gemini" _ "github.com/jackc/pgx/v5/stdlib" // PGX driver for PostgreSQL "github.com/jmoiron/sqlx" ) // MANUALLY SET TO TRUE WHEN MIGRATION HAS BEEN APPLIED func checkIfDone() bool { return true } func MustSelect(tx *sqlx.Tx, dest interface{}, query string, args ...interface{}) { if err := tx.Select(dest, query, args...); err != nil { panic(err) } } // Populates the `host` field func main() { if checkIfDone() { fmt.Println("Migration already applied") return } db := connectToDB() defer db.Close() batchSize := 1000 for { // Start the transaction tx := db.MustBegin() query := `SELECT * FROM snapshots WHERE url NOT LIKE '%1965%' LIMIT $1` var snapshots []gemini.Snapshot MustSelect(tx, &snapshots, query, batchSize) if len(snapshots) == 0 { fmt.Println("No snapshots remaining, done") break } for i, s := range snapshots { _, err := gemini.ParseURL(s.URL.String(), "") if err != nil { panic(fmt.Sprintf("Error parsing URL. ID %d URL %s\n", s.ID, s.URL)) } fmt.Printf("Saving %d %d %s\n", i+1, s.ID, s.URL) err = gemini.UpsertSnapshot(0, tx, &s) if err != nil { panic(fmt.Sprintf("Error saving %s: %s", s.URL, err)) } tx.MustExec(`DELETE FROM snapshots WHERE id=$1`, s.ID) } err := tx.Commit() if err != nil { fmt.Println(err) err := tx.Rollback() if err != nil { panic(err) } } } } func connectToDB() *sqlx.DB { connStr := fmt.Sprintf("postgres://%s:%s@%s:%s/%s", os.Getenv("PG_USER"), os.Getenv("PG_PASSWORD"), os.Getenv("PG_HOST"), os.Getenv("PG_PORT"), os.Getenv("PG_DATABASE"), ) // Create a connection pool db, err := sqlx.Open("pgx", connStr) if err != nil { panic(fmt.Sprintf("Unable to connect to database with URL %s: %v\n", connStr, err)) } db.SetMaxOpenConns(20) err = db.Ping() if err != nil { panic(fmt.Sprintf("Unable to ping database: %v\n", err)) } fmt.Println("Connected to database") return db }