88 lines
2.0 KiB
Go
88 lines
2.0 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
|
|
"gemini-grc/gemini"
|
|
_ "github.com/jackc/pgx/v5/stdlib" // PGX driver for PostgreSQL
|
|
"github.com/jmoiron/sqlx"
|
|
)
|
|
|
|
// MANUALLY SET TO TRUE WHEN MIGRATION HAS BEEN APPLIED
|
|
func checkIfDone() bool { return true }
|
|
|
|
func MustSelect(tx *sqlx.Tx, dest interface{}, query string, args ...interface{}) {
|
|
if err := tx.Select(dest, query, args...); err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
|
|
// Populates the `host` field
|
|
func main() {
|
|
if checkIfDone() {
|
|
fmt.Println("Migration already applied")
|
|
return
|
|
}
|
|
|
|
db := connectToDB()
|
|
defer db.Close()
|
|
|
|
batchSize := 1000
|
|
for {
|
|
// Start the transaction
|
|
tx := db.MustBegin()
|
|
query := `SELECT * FROM snapshots WHERE url NOT LIKE '%1965%' LIMIT $1`
|
|
var snapshots []gemini.Snapshot
|
|
MustSelect(tx, &snapshots, query, batchSize)
|
|
if len(snapshots) == 0 {
|
|
fmt.Println("No snapshots remaining, done")
|
|
break
|
|
}
|
|
for i, s := range snapshots {
|
|
_, err := gemini.ParseURL(s.URL.String(), "")
|
|
if err != nil {
|
|
panic(fmt.Sprintf("Error parsing URL. ID %d URL %s\n", s.ID, s.URL))
|
|
}
|
|
fmt.Printf("Saving %d %d %s\n", i+1, s.ID, s.URL)
|
|
err = gemini.UpsertSnapshot(0, tx, &s)
|
|
if err != nil {
|
|
panic(fmt.Sprintf("Error saving %s: %s", s.URL, err))
|
|
}
|
|
tx.MustExec(`DELETE FROM snapshots WHERE id=$1`, s.ID)
|
|
}
|
|
err := tx.Commit()
|
|
if err != nil {
|
|
fmt.Println(err)
|
|
err := tx.Rollback()
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func connectToDB() *sqlx.DB {
|
|
connStr := fmt.Sprintf("postgres://%s:%s@%s:%s/%s",
|
|
os.Getenv("PG_USER"),
|
|
os.Getenv("PG_PASSWORD"),
|
|
os.Getenv("PG_HOST"),
|
|
os.Getenv("PG_PORT"),
|
|
os.Getenv("PG_DATABASE"),
|
|
)
|
|
|
|
// Create a connection pool
|
|
db, err := sqlx.Open("pgx", connStr)
|
|
if err != nil {
|
|
panic(fmt.Sprintf("Unable to connect to database with URL %s: %v\n", connStr, err))
|
|
}
|
|
db.SetMaxOpenConns(20)
|
|
err = db.Ping()
|
|
if err != nil {
|
|
panic(fmt.Sprintf("Unable to ping database: %v\n", err))
|
|
}
|
|
|
|
fmt.Println("Connected to database")
|
|
return db
|
|
}
|