Compare commits
2 Commits
main
..
4eed099154
| Author | SHA1 | Date | |
|---|---|---|---|
 antanst
|
4eed099154 | ||
|
antanst
|
e989633e03 |
+9
-11
@@ -1,11 +1,9 @@
|
|||||||
.idea
|
/.idea
|
||||||
.goroot
|
/target
|
||||||
**/.#*
|
|
||||||
**/*~
|
|
||||||
/.go
|
# Added by cargo
|
||||||
/cmd
|
#
|
||||||
/db/initdb.sql
|
# already existing elements were commented out
|
||||||
/db/*sh
|
|
||||||
/run*.sh
|
#/target
|
||||||
/gemini-grc
|
|
||||||
/snaps
|
|
||||||
|
|||||||
Generated
+551
@@ -0,0 +1,551 @@
|
|||||||
|
# This file is automatically @generated by Cargo.
|
||||||
|
# It is not intended for manual editing.
|
||||||
|
version = 3
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "aho-corasick"
|
||||||
|
version = "1.1.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
|
||||||
|
dependencies = [
|
||||||
|
"memchr",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "anstream"
|
||||||
|
version = "0.6.15"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526"
|
||||||
|
dependencies = [
|
||||||
|
"anstyle",
|
||||||
|
"anstyle-parse",
|
||||||
|
"anstyle-query",
|
||||||
|
"anstyle-wincon",
|
||||||
|
"colorchoice",
|
||||||
|
"is_terminal_polyfill",
|
||||||
|
"utf8parse",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "anstyle"
|
||||||
|
version = "1.0.8"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "anstyle-parse"
|
||||||
|
version = "0.2.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb"
|
||||||
|
dependencies = [
|
||||||
|
"utf8parse",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "anstyle-query"
|
||||||
|
version = "1.1.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a"
|
||||||
|
dependencies = [
|
||||||
|
"windows-sys",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "anstyle-wincon"
|
||||||
|
version = "3.0.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8"
|
||||||
|
dependencies = [
|
||||||
|
"anstyle",
|
||||||
|
"windows-sys",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "anyhow"
|
||||||
|
version = "1.0.86"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "bitflags"
|
||||||
|
version = "2.6.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "byteorder"
|
||||||
|
version = "1.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cc"
|
||||||
|
version = "1.1.13"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "72db2f7947ecee9b03b510377e8bb9077afa27176fdbff55c51027e976fdcc48"
|
||||||
|
dependencies = [
|
||||||
|
"shlex",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cfg-if"
|
||||||
|
version = "1.0.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "colorchoice"
|
||||||
|
version = "1.0.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "env_filter"
|
||||||
|
version = "0.1.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4f2c92ceda6ceec50f43169f9ee8424fe2db276791afde7b2cd8bc084cb376ab"
|
||||||
|
dependencies = [
|
||||||
|
"log",
|
||||||
|
"regex",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "env_logger"
|
||||||
|
version = "0.11.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e13fa619b91fb2381732789fc5de83b45675e882f66623b7d8cb4f643017018d"
|
||||||
|
dependencies = [
|
||||||
|
"anstream",
|
||||||
|
"anstyle",
|
||||||
|
"env_filter",
|
||||||
|
"humantime",
|
||||||
|
"log",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "foreign-types"
|
||||||
|
version = "0.3.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
|
||||||
|
dependencies = [
|
||||||
|
"foreign-types-shared",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "foreign-types-shared"
|
||||||
|
version = "0.1.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "form_urlencoded"
|
||||||
|
version = "1.2.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456"
|
||||||
|
dependencies = [
|
||||||
|
"percent-encoding",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "gemini-grs"
|
||||||
|
version = "0.1.0"
|
||||||
|
dependencies = [
|
||||||
|
"anyhow",
|
||||||
|
"env_logger",
|
||||||
|
"hex",
|
||||||
|
"log",
|
||||||
|
"nanoid",
|
||||||
|
"openssl",
|
||||||
|
"path-clean",
|
||||||
|
"url",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "getrandom"
|
||||||
|
version = "0.2.15"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"libc",
|
||||||
|
"wasi",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "hex"
|
||||||
|
version = "0.4.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "humantime"
|
||||||
|
version = "2.1.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "idna"
|
||||||
|
version = "0.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6"
|
||||||
|
dependencies = [
|
||||||
|
"unicode-bidi",
|
||||||
|
"unicode-normalization",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "is_terminal_polyfill"
|
||||||
|
version = "1.70.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "libc"
|
||||||
|
version = "0.2.156"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a5f43f184355eefb8d17fc948dbecf6c13be3c141f20d834ae842193a448c72a"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "log"
|
||||||
|
version = "0.4.22"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "memchr"
|
||||||
|
version = "2.7.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "nanoid"
|
||||||
|
version = "0.4.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3ffa00dec017b5b1a8b7cf5e2c008bfda1aa7e0697ac1508b491fdf2622fb4d8"
|
||||||
|
dependencies = [
|
||||||
|
"rand",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "once_cell"
|
||||||
|
version = "1.19.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "openssl"
|
||||||
|
version = "0.10.66"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags",
|
||||||
|
"cfg-if",
|
||||||
|
"foreign-types",
|
||||||
|
"libc",
|
||||||
|
"once_cell",
|
||||||
|
"openssl-macros",
|
||||||
|
"openssl-sys",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "openssl-macros"
|
||||||
|
version = "0.1.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "openssl-sys"
|
||||||
|
version = "0.9.103"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6"
|
||||||
|
dependencies = [
|
||||||
|
"cc",
|
||||||
|
"libc",
|
||||||
|
"pkg-config",
|
||||||
|
"vcpkg",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "path-clean"
|
||||||
|
version = "1.0.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "17359afc20d7ab31fdb42bb844c8b3bb1dabd7dcf7e68428492da7f16966fcef"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "percent-encoding"
|
||||||
|
version = "2.3.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "pkg-config"
|
||||||
|
version = "0.3.30"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "ppv-lite86"
|
||||||
|
version = "0.2.20"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04"
|
||||||
|
dependencies = [
|
||||||
|
"zerocopy",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "proc-macro2"
|
||||||
|
version = "1.0.86"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77"
|
||||||
|
dependencies = [
|
||||||
|
"unicode-ident",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "quote"
|
||||||
|
version = "1.0.36"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rand"
|
||||||
|
version = "0.8.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
"rand_chacha",
|
||||||
|
"rand_core",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rand_chacha"
|
||||||
|
version = "0.3.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
|
||||||
|
dependencies = [
|
||||||
|
"ppv-lite86",
|
||||||
|
"rand_core",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rand_core"
|
||||||
|
version = "0.6.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
|
||||||
|
dependencies = [
|
||||||
|
"getrandom",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "regex"
|
||||||
|
version = "1.10.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619"
|
||||||
|
dependencies = [
|
||||||
|
"aho-corasick",
|
||||||
|
"memchr",
|
||||||
|
"regex-automata",
|
||||||
|
"regex-syntax",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "regex-automata"
|
||||||
|
version = "0.4.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df"
|
||||||
|
dependencies = [
|
||||||
|
"aho-corasick",
|
||||||
|
"memchr",
|
||||||
|
"regex-syntax",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "regex-syntax"
|
||||||
|
version = "0.8.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "shlex"
|
||||||
|
version = "1.3.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "syn"
|
||||||
|
version = "2.0.74"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1fceb41e3d546d0bd83421d3409b1460cc7444cd389341a4c880fe7a042cb3d7"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"unicode-ident",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tinyvec"
|
||||||
|
version = "1.8.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938"
|
||||||
|
dependencies = [
|
||||||
|
"tinyvec_macros",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tinyvec_macros"
|
||||||
|
version = "0.1.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "unicode-bidi"
|
||||||
|
version = "0.3.15"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "unicode-ident"
|
||||||
|
version = "1.0.12"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "unicode-normalization"
|
||||||
|
version = "0.1.23"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5"
|
||||||
|
dependencies = [
|
||||||
|
"tinyvec",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "url"
|
||||||
|
version = "2.5.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c"
|
||||||
|
dependencies = [
|
||||||
|
"form_urlencoded",
|
||||||
|
"idna",
|
||||||
|
"percent-encoding",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "utf8parse"
|
||||||
|
version = "0.2.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "vcpkg"
|
||||||
|
version = "0.2.15"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wasi"
|
||||||
|
version = "0.11.0+wasi-snapshot-preview1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-sys"
|
||||||
|
version = "0.52.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
|
||||||
|
dependencies = [
|
||||||
|
"windows-targets",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-targets"
|
||||||
|
version = "0.52.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
|
||||||
|
dependencies = [
|
||||||
|
"windows_aarch64_gnullvm",
|
||||||
|
"windows_aarch64_msvc",
|
||||||
|
"windows_i686_gnu",
|
||||||
|
"windows_i686_gnullvm",
|
||||||
|
"windows_i686_msvc",
|
||||||
|
"windows_x86_64_gnu",
|
||||||
|
"windows_x86_64_gnullvm",
|
||||||
|
"windows_x86_64_msvc",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows_aarch64_gnullvm"
|
||||||
|
version = "0.52.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows_aarch64_msvc"
|
||||||
|
version = "0.52.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows_i686_gnu"
|
||||||
|
version = "0.52.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows_i686_gnullvm"
|
||||||
|
version = "0.52.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows_i686_msvc"
|
||||||
|
version = "0.52.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows_x86_64_gnu"
|
||||||
|
version = "0.52.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows_x86_64_gnullvm"
|
||||||
|
version = "0.52.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows_x86_64_msvc"
|
||||||
|
version = "0.52.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zerocopy"
|
||||||
|
version = "0.7.35"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
|
||||||
|
dependencies = [
|
||||||
|
"byteorder",
|
||||||
|
"zerocopy-derive",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zerocopy-derive"
|
||||||
|
version = "0.7.35"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
+14
@@ -0,0 +1,14 @@
|
|||||||
|
[package]
|
||||||
|
name = "gemini-grs"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition = "2021"
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
anyhow = "1.0.86"
|
||||||
|
env_logger = "0.11.5"
|
||||||
|
hex = "0.4.3"
|
||||||
|
log = "0.4.22"
|
||||||
|
nanoid = "0.4.0"
|
||||||
|
openssl = "0.10.66"
|
||||||
|
path-clean = "1.0.1"
|
||||||
|
url = "2.5.2"
|
||||||
@@ -1,28 +0,0 @@
|
|||||||
SHELL := /usr/local/bin/oksh
|
|
||||||
export PATH := $(PATH)
|
|
||||||
|
|
||||||
.PHONY: all fmt lint
|
|
||||||
|
|
||||||
all: fmt lint test
|
|
||||||
|
|
||||||
.PHONY: debug
|
|
||||||
debug:
|
|
||||||
@echo "PATH: $(PATH)"
|
|
||||||
@echo "GOPATH: $(shell go env GOPATH)"
|
|
||||||
@which go
|
|
||||||
@which gofumpt
|
|
||||||
@which gci
|
|
||||||
@which golangci-lint
|
|
||||||
|
|
||||||
# Test
|
|
||||||
test:
|
|
||||||
go test -v ./...
|
|
||||||
|
|
||||||
# Format code
|
|
||||||
fmt:
|
|
||||||
gofumpt -l -w .
|
|
||||||
gci write .
|
|
||||||
|
|
||||||
# Run linter
|
|
||||||
lint: fmt
|
|
||||||
golangci-lint run
|
|
||||||
@@ -1,27 +1,51 @@
|
|||||||
# gemini-grc
|
# gemini-grs
|
||||||
|
|
||||||
A Gemini crawler.
|
A Gemini protocol server written in Rust.
|
||||||
|
|
||||||
URLs to visit as well as data from visited URLs are stored into "snapshots" in the database.
|
|
||||||
|
|
||||||
## Done
|
|
||||||
- [x] Concurrent downloading with workers
|
|
||||||
- [x] Concurrent connection limit per host
|
|
||||||
- [x] URL Blacklist
|
|
||||||
- [x] Save image/* and text/* files
|
|
||||||
- [x] Configuration via environment variables
|
|
||||||
- [x] Storing snapshots in PostgreSQL
|
|
||||||
- [x] Proper response header & body UTF-8 and format validation
|
|
||||||
- [x] Follow robots.txt, see gemini://geminiprotocol.net/docs/companion/robots.gmi
|
|
||||||
- [x] Handle redirects (3X status codes)
|
|
||||||
|
|
||||||
## TODO
|
## TODO
|
||||||
- [ ] Better URL normalization
|
|
||||||
- [ ] Provide a TLS cert for sites that require it, like Astrobotany
|
|
||||||
|
|
||||||
## TODO for later
|
- [x] Configuration via environment variables
|
||||||
- [ ] Gopher
|
- [x] Proper logging
|
||||||
- [ ] Scroll gemini://auragem.letz.dev/devlog/20240316.gmi
|
- [x] Serve images
|
||||||
- [ ] Spartan
|
- [ ] Read and send file in chunks instead of serving whole file from memory
|
||||||
- [ ] Nex
|
- [ ] Expand to serve other protocols (spartan, scroll, titan etc.)
|
||||||
- [ ] SuperTXT https://supertxt.net/00-intro.html
|
- [ ] Async I/O?
|
||||||
|
|
||||||
|
## External dependencies
|
||||||
|
|
||||||
|
OpenSSL
|
||||||
|
|
||||||
|
## Building
|
||||||
|
|
||||||
|
```shell
|
||||||
|
cargo build --release
|
||||||
|
```
|
||||||
|
|
||||||
|
## Running
|
||||||
|
|
||||||
|
Generate your TLS keys:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
openssl genrsa -out key.pem 2048
|
||||||
|
openssl req -new -key key.pem -out request.pem
|
||||||
|
openssl x509 -req -days 36500 -in request.pem -signkey key.pem -out cert.pem
|
||||||
|
```
|
||||||
|
|
||||||
|
Environment variables with examples:
|
||||||
|
|
||||||
|
- GEMINI_SERVER_HOSTNAME="0.0.0.0:1965"
|
||||||
|
- GEMINI_SERVER_TLS_KEY_FILENAME="/where/you/put/key.pem"
|
||||||
|
- GEMINI_SERVER_TLS_CERT_FILENAME="/where/you/put/cert.pem"
|
||||||
|
- GEMINI_SERVER_ROOT_DIRECTORY="/files/to/serve"
|
||||||
|
- RUST_LOG="debug"
|
||||||
|
|
||||||
|
Example command:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
GEMINI_SERVER_TLS_KEY_FILENAME=/home/user/server/key.pem \
|
||||||
|
GEMINI_SERVER_TLS_CERT_FILENAME=/home/user/server/cert.pem \
|
||||||
|
GEMINI_SERVER_ROOT_DIRECTORY=/gemini-root \
|
||||||
|
RUST_LOG=debug \
|
||||||
|
./target/release/gemini-grs
|
||||||
|
```
|
||||||
|
|
||||||
|
|||||||
@@ -1,2 +0,0 @@
|
|||||||
gemi.dev
|
|
||||||
mastogem.picasoft.net
|
|
||||||
@@ -1,156 +0,0 @@
|
|||||||
package config
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"strconv"
|
|
||||||
|
|
||||||
"github.com/rs/zerolog"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Environment variable names.
|
|
||||||
const (
|
|
||||||
EnvLogLevel = "LOG_LEVEL"
|
|
||||||
EnvNumWorkers = "NUM_OF_WORKERS"
|
|
||||||
EnvWorkerBatchSize = "WORKER_BATCH_SIZE"
|
|
||||||
EnvMaxResponseSize = "MAX_RESPONSE_SIZE"
|
|
||||||
EnvResponseTimeout = "RESPONSE_TIMEOUT"
|
|
||||||
EnvPanicOnUnexpectedError = "PANIC_ON_UNEXPECTED_ERROR"
|
|
||||||
EnvBlacklistPath = "BLACKLIST_PATH"
|
|
||||||
EnvDryRun = "DRY_RUN"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Config holds the application configuration loaded from environment variables.
|
|
||||||
type Config struct {
|
|
||||||
LogLevel zerolog.Level // Logging level (debug, info, warn, error)
|
|
||||||
MaxResponseSize int // Maximum size of response in bytes
|
|
||||||
NumOfWorkers int // Number of concurrent workers
|
|
||||||
ResponseTimeout int // Timeout for responses in seconds
|
|
||||||
WorkerBatchSize int // Batch size for worker processing
|
|
||||||
PanicOnUnexpectedError bool // Panic on unexpected errors when visiting a URL
|
|
||||||
BlacklistPath string // File that has blacklisted strings of "host:port"
|
|
||||||
DryRun bool // If false, don't write to disk
|
|
||||||
}
|
|
||||||
|
|
||||||
var CONFIG Config //nolint:gochecknoglobals
|
|
||||||
|
|
||||||
// parsePositiveInt parses and validates positive integer values.
|
|
||||||
func parsePositiveInt(param, value string) (int, error) {
|
|
||||||
val, err := strconv.Atoi(value)
|
|
||||||
if err != nil {
|
|
||||||
return 0, ValidationError{
|
|
||||||
Param: param,
|
|
||||||
Value: value,
|
|
||||||
Reason: "must be a valid integer",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if val <= 0 {
|
|
||||||
return 0, ValidationError{
|
|
||||||
Param: param,
|
|
||||||
Value: value,
|
|
||||||
Reason: "must be positive",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return val, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseBool(param, value string) (bool, error) {
|
|
||||||
val, err := strconv.ParseBool(value)
|
|
||||||
if err != nil {
|
|
||||||
return false, ValidationError{
|
|
||||||
Param: param,
|
|
||||||
Value: value,
|
|
||||||
Reason: "cannot be converted to boolean",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return val, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetConfig loads and validates configuration from environment variables
|
|
||||||
func GetConfig() *Config {
|
|
||||||
config := &Config{}
|
|
||||||
|
|
||||||
// Map of environment variables to their parsing functions
|
|
||||||
parsers := map[string]func(string) error{
|
|
||||||
EnvLogLevel: func(v string) error {
|
|
||||||
level, err := zerolog.ParseLevel(v)
|
|
||||||
if err != nil {
|
|
||||||
return ValidationError{
|
|
||||||
Param: EnvLogLevel,
|
|
||||||
Value: v,
|
|
||||||
Reason: "must be one of: debug, info, warn, error",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
config.LogLevel = level
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
EnvNumWorkers: func(v string) error {
|
|
||||||
val, err := parsePositiveInt(EnvNumWorkers, v)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
config.NumOfWorkers = val
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
EnvWorkerBatchSize: func(v string) error {
|
|
||||||
val, err := parsePositiveInt(EnvWorkerBatchSize, v)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
config.WorkerBatchSize = val
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
EnvMaxResponseSize: func(v string) error {
|
|
||||||
val, err := parsePositiveInt(EnvMaxResponseSize, v)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
config.MaxResponseSize = val
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
EnvResponseTimeout: func(v string) error {
|
|
||||||
val, err := parsePositiveInt(EnvResponseTimeout, v)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
config.ResponseTimeout = val
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
EnvPanicOnUnexpectedError: func(v string) error {
|
|
||||||
val, err := parseBool(EnvPanicOnUnexpectedError, v)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
config.PanicOnUnexpectedError = val
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
EnvBlacklistPath: func(v string) error {
|
|
||||||
config.BlacklistPath = v
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
EnvDryRun: func(v string) error {
|
|
||||||
val, err := parseBool(EnvDryRun, v)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
config.DryRun = val
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
// Process each environment variable
|
|
||||||
for envVar, parser := range parsers {
|
|
||||||
value, ok := os.LookupEnv(envVar)
|
|
||||||
if !ok {
|
|
||||||
fmt.Fprintf(os.Stderr, "Missing required environment variable: %s\n", envVar)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := parser(value); err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "Configuration error: %v\n", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return config
|
|
||||||
}
|
|
||||||
@@ -1,14 +0,0 @@
|
|||||||
package config
|
|
||||||
|
|
||||||
import "fmt"
|
|
||||||
|
|
||||||
// ValidationError represents a config validation error
|
|
||||||
type ValidationError struct {
|
|
||||||
Param string
|
|
||||||
Value string
|
|
||||||
Reason string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e ValidationError) Error() string {
|
|
||||||
return fmt.Sprintf("invalid value '%s' for %s: %s", e.Value, e.Param, e.Reason)
|
|
||||||
}
|
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
BEGIN;
|
|
||||||
|
|
||||||
-- Increase statement timeout
|
|
||||||
SET statement_timeout = '10min';
|
|
||||||
|
|
||||||
-- Step 1: Create a new table with the same schema
|
|
||||||
CREATE TABLE backup (LIKE snapshots INCLUDING ALL);
|
|
||||||
|
|
||||||
-- Step 2: Copy data from the old table to the new one
|
|
||||||
INSERT INTO backup SELECT * FROM snapshots;
|
|
||||||
|
|
||||||
-- (Optional) Step 3: Truncate the original table if you are moving the data
|
|
||||||
-- TRUNCATE TABLE snapshots;
|
|
||||||
|
|
||||||
-- Commit the transaction if everything went well
|
|
||||||
COMMIT;
|
|
||||||
@@ -1,26 +0,0 @@
|
|||||||
-- Explanation:
|
|
||||||
|
|
||||||
-- WITH DuplicateSnapshots AS:
|
|
||||||
-- This is a Common Table Expression (CTE) that selects all rows from the snapshots table.
|
|
||||||
-- ROW_NUMBER() OVER (PARTITION BY url ORDER BY id): This assigns a unique row number to each row with the same url. The PARTITION BY url groups the rows by url, and ORDER BY id ensures that the row with the smallest id is given row_num = 1.
|
|
||||||
-- DELETE FROM snapshots WHERE id IN:
|
|
||||||
-- The DELETE statement deletes rows from the snapshots table where the id is in the result of the subquery.
|
|
||||||
-- WHERE row_num > 1:
|
|
||||||
-- In the subquery, we select only rows where row_num > 1, which means only the duplicate rows (since row_num = 1 is the one row we want to keep).
|
|
||||||
|
|
||||||
-- Result:
|
|
||||||
|
|
||||||
-- This query will delete all duplicate rows from the snapshots table, keeping only the row with the smallest id for each url.
|
|
||||||
-- If multiple rows share the same url, only the first one (based on id) will be retained.
|
|
||||||
|
|
||||||
WITH DuplicateSnapshots AS (
|
|
||||||
SELECT id,
|
|
||||||
ROW_NUMBER() OVER (PARTITION BY url ORDER BY id) AS row_num
|
|
||||||
FROM snapshots
|
|
||||||
)
|
|
||||||
DELETE FROM snapshots
|
|
||||||
WHERE id IN (
|
|
||||||
SELECT id
|
|
||||||
FROM DuplicateSnapshots
|
|
||||||
WHERE row_num > 1
|
|
||||||
);
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
delete FROM snapshots
|
|
||||||
WHERE host IN (
|
|
||||||
SELECT DISTINCT host
|
|
||||||
FROM snapshots
|
|
||||||
WHERE error LIKE 'robots.txt%'
|
|
||||||
)
|
|
||||||
AND url LIKE 'gemini://' || host || '/%';
|
|
||||||
@@ -1,5 +0,0 @@
|
|||||||
SELECT error, count(error) as count
|
|
||||||
FROM snapshots
|
|
||||||
GROUP BY error
|
|
||||||
ORDER BY count DESC
|
|
||||||
LIMIT 20;
|
|
||||||
@@ -1,22 +0,0 @@
|
|||||||
-- Here's an SQL script that will find and remove snapshots without port numbers
|
|
||||||
-- when there exists a duplicate with the default port 1965.
|
|
||||||
|
|
||||||
-- Before running this DELETE though, you might want to
|
|
||||||
-- verify the matches first with this SELECT:
|
|
||||||
WITH duplicates AS (
|
|
||||||
SELECT s1.id as id_without_port, s2.id as id_with_port, s1.url as url_without_port, s2.url as url_with_port
|
|
||||||
FROM snapshots s1
|
|
||||||
JOIN snapshots s2
|
|
||||||
ON s2.url = s1.url || ':1965'
|
|
||||||
)
|
|
||||||
SELECT * FROM duplicates;
|
|
||||||
|
|
||||||
-- Now delete them for real:
|
|
||||||
WITH duplicates AS (
|
|
||||||
SELECT s1.id as id_without_port, s2.id as id_with_port, s1.url as url_without_port, s2.url as url_with_port
|
|
||||||
FROM snapshots s1
|
|
||||||
JOIN snapshots s2
|
|
||||||
ON s2.url = s1.url || ':1965'
|
|
||||||
)
|
|
||||||
DELETE FROM snapshots
|
|
||||||
WHERE id IN (SELECT id_without_port FROM duplicates);
|
|
||||||
@@ -1,5 +0,0 @@
|
|||||||
SELECT host, COUNT(*) AS row_count
|
|
||||||
FROM snapshots
|
|
||||||
GROUP BY host
|
|
||||||
ORDER BY row_count DESC
|
|
||||||
LIMIT 10;
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
SELECT host, COUNT(*) AS row_count
|
|
||||||
FROM snapshots
|
|
||||||
WHERE response_code IS NOT NULL
|
|
||||||
AND error IS NULL
|
|
||||||
GROUP BY host
|
|
||||||
ORDER BY row_count DESC
|
|
||||||
LIMIT 10;
|
|
||||||
@@ -1,51 +0,0 @@
|
|||||||
-- DB creation and users
|
|
||||||
CREATE USER gemini;
|
|
||||||
ALTER USER gemini WITH PASSWORD 'gemini';
|
|
||||||
CREATE DATABASE gemini;
|
|
||||||
GRANT ALL PRIVILEGES ON DATABASE gemini TO gemini;
|
|
||||||
ALTER DATABASE gemini OWNER TO gemini;
|
|
||||||
GRANT ALL PRIVILEGES ON SCHEMA public TO gemini;
|
|
||||||
GRANT ALL PRIVILEGES ON gemini TO gemini;
|
|
||||||
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO gemini;
|
|
||||||
|
|
||||||
-- Extensions
|
|
||||||
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
|
||||||
|
|
||||||
|
|
||||||
\c gemini
|
|
||||||
|
|
||||||
-- Tables
|
|
||||||
DROP TABLE IF EXISTS snapshots;
|
|
||||||
|
|
||||||
CREATE TABLE snapshots (
|
|
||||||
id SERIAL PRIMARY KEY,
|
|
||||||
uid TEXT NOT NULL UNIQUE,
|
|
||||||
url TEXT NOT NULL UNIQUE,
|
|
||||||
host TEXT NOT NULL,
|
|
||||||
timestamp TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
||||||
mimetype TEXT,
|
|
||||||
data BYTEA,
|
|
||||||
gemtext TEXT,
|
|
||||||
links JSONB,
|
|
||||||
lang TEXT,
|
|
||||||
response_code INTEGER,
|
|
||||||
error TEXT
|
|
||||||
);
|
|
||||||
|
|
||||||
ALTER TABLE snapshots OWNER TO gemini;
|
|
||||||
|
|
||||||
CREATE INDEX idx_uid ON snapshots (uid);
|
|
||||||
CREATE INDEX idx_url ON snapshots (url);
|
|
||||||
CREATE INDEX idx_timestamp ON snapshots (timestamp);
|
|
||||||
CREATE INDEX idx_mimetype ON snapshots (mimetype);
|
|
||||||
CREATE INDEX idx_lang ON snapshots (lang);
|
|
||||||
CREATE INDEX idx_response_code ON snapshots (response_code);
|
|
||||||
CREATE INDEX idx_error ON snapshots (error);
|
|
||||||
CREATE INDEX idx_host ON snapshots (host);
|
|
||||||
CREATE INDEX unique_uid_url ON snapshots (uid, url);
|
|
||||||
|
|
||||||
CREATE INDEX idx_snapshots_unprocessed_no_data ON snapshots (host)
|
|
||||||
WHERE response_code IS NULL AND error IS NULL
|
|
||||||
INCLUDE (id, uid, url, timestamp, mimetype, gemtext, links, lang);
|
|
||||||
|
|
||||||
CREATE INDEX idx_response_code_error_nulls ON snapshots (response_code, error) WHERE response_code IS NULL AND error IS NULL;
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
-- Step 1: Delete duplicate entries, keeping the last one based on timestamp
|
|
||||||
-- Use a CTE to mark duplicates and delete them efficiently
|
|
||||||
WITH ranked_snapshots AS (
|
|
||||||
SELECT
|
|
||||||
id,
|
|
||||||
url,
|
|
||||||
ROW_NUMBER() OVER(PARTITION BY url ORDER BY timestamp DESC) AS row_num
|
|
||||||
FROM
|
|
||||||
snapshots
|
|
||||||
)
|
|
||||||
DELETE FROM snapshots
|
|
||||||
USING ranked_snapshots
|
|
||||||
WHERE snapshots.id = ranked_snapshots.id
|
|
||||||
AND ranked_snapshots.row_num > 1;
|
|
||||||
|
|
||||||
-- Step 2: Add a unique constraint on the url column to prevent future duplicates
|
|
||||||
ALTER TABLE snapshots
|
|
||||||
ADD CONSTRAINT unique_url UNIQUE (url);
|
|
||||||
@@ -1,87 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
|
|
||||||
"gemini-grc/gemini"
|
|
||||||
_ "github.com/jackc/pgx/v5/stdlib" // PGX driver for PostgreSQL
|
|
||||||
"github.com/jmoiron/sqlx"
|
|
||||||
)
|
|
||||||
|
|
||||||
// MANUALLY SET TO TRUE WHEN MIGRATION HAS BEEN APPLIED
|
|
||||||
func checkIfDone() bool { return true }
|
|
||||||
|
|
||||||
func MustSelect(tx *sqlx.Tx, dest interface{}, query string, args ...interface{}) {
|
|
||||||
if err := tx.Select(dest, query, args...); err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Populates the `host` field
|
|
||||||
func main() {
|
|
||||||
if checkIfDone() {
|
|
||||||
fmt.Println("Migration already applied")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
db := connectToDB()
|
|
||||||
defer db.Close()
|
|
||||||
|
|
||||||
batchSize := 1000
|
|
||||||
for {
|
|
||||||
// Start the transaction
|
|
||||||
tx := db.MustBegin()
|
|
||||||
query := `SELECT * FROM snapshots WHERE url NOT LIKE '%1965%' LIMIT $1`
|
|
||||||
var snapshots []gemini.Snapshot
|
|
||||||
MustSelect(tx, &snapshots, query, batchSize)
|
|
||||||
if len(snapshots) == 0 {
|
|
||||||
fmt.Println("No snapshots remaining, done")
|
|
||||||
break
|
|
||||||
}
|
|
||||||
for i, s := range snapshots {
|
|
||||||
_, err := gemini.ParseURL(s.URL.String(), "")
|
|
||||||
if err != nil {
|
|
||||||
panic(fmt.Sprintf("Error parsing URL. ID %d URL %s\n", s.ID, s.URL))
|
|
||||||
}
|
|
||||||
fmt.Printf("Saving %d %d %s\n", i+1, s.ID, s.URL)
|
|
||||||
err = gemini.UpsertSnapshot(0, tx, &s)
|
|
||||||
if err != nil {
|
|
||||||
panic(fmt.Sprintf("Error saving %s: %s", s.URL, err))
|
|
||||||
}
|
|
||||||
tx.MustExec(`DELETE FROM snapshots WHERE id=$1`, s.ID)
|
|
||||||
}
|
|
||||||
err := tx.Commit()
|
|
||||||
if err != nil {
|
|
||||||
fmt.Println(err)
|
|
||||||
err := tx.Rollback()
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func connectToDB() *sqlx.DB {
|
|
||||||
connStr := fmt.Sprintf("postgres://%s:%s@%s:%s/%s",
|
|
||||||
os.Getenv("PG_USER"),
|
|
||||||
os.Getenv("PG_PASSWORD"),
|
|
||||||
os.Getenv("PG_HOST"),
|
|
||||||
os.Getenv("PG_PORT"),
|
|
||||||
os.Getenv("PG_DATABASE"),
|
|
||||||
)
|
|
||||||
|
|
||||||
// Create a connection pool
|
|
||||||
db, err := sqlx.Open("pgx", connStr)
|
|
||||||
if err != nil {
|
|
||||||
panic(fmt.Sprintf("Unable to connect to database with URL %s: %v\n", connStr, err))
|
|
||||||
}
|
|
||||||
db.SetMaxOpenConns(20)
|
|
||||||
err = db.Ping()
|
|
||||||
if err != nil {
|
|
||||||
panic(fmt.Sprintf("Unable to ping database: %v\n", err))
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Println("Connected to database")
|
|
||||||
return db
|
|
||||||
}
|
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
SELECT
|
|
||||||
query,
|
|
||||||
total_exec_time AS total_time, -- total time spent on the query execution
|
|
||||||
calls, -- number of times the query has been called
|
|
||||||
mean_exec_time AS mean_time -- average time per execution
|
|
||||||
-- max_exec_time AS max_time -- maximum time taken for any single execution
|
|
||||||
FROM
|
|
||||||
pg_stat_statements
|
|
||||||
ORDER BY
|
|
||||||
total_exec_time DESC -- order by total execution time
|
|
||||||
LIMIT 5;
|
|
||||||
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
SELECT pg_stat_statements_reset();
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gemini-grc/uid"
|
|
||||||
"github.com/jmoiron/sqlx"
|
|
||||||
)
|
|
||||||
|
|
||||||
func PopulateDB(db *sqlx.DB) {
|
|
||||||
// Delete all rows in the snapshots table
|
|
||||||
db.MustExec("TRUNCATE snapshots;")
|
|
||||||
|
|
||||||
// Prepare the query for inserting a snapshot with uid, url, and timestamp
|
|
||||||
query := `INSERT INTO snapshots(uid, url, timestamp)
|
|
||||||
VALUES ($1, $2, $3)`
|
|
||||||
|
|
||||||
// Calculate the timestamp for 2 days ago
|
|
||||||
timestamp := time.Now().Add(-48 * time.Hour)
|
|
||||||
|
|
||||||
db.MustExec(query, uid.UID(), "gemini://geminiprotocol.net/", timestamp)
|
|
||||||
db.MustExec(query, uid.UID(), "gemini://warmedal.se/~antenna", timestamp)
|
|
||||||
db.MustExec(query, uid.UID(), "gemini://skyjake.fi/~Cosmos/", timestamp)
|
|
||||||
db.MustExec(query, uid.UID(), "gemini://gemini.circumlunar.space/capcom/", timestamp)
|
|
||||||
db.MustExec(query, uid.UID(), "gemini://auragem.letz.dev/", timestamp)
|
|
||||||
db.MustExec(query, uid.UID(), "gemini://gemplex.space/", timestamp)
|
|
||||||
db.MustExec(query, uid.UID(), "gemini://kennedy.gemi.dev/", timestamp)
|
|
||||||
db.MustExec(query, uid.UID(), "gemini://tlgs.one/", timestamp)
|
|
||||||
}
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
BEGIN;
|
|
||||||
|
|
||||||
SET statement_timeout = '10min';
|
|
||||||
|
|
||||||
TRUNCATE TABLE snapshots;
|
|
||||||
|
|
||||||
INSERT INTO snapshots SELECT * FROM backup;
|
|
||||||
|
|
||||||
COMMIT;
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
WITH DuplicateSnapshots AS (
|
|
||||||
SELECT id,
|
|
||||||
url,
|
|
||||||
ROW_NUMBER() OVER (PARTITION BY url ORDER BY id) AS row_num
|
|
||||||
FROM snapshots
|
|
||||||
)
|
|
||||||
SELECT *
|
|
||||||
FROM DuplicateSnapshots
|
|
||||||
WHERE row_num > 1;
|
|
||||||
@@ -1,5 +0,0 @@
|
|||||||
SELECT
|
|
||||||
COUNT(CASE WHEN response_code IS NOT NULL AND error IS NULL THEN 1 END) AS "Visited",
|
|
||||||
COUNT(CASE WHEN response_code IS NULL AND error IS NULL THEN 1 END) AS "Pending",
|
|
||||||
COUNT(CASE WHEN error IS NOT NULL THEN 1 END) AS "Errors"
|
|
||||||
FROM snapshots;
|
|
||||||
@@ -1,5 +0,0 @@
|
|||||||
SELECT
|
|
||||||
COUNT(*) AS "All",
|
|
||||||
COUNT(CASE WHEN URL ~ '://[^:]+:[0-9]+' THEN 1 END) AS "With port",
|
|
||||||
COUNT(CASE WHEN URL !~ '://[^:]+:[0-9]+' THEN 1 END) AS "Without port"
|
|
||||||
FROM snapshots;
|
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
# Max response size 10MiB
|
|
||||||
MAX_RESPONSE_SIZE=10485760 \
|
|
||||||
LOG_LEVEL=debug \
|
|
||||||
ROOT_PATH=./snaps \
|
|
||||||
RESPONSE_TIMEOUT=10 \
|
|
||||||
NUM_OF_WORKERS=1 \
|
|
||||||
WORKER_BATCH_SIZE=1 \
|
|
||||||
PG_DATABASE=gemini \
|
|
||||||
PG_HOST=127.0.0.1 \
|
|
||||||
PG_PORT=5433 \
|
|
||||||
PG_USER=gemini \
|
|
||||||
PG_PASSWORD=gemini \
|
|
||||||
dlv debug
|
|
||||||
@@ -1,11 +0,0 @@
|
|||||||
// 31 redirect
|
|
||||||
gemini://gemini.circumlunar.space
|
|
||||||
|
|
||||||
// body with null byte
|
|
||||||
gemini://kennedy.gemi.dev/archive/cached?url=gemini://spam.works/mirrors/textfiles/fun/consult.how&t=638427244900000000&raw=False
|
|
||||||
|
|
||||||
// has invalid url
|
|
||||||
gemini://tlgs.one/known-hosts
|
|
||||||
|
|
||||||
// Needs SNI TLS info (our bug)
|
|
||||||
gemini://hanzbrix.pollux.casa/gemlog/20241002.gmi
|
|
||||||
@@ -1,51 +0,0 @@
|
|||||||
package gemini
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"gemini-grc/config"
|
|
||||||
"gemini-grc/logging"
|
|
||||||
)
|
|
||||||
|
|
||||||
var Blacklist *[]string //nolint:gochecknoglobals
|
|
||||||
|
|
||||||
func LoadBlacklist() {
|
|
||||||
if Blacklist == nil {
|
|
||||||
data, err := os.ReadFile(config.CONFIG.BlacklistPath)
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
Blacklist = &[]string{}
|
|
||||||
logging.LogWarn("Could not load Blacklist file: %v", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
lines := strings.Split(string(data), "\n")
|
|
||||||
|
|
||||||
// Ignore lines starting with '#' (comments)
|
|
||||||
filteredLines := func() []string {
|
|
||||||
out := make([]string, 0, len(lines))
|
|
||||||
for _, line := range lines {
|
|
||||||
if !strings.HasPrefix(line, "#") {
|
|
||||||
out = append(out, line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}()
|
|
||||||
|
|
||||||
if len(lines) > 0 {
|
|
||||||
Blacklist = &filteredLines
|
|
||||||
logging.LogInfo("Blacklist has %d entries", len(*Blacklist))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func IsBlacklisted(url URL) bool {
|
|
||||||
hostWithPort := fmt.Sprintf("%s:%d", url.Hostname, url.Port)
|
|
||||||
for _, v := range *Blacklist {
|
|
||||||
if v == url.Hostname || v == hostWithPort {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
@@ -1,32 +0,0 @@
|
|||||||
package gemini
|
|
||||||
|
|
||||||
import (
|
|
||||||
"gemini-grc/logging"
|
|
||||||
)
|
|
||||||
|
|
||||||
var IpPool = IpAddressPool{IPs: make(map[string]int)}
|
|
||||||
|
|
||||||
func AddIPsToPool(ips []string) {
|
|
||||||
IpPool.Lock.Lock()
|
|
||||||
for _, ip := range ips {
|
|
||||||
logging.LogDebug("Adding %s to pool", ip)
|
|
||||||
IpPool.IPs[ip]++
|
|
||||||
}
|
|
||||||
IpPool.Lock.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
func RemoveIPsFromPool(IPs []string) {
|
|
||||||
IpPool.Lock.Lock()
|
|
||||||
for _, ip := range IPs {
|
|
||||||
_, ok := IpPool.IPs[ip]
|
|
||||||
if ok {
|
|
||||||
logging.LogDebug("Removing %s from pool", ip)
|
|
||||||
if IpPool.IPs[ip] == 1 {
|
|
||||||
delete(IpPool.IPs, ip)
|
|
||||||
} else {
|
|
||||||
IpPool.IPs[ip]--
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
IpPool.Lock.Unlock()
|
|
||||||
}
|
|
||||||
@@ -1,101 +0,0 @@
|
|||||||
package gemini
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
)
|
|
||||||
|
|
||||||
type GeminiError struct {
|
|
||||||
Msg string
|
|
||||||
Code int
|
|
||||||
Header string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *GeminiError) Error() string {
|
|
||||||
return fmt.Sprintf("%s: %s", e.Msg, e.Header)
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewErrGeminiStatusCode(code int, header string) error {
|
|
||||||
var msg string
|
|
||||||
switch {
|
|
||||||
case code >= 10 && code < 20:
|
|
||||||
msg = "needs input"
|
|
||||||
case code >= 30 && code < 40:
|
|
||||||
msg = "redirect"
|
|
||||||
case code >= 40 && code < 50:
|
|
||||||
msg = "bad request"
|
|
||||||
case code >= 50 && code < 60:
|
|
||||||
msg = "server error"
|
|
||||||
case code >= 60 && code < 70:
|
|
||||||
msg = "TLS error"
|
|
||||||
default:
|
|
||||||
msg = "unexpected status code"
|
|
||||||
}
|
|
||||||
return &GeminiError{
|
|
||||||
Msg: msg,
|
|
||||||
Code: code,
|
|
||||||
Header: header,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
ErrGeminiRobotsParse = errors.New("gemini robots.txt parse error")
|
|
||||||
ErrGeminiRobotsDisallowed = errors.New("gemini robots.txt disallowed")
|
|
||||||
ErrGeminiResponseHeader = errors.New("gemini response header error")
|
|
||||||
ErrGeminiLinkLineParse = errors.New("gemini link line parse error")
|
|
||||||
|
|
||||||
ErrURLParse = errors.New("URL parse error")
|
|
||||||
ErrURLDecode = errors.New("URL decode error")
|
|
||||||
ErrUTF8Parse = errors.New("UTF-8 parse error")
|
|
||||||
ErrTextParse = errors.New("text parse error")
|
|
||||||
|
|
||||||
ErrNetwork = errors.New("network error")
|
|
||||||
ErrNetworkDNS = errors.New("network DNS error")
|
|
||||||
ErrNetworkTLS = errors.New("network TLS error")
|
|
||||||
ErrNetworkSetConnectionDeadline = errors.New("network error - cannot set connection deadline")
|
|
||||||
ErrNetworkCannotWrite = errors.New("network error - cannot write")
|
|
||||||
ErrNetworkResponseSizeExceededMax = errors.New("network error - response size exceeded maximum size")
|
|
||||||
|
|
||||||
ErrDatabase = errors.New("database error")
|
|
||||||
)
|
|
||||||
|
|
||||||
// We could have used a map for speed, but
|
|
||||||
// we would lose ability to check wrapped
|
|
||||||
// errors via errors.Is().
|
|
||||||
|
|
||||||
var errGemini *GeminiError
|
|
||||||
|
|
||||||
var knownErrors = []error{ //nolint:gochecknoglobals
|
|
||||||
errGemini,
|
|
||||||
ErrGeminiLinkLineParse,
|
|
||||||
ErrGeminiRobotsParse,
|
|
||||||
ErrGeminiRobotsDisallowed,
|
|
||||||
ErrGeminiResponseHeader,
|
|
||||||
|
|
||||||
ErrURLParse,
|
|
||||||
ErrURLDecode,
|
|
||||||
ErrUTF8Parse,
|
|
||||||
ErrTextParse,
|
|
||||||
|
|
||||||
ErrNetwork,
|
|
||||||
ErrNetworkDNS,
|
|
||||||
ErrNetworkTLS,
|
|
||||||
ErrNetworkSetConnectionDeadline,
|
|
||||||
ErrNetworkCannotWrite,
|
|
||||||
ErrNetworkResponseSizeExceededMax,
|
|
||||||
|
|
||||||
ErrDatabase,
|
|
||||||
}
|
|
||||||
|
|
||||||
func IsKnownError(err error) bool {
|
|
||||||
for _, known := range knownErrors {
|
|
||||||
if errors.Is(err, known) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Check for wrapped errors as well
|
|
||||||
if errors.As(err, new(*GeminiError)) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
package gemini
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestErrGemini(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
err := NewErrGeminiStatusCode(50, "50 server error")
|
|
||||||
if !errors.As(err, new(*GeminiError)) {
|
|
||||||
t.Errorf("TestErrGemini fail")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestErrGeminiWrapped(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
err := NewErrGeminiStatusCode(50, "50 server error")
|
|
||||||
errWrapped := fmt.Errorf("%w wrapped", err)
|
|
||||||
if !errors.As(errWrapped, new(*GeminiError)) {
|
|
||||||
t.Errorf("TestErrGeminiWrapped fail")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
-113
@@ -1,113 +0,0 @@
|
|||||||
package gemini
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"net/url"
|
|
||||||
"os"
|
|
||||||
"path"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"gemini-grc/logging"
|
|
||||||
)
|
|
||||||
|
|
||||||
// sanitizePath encodes invalid filesystem characters using URL encoding.
|
|
||||||
// Example:
|
|
||||||
// /example/path/to/page?query=param&another=value
|
|
||||||
// would become
|
|
||||||
// example/path/to/page%3Fquery%3Dparam%26another%3Dvalue
|
|
||||||
func sanitizePath(p string) string {
|
|
||||||
// Split the path into its components
|
|
||||||
components := strings.Split(p, "/")
|
|
||||||
|
|
||||||
// Encode each component separately
|
|
||||||
for i, component := range components {
|
|
||||||
// Decode any existing percent-encoded characters
|
|
||||||
decodedComponent, err := url.PathUnescape(component)
|
|
||||||
if err != nil {
|
|
||||||
decodedComponent = component // Fallback to original if unescape fails
|
|
||||||
}
|
|
||||||
|
|
||||||
// Encode the component to escape invalid filesystem characters
|
|
||||||
encodedComponent := url.QueryEscape(decodedComponent)
|
|
||||||
|
|
||||||
// Replace '+' (from QueryEscape) with '%20' to handle spaces correctly
|
|
||||||
encodedComponent = strings.ReplaceAll(encodedComponent, "+", "%20")
|
|
||||||
|
|
||||||
components[i] = encodedComponent
|
|
||||||
}
|
|
||||||
|
|
||||||
// Rejoin the components into a sanitized path
|
|
||||||
safe := filepath.Join(components...)
|
|
||||||
|
|
||||||
return safe
|
|
||||||
}
|
|
||||||
|
|
||||||
// getFilePath constructs a safe file path from the root path and URL path.
|
|
||||||
// It URL-encodes invalid filesystem characters to ensure the path is valid.
|
|
||||||
func calcFilePath(rootPath, urlPath string) (string, error) {
|
|
||||||
// Normalize the URL path
|
|
||||||
cleanPath := filepath.Clean(urlPath)
|
|
||||||
|
|
||||||
// Safe check to prevent directory traversal
|
|
||||||
if strings.Contains(cleanPath, "..") {
|
|
||||||
return "", fmt.Errorf("Invalid URL path: contains directory traversal")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sanitize the path by encoding invalid characters
|
|
||||||
safePath := sanitizePath(cleanPath)
|
|
||||||
|
|
||||||
// Join the root path and the sanitized URL path
|
|
||||||
finalPath := filepath.Join(rootPath, safePath)
|
|
||||||
|
|
||||||
return finalPath, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func SaveToFile(rootPath string, s *Snapshot, done chan struct{}) {
|
|
||||||
parentPath := path.Join(rootPath, s.URL.Hostname)
|
|
||||||
urlPath := s.URL.Path
|
|
||||||
// If path is empty, add `index.gmi` as the file to save
|
|
||||||
if urlPath == "" || urlPath == "." {
|
|
||||||
urlPath = "index.gmi"
|
|
||||||
}
|
|
||||||
// If path ends with '/' then add index.gmi for the
|
|
||||||
// directory to be created.
|
|
||||||
if strings.HasSuffix(urlPath, "/") {
|
|
||||||
urlPath = strings.Join([]string{urlPath, "index.gmi"}, "")
|
|
||||||
}
|
|
||||||
|
|
||||||
finalPath, err := calcFilePath(parentPath, urlPath)
|
|
||||||
if err != nil {
|
|
||||||
logging.LogError("GeminiError saving %s: %w", s.URL, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
// Ensure the directory exists
|
|
||||||
dir := filepath.Dir(finalPath)
|
|
||||||
if err := os.MkdirAll(dir, os.ModePerm); err != nil {
|
|
||||||
logging.LogError("Failed to create directory: %w", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if s.MimeType.Valid && s.MimeType.String == "text/gemini" {
|
|
||||||
err = os.WriteFile(finalPath, (*s).Data.V, 0o666)
|
|
||||||
} else {
|
|
||||||
err = os.WriteFile(finalPath, []byte((*s).GemText.String), 0o666)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
logging.LogError("GeminiError saving %s: %w", s.URL.Full, err)
|
|
||||||
}
|
|
||||||
close(done)
|
|
||||||
}
|
|
||||||
|
|
||||||
func ReadLines(path string) []string {
|
|
||||||
data, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
panic(fmt.Sprintf("Failed to read file: %s", err))
|
|
||||||
}
|
|
||||||
lines := strings.Split(string(data), "\n")
|
|
||||||
// Remove last line if empty
|
|
||||||
// (happens when file ends with '\n')
|
|
||||||
if lines[len(lines)-1] == "" {
|
|
||||||
lines = lines[:len(lines)-1]
|
|
||||||
}
|
|
||||||
return lines
|
|
||||||
}
|
|
||||||
@@ -1,144 +0,0 @@
|
|||||||
package gemini
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"net/url"
|
|
||||||
"regexp"
|
|
||||||
"strconv"
|
|
||||||
|
|
||||||
"gemini-grc/logging"
|
|
||||||
)
|
|
||||||
|
|
||||||
func GetPageLinks(currentURL URL, gemtext string) LinkList {
|
|
||||||
// Grab link lines
|
|
||||||
linkLines := ExtractLinkLines(gemtext)
|
|
||||||
if len(linkLines) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
var linkURLs LinkList
|
|
||||||
// Normalize URLs in links, and store them in snapshot
|
|
||||||
for _, line := range linkLines {
|
|
||||||
normalizedLink, descr, err := NormalizeLink(line, currentURL.String())
|
|
||||||
if err != nil {
|
|
||||||
logging.LogDebug("Cannot normalize URL in line '%s': %v", line, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
geminiUrl, err := ParseURL(normalizedLink, descr)
|
|
||||||
if err != nil {
|
|
||||||
logging.LogDebug("Cannot parse URL in link '%s': %v", line, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
logging.LogDebug(geminiUrl.String())
|
|
||||||
linkURLs = append(linkURLs, *geminiUrl)
|
|
||||||
}
|
|
||||||
return linkURLs
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExtractLinkLines takes a Gemtext document as a string and returns all lines that are link lines
|
|
||||||
func ExtractLinkLines(gemtext string) []string {
|
|
||||||
// Define the regular expression pattern to match link lines
|
|
||||||
re := regexp.MustCompile(`(?m)^=>[ \t]+.*`)
|
|
||||||
|
|
||||||
// Find all matches using the regular expression
|
|
||||||
matches := re.FindAllString(gemtext, -1)
|
|
||||||
|
|
||||||
return matches
|
|
||||||
}
|
|
||||||
|
|
||||||
// NormalizeLink takes a single link line and the current URL,
|
|
||||||
// return the URL converted to an absolute URL
|
|
||||||
// and its description.
|
|
||||||
func NormalizeLink(linkLine string, currentURL string) (string, string, error) {
|
|
||||||
// Parse the current URL
|
|
||||||
baseURL, err := url.Parse(currentURL)
|
|
||||||
if err != nil {
|
|
||||||
return "", "", fmt.Errorf("%w: %w", ErrURLParse, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Regular expression to extract the URL part from a link line
|
|
||||||
re := regexp.MustCompile(`^=>[ \t]+(\S+)([ \t]+.*)?`)
|
|
||||||
|
|
||||||
// Use regex to extract the URL and the rest of the line
|
|
||||||
matches := re.FindStringSubmatch(linkLine)
|
|
||||||
if len(matches) == 0 {
|
|
||||||
// If the line doesn't match the expected format, return it unchanged
|
|
||||||
return "", "", fmt.Errorf("%w for link line %s", ErrGeminiLinkLineParse, linkLine)
|
|
||||||
}
|
|
||||||
|
|
||||||
originalURLStr := matches[1]
|
|
||||||
_, err = url.QueryUnescape(originalURLStr)
|
|
||||||
if err != nil {
|
|
||||||
return "", "", fmt.Errorf("%w: %w", ErrURLDecode, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
restOfLine := ""
|
|
||||||
if len(matches) > 2 {
|
|
||||||
restOfLine = matches[2]
|
|
||||||
}
|
|
||||||
|
|
||||||
// Parse the URL from the link line
|
|
||||||
parsedURL, err := url.Parse(originalURLStr)
|
|
||||||
if err != nil {
|
|
||||||
// If URL parsing fails, return an error
|
|
||||||
return "", "", fmt.Errorf("%w: %w", ErrURLParse, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Resolve relative URLs against the base URL
|
|
||||||
if !parsedURL.IsAbs() {
|
|
||||||
parsedURL = baseURL.ResolveReference(parsedURL)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Construct the canonicalized link line
|
|
||||||
canonicalURLStr := parsedURL.String()
|
|
||||||
|
|
||||||
// Remove usual first space from URL description:
|
|
||||||
// => URL description
|
|
||||||
// ^^^^^^^^^^^^
|
|
||||||
if len(restOfLine) > 0 && restOfLine[0] == ' ' {
|
|
||||||
restOfLine = restOfLine[1:]
|
|
||||||
}
|
|
||||||
|
|
||||||
return canonicalURLStr, restOfLine, nil
|
|
||||||
// canonicalizedLine := fmt.Sprintf("=> %s%s", canonicalURLStr, restOfLine)
|
|
||||||
// return canonicalizedLine, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ParseFirstTwoDigits takes a string and returns the first one or two digits as an int.
|
|
||||||
// If no valid digits are found, it returns an error.
|
|
||||||
func ParseFirstTwoDigits(input string) (int, error) {
|
|
||||||
// Define the regular expression pattern to match one or two leading digits
|
|
||||||
re := regexp.MustCompile(`^(\d{1,2})`)
|
|
||||||
|
|
||||||
// Find the first match in the string
|
|
||||||
matches := re.FindStringSubmatch(input)
|
|
||||||
if len(matches) == 0 {
|
|
||||||
return 0, fmt.Errorf("%w", ErrGeminiResponseHeader)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Parse the captured match as an integer
|
|
||||||
snapshot, err := strconv.Atoi(matches[1])
|
|
||||||
if err != nil {
|
|
||||||
return 0, fmt.Errorf("%w: %w", ErrTextParse, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return snapshot, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// extractRedirectTarget returns the redirection
|
|
||||||
// URL by parsing the header (or error message)
|
|
||||||
func extractRedirectTarget(currentURL URL, input string) (*URL, error) {
|
|
||||||
// \d+ - matches one or more digits
|
|
||||||
// \s+ - matches one or more whitespace
|
|
||||||
// ([^\r]+) - captures everything until it hits a \r (or end of string)
|
|
||||||
pattern := `\d+\s+([^\r]+)`
|
|
||||||
re := regexp.MustCompile(pattern)
|
|
||||||
matches := re.FindStringSubmatch(input)
|
|
||||||
if len(matches) < 2 {
|
|
||||||
return nil, fmt.Errorf("%w: Cannot find redirect target from header %s", ErrGeminiResponseHeader, input)
|
|
||||||
}
|
|
||||||
newURL, err := DeriveAbsoluteURL(currentURL, matches[1])
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: Cannot find redirect target from header: %w", ErrGeminiResponseHeader, err)
|
|
||||||
}
|
|
||||||
return newURL, nil
|
|
||||||
}
|
|
||||||
@@ -1,47 +0,0 @@
|
|||||||
package gemini
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestExtractRedirectTargetFullURL(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
currentURL, _ := ParseURL("gemini://smol.gr", "")
|
|
||||||
input := "redirect: 31 gemini://target.gr"
|
|
||||||
result, err := extractRedirectTarget(*currentURL, input)
|
|
||||||
if err != nil || (result.String() != "gemini://target.gr:1965") {
|
|
||||||
t.Errorf("fail: %s", result)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestExtractRedirectTargetRelativeURL(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
currentURL, _ := ParseURL("gemini://smol.gr", "")
|
|
||||||
input := "redirect: 31 /a/b"
|
|
||||||
result, err := extractRedirectTarget(*currentURL, input)
|
|
||||||
if err != nil || (result.String() != "gemini://smol.gr:1965/a/b") {
|
|
||||||
t.Errorf("fail: %s", result)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestExtractRedirectTargetRelativeURL2(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
currentURL, _ := ParseURL("gemini://nox.im:1965", "")
|
|
||||||
input := "redirect: 31 ./"
|
|
||||||
result, err := extractRedirectTarget(*currentURL, input)
|
|
||||||
if err != nil || (result.String() != "gemini://nox.im:1965/") {
|
|
||||||
t.Errorf("fail: %s", result)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestExtractRedirectTargetWrong(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
currentURL, _ := ParseURL("gemini://smol.gr", "")
|
|
||||||
input := "redirect: 31 fsdsdf"
|
|
||||||
result, err := extractRedirectTarget(*currentURL, input)
|
|
||||||
fmt.Println(err)
|
|
||||||
if result != nil || err == nil {
|
|
||||||
t.Errorf("fail: result should be nil, err is %s", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,97 +0,0 @@
|
|||||||
package gemini
|
|
||||||
|
|
||||||
import (
|
|
||||||
"database/sql/driver"
|
|
||||||
"fmt"
|
|
||||||
"gemini-grc/logging"
|
|
||||||
"net/url"
|
|
||||||
"path"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
type URL struct {
|
|
||||||
Protocol string `json:"protocol,omitempty"`
|
|
||||||
Hostname string `json:"hostname,omitempty"`
|
|
||||||
Port int `json:"port,omitempty"`
|
|
||||||
Path string `json:"path,omitempty"`
|
|
||||||
Descr string `json:"descr,omitempty"`
|
|
||||||
Full string `json:"full,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *URL) Scan(value interface{}) error {
|
|
||||||
if value == nil {
|
|
||||||
// Clear the fields in the current GeminiUrl object (not the pointer itself)
|
|
||||||
*u = URL{}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
b, ok := value.(string)
|
|
||||||
if !ok {
|
|
||||||
return fmt.Errorf("failed to scan GeminiUrl: expected string, got %T", value)
|
|
||||||
}
|
|
||||||
parsedURL, err := ParseURL(b, "")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
*u = *parsedURL
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u URL) String() string {
|
|
||||||
return u.Full
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u URL) StringNoDefaultPort() string {
|
|
||||||
if u.Port == 1965 {
|
|
||||||
return fmt.Sprintf("%s://%s%s", u.Protocol, u.Hostname, u.Path)
|
|
||||||
}
|
|
||||||
return u.Full
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u URL) Value() (driver.Value, error) {
|
|
||||||
if u.Full == "" {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return u.Full, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func ParseURL(input string, descr string) (*URL, error) {
|
|
||||||
u, err := url.Parse(input)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: Input %s GeminiError %w", ErrURLParse, input, err)
|
|
||||||
}
|
|
||||||
protocol := u.Scheme
|
|
||||||
hostname := u.Hostname()
|
|
||||||
strPort := u.Port()
|
|
||||||
path := u.Path
|
|
||||||
if strPort == "" {
|
|
||||||
strPort = "1965"
|
|
||||||
}
|
|
||||||
port, err := strconv.Atoi(strPort)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: Input %s GeminiError %w", ErrURLParse, input, err)
|
|
||||||
}
|
|
||||||
full := fmt.Sprintf("%s://%s:%d%s", protocol, hostname, port, path)
|
|
||||||
return &URL{Protocol: protocol, Hostname: hostname, Port: port, Path: path, Descr: descr, Full: full}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func DeriveAbsoluteURL(currentURL URL, input string) (*URL, error) {
|
|
||||||
logging.LogDebug("Calculating redirect URL. Current %s header string %s", currentURL, input)
|
|
||||||
// If URL is absolute, return just it
|
|
||||||
if strings.Contains(input, "://") {
|
|
||||||
return ParseURL(input, "")
|
|
||||||
}
|
|
||||||
// input is a path. Clean it and construct
|
|
||||||
// new path
|
|
||||||
var newPath string
|
|
||||||
// Handle weird cases found in the wild
|
|
||||||
if strings.HasPrefix(input, "/") {
|
|
||||||
newPath = path.Clean(input)
|
|
||||||
} else if input == "./" || input == "." {
|
|
||||||
newPath = path.Join(currentURL.Path, "/")
|
|
||||||
} else {
|
|
||||||
newPath = path.Join(currentURL.Path, path.Clean(input))
|
|
||||||
}
|
|
||||||
strURL := fmt.Sprintf("%s://%s:%d%s", currentURL.Protocol, currentURL.Hostname, currentURL.Port, newPath)
|
|
||||||
return ParseURL(strURL, "")
|
|
||||||
}
|
|
||||||
@@ -1,103 +0,0 @@
|
|||||||
package gemini
|
|
||||||
|
|
||||||
import (
|
|
||||||
"reflect"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestParseURL(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
input := "gemini://caolan.uk/cgi-bin/weather.py/wxfcs/3162"
|
|
||||||
parsed, err := ParseURL(input, "")
|
|
||||||
value, _ := parsed.Value()
|
|
||||||
if err != nil || !(value == "gemini://caolan.uk:1965/cgi-bin/weather.py/wxfcs/3162") {
|
|
||||||
t.Errorf("fail: %s", parsed)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDeriveAbsoluteURL_abs_url_input(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
currentURL := URL{
|
|
||||||
Protocol: "gemini",
|
|
||||||
Hostname: "smol.gr",
|
|
||||||
Port: 1965,
|
|
||||||
Path: "/a/b",
|
|
||||||
Descr: "Nothing",
|
|
||||||
Full: "gemini://smol.gr:1965/a/b",
|
|
||||||
}
|
|
||||||
input := "gemini://a.b/c"
|
|
||||||
output, err := DeriveAbsoluteURL(currentURL, input)
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("fail: %v", err)
|
|
||||||
}
|
|
||||||
expected := &URL{
|
|
||||||
Protocol: "gemini",
|
|
||||||
Hostname: "a.b",
|
|
||||||
Port: 1965,
|
|
||||||
Path: "/c",
|
|
||||||
Descr: "",
|
|
||||||
Full: "gemini://a.b:1965/c",
|
|
||||||
}
|
|
||||||
pass := reflect.DeepEqual(output, expected)
|
|
||||||
if !pass {
|
|
||||||
t.Errorf("fail: %#v != %#v", output, expected)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDeriveAbsoluteURL_abs_path_input(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
currentURL := URL{
|
|
||||||
Protocol: "gemini",
|
|
||||||
Hostname: "smol.gr",
|
|
||||||
Port: 1965,
|
|
||||||
Path: "/a/b",
|
|
||||||
Descr: "Nothing",
|
|
||||||
Full: "gemini://smol.gr:1965/a/b",
|
|
||||||
}
|
|
||||||
input := "/c"
|
|
||||||
output, err := DeriveAbsoluteURL(currentURL, input)
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("fail: %v", err)
|
|
||||||
}
|
|
||||||
expected := &URL{
|
|
||||||
Protocol: "gemini",
|
|
||||||
Hostname: "smol.gr",
|
|
||||||
Port: 1965,
|
|
||||||
Path: "/c",
|
|
||||||
Descr: "",
|
|
||||||
Full: "gemini://smol.gr:1965/c",
|
|
||||||
}
|
|
||||||
pass := reflect.DeepEqual(output, expected)
|
|
||||||
if !pass {
|
|
||||||
t.Errorf("fail: %#v != %#v", output, expected)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDeriveAbsoluteURL_rel_path_input(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
currentURL := URL{
|
|
||||||
Protocol: "gemini",
|
|
||||||
Hostname: "smol.gr",
|
|
||||||
Port: 1965,
|
|
||||||
Path: "/a/b",
|
|
||||||
Descr: "Nothing",
|
|
||||||
Full: "gemini://smol.gr:1965/a/b",
|
|
||||||
}
|
|
||||||
input := "c/d"
|
|
||||||
output, err := DeriveAbsoluteURL(currentURL, input)
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("fail: %v", err)
|
|
||||||
}
|
|
||||||
expected := &URL{
|
|
||||||
Protocol: "gemini",
|
|
||||||
Hostname: "smol.gr",
|
|
||||||
Port: 1965,
|
|
||||||
Path: "/a/b/c/d",
|
|
||||||
Descr: "",
|
|
||||||
Full: "gemini://smol.gr:1965/a/b/c/d",
|
|
||||||
}
|
|
||||||
pass := reflect.DeepEqual(output, expected)
|
|
||||||
if !pass {
|
|
||||||
t.Errorf("fail: %#v != %#v", output, expected)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,54 +0,0 @@
|
|||||||
package gemini
|
|
||||||
|
|
||||||
import "sync"
|
|
||||||
|
|
||||||
// Used to limit requests per
|
|
||||||
// IP address. Maps IP address
|
|
||||||
// to number of active connections.
|
|
||||||
type IpAddressPool struct {
|
|
||||||
IPs map[string]int
|
|
||||||
Lock sync.RWMutex
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *IpAddressPool) Set(key string, value int) {
|
|
||||||
p.Lock.Lock() // Lock for writing
|
|
||||||
defer p.Lock.Unlock() // Ensure mutex is unlocked after the write
|
|
||||||
p.IPs[key] = value
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *IpAddressPool) Get(key string) int {
|
|
||||||
p.Lock.RLock() // Lock for reading
|
|
||||||
defer p.Lock.RUnlock() // Ensure mutex is unlocked after reading
|
|
||||||
if value, ok := p.IPs[key]; !ok {
|
|
||||||
return 0
|
|
||||||
} else {
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *IpAddressPool) Delete(key string) {
|
|
||||||
p.Lock.Lock()
|
|
||||||
defer p.Lock.Unlock()
|
|
||||||
delete(p.IPs, key)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *IpAddressPool) Incr(key string) {
|
|
||||||
p.Lock.Lock()
|
|
||||||
defer p.Lock.Unlock()
|
|
||||||
if _, ok := p.IPs[key]; !ok {
|
|
||||||
p.IPs[key] = 1
|
|
||||||
} else {
|
|
||||||
p.IPs[key] = p.IPs[key] + 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *IpAddressPool) Decr(key string) {
|
|
||||||
p.Lock.Lock()
|
|
||||||
defer p.Lock.Unlock()
|
|
||||||
if val, ok := p.IPs[key]; ok {
|
|
||||||
p.IPs[key] = val - 1
|
|
||||||
if p.IPs[key] == 0 {
|
|
||||||
delete(p.IPs, key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,244 +0,0 @@
|
|||||||
package gemini
|
|
||||||
|
|
||||||
import (
|
|
||||||
"crypto/tls"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"gemini-grc/logging"
|
|
||||||
"io"
|
|
||||||
"net"
|
|
||||||
gourl "net/url"
|
|
||||||
"regexp"
|
|
||||||
"slices"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gemini-grc/config"
|
|
||||||
|
|
||||||
"github.com/guregu/null/v5"
|
|
||||||
)
|
|
||||||
|
|
||||||
type PageData struct {
|
|
||||||
ResponseCode int
|
|
||||||
ResponseHeader string
|
|
||||||
MimeType string
|
|
||||||
Lang string
|
|
||||||
GemText string
|
|
||||||
Data []byte
|
|
||||||
}
|
|
||||||
|
|
||||||
// Resolve the URL hostname and
|
|
||||||
// check if we already have an open
|
|
||||||
// connection to this host.
|
|
||||||
// If we can connect, return a list
|
|
||||||
// of the resolved IPs.
|
|
||||||
func getHostIPAddresses(hostname string) ([]string, error) {
|
|
||||||
addrs, err := net.LookupHost(hostname)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w:%w", ErrNetworkDNS, err)
|
|
||||||
}
|
|
||||||
IpPool.Lock.RLock()
|
|
||||||
defer func() {
|
|
||||||
IpPool.Lock.RUnlock()
|
|
||||||
}()
|
|
||||||
return addrs, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func ConnectAndGetData(url string) ([]byte, error) {
|
|
||||||
parsedURL, err := gourl.Parse(url)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %w", ErrURLParse, err)
|
|
||||||
}
|
|
||||||
hostname := parsedURL.Hostname()
|
|
||||||
port := parsedURL.Port()
|
|
||||||
if port == "" {
|
|
||||||
port = "1965"
|
|
||||||
}
|
|
||||||
host := fmt.Sprintf("%s:%s", hostname, port)
|
|
||||||
// Establish the underlying TCP connection.
|
|
||||||
dialer := &net.Dialer{
|
|
||||||
Timeout: time.Duration(config.CONFIG.ResponseTimeout) * time.Second,
|
|
||||||
}
|
|
||||||
conn, err := dialer.Dial("tcp", host)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %w", ErrNetwork, err)
|
|
||||||
}
|
|
||||||
// Make sure we always close the connection.
|
|
||||||
defer func() {
|
|
||||||
// No need to handle error:
|
|
||||||
// Connection will time out eventually if still open somehow.
|
|
||||||
_ = conn.Close()
|
|
||||||
}()
|
|
||||||
|
|
||||||
// Set read and write timeouts on the TCP connection.
|
|
||||||
err = conn.SetReadDeadline(time.Now().Add(time.Duration(config.CONFIG.ResponseTimeout) * time.Second))
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %w", ErrNetworkSetConnectionDeadline, err)
|
|
||||||
}
|
|
||||||
err = conn.SetWriteDeadline(time.Now().Add(time.Duration(config.CONFIG.ResponseTimeout) * time.Second))
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %w", ErrNetworkSetConnectionDeadline, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Perform the TLS handshake
|
|
||||||
tlsConfig := &tls.Config{
|
|
||||||
InsecureSkipVerify: true, //nolint:gosec // Accept all TLS certs, even if insecure.
|
|
||||||
ServerName: parsedURL.Hostname(), // SNI should not include port
|
|
||||||
// MinVersion: tls.VersionTLS12, // Use a minimum TLS version. Warning breaks a lot of sites.
|
|
||||||
}
|
|
||||||
tlsConn := tls.Client(conn, tlsConfig)
|
|
||||||
if err := tlsConn.Handshake(); err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %w", ErrNetworkTLS, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// We read `buf`-sized chunks and add data to `data`.
|
|
||||||
buf := make([]byte, 4096)
|
|
||||||
var data []byte
|
|
||||||
|
|
||||||
// Send Gemini request to trigger server response.
|
|
||||||
// Fix for stupid server bug:
|
|
||||||
// Some servers return 'Header: 53 No proxying to other hosts or ports!'
|
|
||||||
// when the port is 1965 and is still specified explicitely in the URL.
|
|
||||||
_url, _ := ParseURL(url, "")
|
|
||||||
_, err = tlsConn.Write([]byte(fmt.Sprintf("%s\r\n", _url.StringNoDefaultPort())))
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %w", ErrNetworkCannotWrite, err)
|
|
||||||
}
|
|
||||||
// Read response bytes in len(buf) byte chunks
|
|
||||||
for {
|
|
||||||
n, err := tlsConn.Read(buf)
|
|
||||||
if n > 0 {
|
|
||||||
data = append(data, buf[:n]...)
|
|
||||||
}
|
|
||||||
if len(data) > config.CONFIG.MaxResponseSize {
|
|
||||||
return nil, fmt.Errorf("%w: %v", ErrNetworkResponseSizeExceededMax, config.CONFIG.MaxResponseSize)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, io.EOF) {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
return nil, fmt.Errorf("%w: %w", ErrNetwork, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return data, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Visit given URL, using the Gemini protocol.
|
|
||||||
// Mutates given Snapshot with the data.
|
|
||||||
// In case of error, we store the error string
|
|
||||||
// inside snapshot and return the error.
|
|
||||||
func Visit(s *Snapshot) (err error) {
|
|
||||||
// Don't forget to also store error
|
|
||||||
// response code (if we have one)
|
|
||||||
defer func() {
|
|
||||||
if err != nil {
|
|
||||||
s.Error = null.StringFrom(err.Error())
|
|
||||||
if errors.As(err, new(*GeminiError)) {
|
|
||||||
s.ResponseCode = null.IntFrom(int64(err.(*GeminiError).Code))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
data, err := ConnectAndGetData(s.URL.String())
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
pageData, err := processData(data)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
//marshalled, _ := json.MarshalIndent(pageData, "", " ")
|
|
||||||
//fmt.Printf("%s\n", marshalled)
|
|
||||||
s.Header = null.StringFrom(pageData.ResponseHeader)
|
|
||||||
s.ResponseCode = null.IntFrom(int64(pageData.ResponseCode))
|
|
||||||
s.MimeType = null.StringFrom(pageData.MimeType)
|
|
||||||
s.Lang = null.StringFrom(pageData.Lang)
|
|
||||||
if pageData.GemText != "" {
|
|
||||||
s.GemText = null.StringFrom(pageData.GemText)
|
|
||||||
}
|
|
||||||
if pageData.Data != nil {
|
|
||||||
s.Data = null.ValueFrom(pageData.Data)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// processData returne results from
|
|
||||||
// parsing Gemini header data:
|
|
||||||
// Code, mime type and lang (optional)
|
|
||||||
// Returns error if header was invalid
|
|
||||||
func processData(data []byte) (*PageData, error) {
|
|
||||||
header, body, err := getHeadersAndData(data)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
code, mimeType, lang := getMimeTypeAndLang(header)
|
|
||||||
logging.LogDebug("Header: %s", strings.TrimSpace(header))
|
|
||||||
if code != 20 {
|
|
||||||
return nil, NewErrGeminiStatusCode(code, header)
|
|
||||||
}
|
|
||||||
|
|
||||||
pageData := PageData{
|
|
||||||
ResponseCode: code,
|
|
||||||
ResponseHeader: header,
|
|
||||||
MimeType: mimeType,
|
|
||||||
Lang: lang,
|
|
||||||
}
|
|
||||||
// If we've got a Gemini document, populate
|
|
||||||
// `GemText` field, otherwise raw data goes to `Data`.
|
|
||||||
if mimeType == "text/gemini" {
|
|
||||||
validBody, err := BytesToValidUTF8(body)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %w", ErrUTF8Parse, err)
|
|
||||||
}
|
|
||||||
pageData.GemText = validBody
|
|
||||||
} else {
|
|
||||||
pageData.Data = body
|
|
||||||
}
|
|
||||||
return &pageData, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Checks for a Gemini header, which is
|
|
||||||
// basically the first line of the response
|
|
||||||
// and should contain the response code,
|
|
||||||
// mimeType and language.
|
|
||||||
func getHeadersAndData(data []byte) (string, []byte, error) {
|
|
||||||
firstLineEnds := slices.Index(data, '\n')
|
|
||||||
if firstLineEnds == -1 {
|
|
||||||
return "", nil, ErrGeminiResponseHeader
|
|
||||||
}
|
|
||||||
firstLine := string(data[:firstLineEnds])
|
|
||||||
rest := data[firstLineEnds+1:]
|
|
||||||
return firstLine, rest, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Parses code, mime type and language
|
|
||||||
// from a Gemini header.
|
|
||||||
// Examples:
|
|
||||||
// `20 text/gemini lang=en` (code, mimetype, lang)
|
|
||||||
// `20 text/gemini` (code, mimetype)
|
|
||||||
// `31 gemini://redirected.to/other/site` (code)
|
|
||||||
func getMimeTypeAndLang(headers string) (int, string, string) {
|
|
||||||
// Regex that parses code, mimetype & optional charset/lang parameters
|
|
||||||
re := regexp.MustCompile(`^(\d+)\s+([a-zA-Z0-9/\-+]+)(?:[;\s]+(?:(?:charset|lang)=([a-zA-Z0-9-]+)))?\s*$`)
|
|
||||||
matches := re.FindStringSubmatch(headers)
|
|
||||||
if matches == nil || len(matches) <= 1 {
|
|
||||||
// Try to get code at least
|
|
||||||
re := regexp.MustCompile(`^(\d+)\s+`)
|
|
||||||
matches := re.FindStringSubmatch(headers)
|
|
||||||
if matches == nil || len(matches) <= 1 {
|
|
||||||
return 0, "", ""
|
|
||||||
}
|
|
||||||
code, err := strconv.Atoi(matches[1])
|
|
||||||
if err != nil {
|
|
||||||
return 0, "", ""
|
|
||||||
}
|
|
||||||
return code, "", ""
|
|
||||||
}
|
|
||||||
code, err := strconv.Atoi(matches[1])
|
|
||||||
if err != nil {
|
|
||||||
return 0, "", ""
|
|
||||||
}
|
|
||||||
mimeType := matches[2]
|
|
||||||
param := matches[3] // This will capture either charset or lang value
|
|
||||||
return code, mimeType, param
|
|
||||||
}
|
|
||||||
@@ -1,78 +0,0 @@
|
|||||||
package gemini
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Test for input: `20 text/gemini`
|
|
||||||
func TestGetMimeTypeAndLang1(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
code, mimeType, lang := getMimeTypeAndLang("20 text/gemini")
|
|
||||||
if code != 20 || mimeType != "text/gemini" || lang != "" {
|
|
||||||
t.Errorf("Expected (20, 'text/gemini', ''), got (%d, '%s', '%s')", code, mimeType, lang)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetMimeTypeAndLang11(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
code, mimeType, lang := getMimeTypeAndLang("20 text/gemini\n")
|
|
||||||
if code != 20 || mimeType != "text/gemini" || lang != "" {
|
|
||||||
t.Errorf("Expected (20, 'text/gemini', ''), got (%d, '%s', '%s')", code, mimeType, lang)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetMimeTypeAndLang12(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
code, mimeType, lang := getMimeTypeAndLang("20 text/plain; charset=utf-8")
|
|
||||||
if code != 20 || mimeType != "text/plain" || lang != "utf-8" {
|
|
||||||
t.Errorf("Expected (20, 'text/plain', ''), got (%d, '%s', '%s')", code, mimeType, lang)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetMimeTypeAndLang13(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
code, mimeType, lang := getMimeTypeAndLang("20 text/gemini; charset=utf-8")
|
|
||||||
if code != 20 || mimeType != "text/gemini" || lang != "utf-8" {
|
|
||||||
t.Errorf("Expected (20, 'text/plain', ''), got (%d, '%s', '%s')", code, mimeType, lang)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetTypeAndLang2(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
code, mimeType, lang := getMimeTypeAndLang("20 text/gemini charset=en")
|
|
||||||
if code != 20 || mimeType != "text/gemini" || lang != "en" {
|
|
||||||
t.Errorf("Expected (20, 'text/gemini', 'en'), got (%d, '%s', '%s')", code, mimeType, lang)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetTypeAndLang21(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
code, mimeType, lang := getMimeTypeAndLang("20 text/gemini lang=en")
|
|
||||||
if code != 20 || mimeType != "text/gemini" || lang != "en" {
|
|
||||||
t.Errorf("Expected (20, 'text/gemini', 'en'), got (%d, '%s', '%s')", code, mimeType, lang)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetMimeTypeAndLang3(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
code, mimeType, lang := getMimeTypeAndLang("31 gemini://redirect.to/page")
|
|
||||||
if code != 31 || mimeType != "" || lang != "" {
|
|
||||||
t.Errorf("Expected (20, '', ''), got (%d, '%s', '%s')", code, mimeType, lang)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetMimeTypeAndLang4(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
code, mimeType, lang := getMimeTypeAndLang("aaafdasdasd")
|
|
||||||
if code != 0 || mimeType != "" || lang != "" {
|
|
||||||
t.Errorf("Expected (0, '', ''), got (%d, '%s', '%s')", code, mimeType, lang)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetMimeTypeAndLang5(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
code, mimeType, lang := getMimeTypeAndLang("")
|
|
||||||
if code != 0 || mimeType != "" || lang != "" {
|
|
||||||
t.Errorf("Expected (0, '', ''), got (%d, '%s', '%s')", code, mimeType, lang)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,142 +0,0 @@
|
|||||||
package gemini
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"gemini-grc/config"
|
|
||||||
"os"
|
|
||||||
|
|
||||||
"gemini-grc/logging"
|
|
||||||
|
|
||||||
_ "github.com/jackc/pgx/v5/stdlib" // PGX driver for PostgreSQL
|
|
||||||
"github.com/jmoiron/sqlx"
|
|
||||||
)
|
|
||||||
|
|
||||||
func ConnectToDB() *sqlx.DB {
|
|
||||||
connStr := fmt.Sprintf("postgres://%s:%s@%s:%s/%s",
|
|
||||||
os.Getenv("PG_USER"),
|
|
||||||
os.Getenv("PG_PASSWORD"),
|
|
||||||
os.Getenv("PG_HOST"),
|
|
||||||
os.Getenv("PG_PORT"),
|
|
||||||
os.Getenv("PG_DATABASE"),
|
|
||||||
)
|
|
||||||
|
|
||||||
// Create a connection pool
|
|
||||||
db, err := sqlx.Open("pgx", connStr)
|
|
||||||
if err != nil {
|
|
||||||
panic(fmt.Sprintf("Unable to connect to database with URL %s: %v\n", connStr, err))
|
|
||||||
}
|
|
||||||
db.SetMaxOpenConns(20)
|
|
||||||
err = db.Ping()
|
|
||||||
if err != nil {
|
|
||||||
panic(fmt.Sprintf("Unable to ping database: %v\n", err))
|
|
||||||
}
|
|
||||||
|
|
||||||
logging.LogDebug("Connected to database")
|
|
||||||
return db
|
|
||||||
}
|
|
||||||
|
|
||||||
func SaveSnapshotIfNew(tx *sqlx.Tx, s *Snapshot) error {
|
|
||||||
marshalled, err := json.MarshalIndent(s, "", " ")
|
|
||||||
if err != nil {
|
|
||||||
panic(fmt.Sprintf("JSON serialization error for %v", s))
|
|
||||||
}
|
|
||||||
if config.CONFIG.DryRun {
|
|
||||||
logging.LogDebug("Would insert (if new) snapshot %s", marshalled)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
query := `
|
|
||||||
INSERT INTO snapshots (url, host, timestamp, mimetype, data, gemtext, links, lang, response_code, error)
|
|
||||||
VALUES (:url, :host, :timestamp, :mimetype, :data, :gemtext, :links, :lang, :response_code, :error)
|
|
||||||
ON CONFLICT (url) DO NOTHING
|
|
||||||
`
|
|
||||||
_, err = tx.NamedExec(query, s)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("[%s] GeminiError inserting snapshot: %w", s.URL, err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func UpsertSnapshot(id int, tx *sqlx.Tx, s *Snapshot) error {
|
|
||||||
marshalled, err := json.MarshalIndent(s, "", " ")
|
|
||||||
if err != nil {
|
|
||||||
panic(fmt.Sprintf("JSON serialization error for %v", s))
|
|
||||||
}
|
|
||||||
if config.CONFIG.DryRun {
|
|
||||||
logging.LogDebug("[%d] Would upsert snapshot %s", id, marshalled)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
query := `
|
|
||||||
INSERT INTO snapshots (url, host, timestamp, mimetype, data, gemtext, links, lang, response_code, error)
|
|
||||||
VALUES (:url, :host, :timestamp, :mimetype, :data, :gemtext, :links, :lang, :response_code, :error)
|
|
||||||
ON CONFLICT (url) DO UPDATE SET
|
|
||||||
url = EXCLUDED.url,
|
|
||||||
host = EXCLUDED.host,
|
|
||||||
timestamp = EXCLUDED.timestamp,
|
|
||||||
mimetype = EXCLUDED.mimetype,
|
|
||||||
data = EXCLUDED.data,
|
|
||||||
gemtext = EXCLUDED.gemtext,
|
|
||||||
links = EXCLUDED.links,
|
|
||||||
lang = EXCLUDED.lang,
|
|
||||||
response_code = EXCLUDED.response_code,
|
|
||||||
error = EXCLUDED.error`
|
|
||||||
_, err = tx.NamedExec(query, s)
|
|
||||||
//if err != nil {
|
|
||||||
// logging.LogError("[%s] GeminiError upserting snapshot: %w", s.URL, err)
|
|
||||||
// panic("This shouldn't happen")
|
|
||||||
//}
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("[%s] GeminiError upserting snapshot: %w", s.URL, err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func SaveLinksToDBinBatches(tx *sqlx.Tx, snapshots []*Snapshot) error {
|
|
||||||
if config.CONFIG.DryRun {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Approximately 5,957 rows maximum (65535/11 parameters), use 5000 to be safe
|
|
||||||
const batchSize = 5000
|
|
||||||
|
|
||||||
query := `
|
|
||||||
INSERT INTO snapshots (url, host, timestamp, mimetype, data, gemtext, links, lang, response_code, error)
|
|
||||||
VALUES (:url, :host, :timestamp, :mimetype, :data, :gemtext, :links, :lang, :response_code, :error)
|
|
||||||
ON CONFLICT (url) DO NOTHING
|
|
||||||
`
|
|
||||||
|
|
||||||
for i := 0; i < len(snapshots); i += batchSize {
|
|
||||||
end := i + batchSize
|
|
||||||
if end > len(snapshots) {
|
|
||||||
end = len(snapshots)
|
|
||||||
}
|
|
||||||
|
|
||||||
batch := snapshots[i:end]
|
|
||||||
|
|
||||||
_, err := tx.NamedExec(query, batch)
|
|
||||||
if err != nil {
|
|
||||||
logging.LogError("GeminiError batch inserting snapshots: %w", err)
|
|
||||||
return fmt.Errorf("DB error: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func SaveLinksToDB(tx *sqlx.Tx, snapshots []*Snapshot) error {
|
|
||||||
if config.CONFIG.DryRun {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
query := `
|
|
||||||
INSERT INTO snapshots (url, host, timestamp, mimetype, data, gemtext, links, lang, response_code, error)
|
|
||||||
VALUES (:url, :host, :timestamp, :mimetype, :data, :gemtext, :links, :lang, :response_code, :error)
|
|
||||||
ON CONFLICT (url) DO NOTHING
|
|
||||||
`
|
|
||||||
_, err := tx.NamedExec(query, snapshots)
|
|
||||||
if err != nil {
|
|
||||||
logging.LogError("GeminiError batch inserting snapshots: %w", err)
|
|
||||||
return fmt.Errorf("DB error: %w", err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,59 +0,0 @@
|
|||||||
package gemini
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"unicode/utf8"
|
|
||||||
|
|
||||||
"golang.org/x/text/encoding/charmap"
|
|
||||||
"golang.org/x/text/encoding/japanese"
|
|
||||||
"golang.org/x/text/encoding/korean"
|
|
||||||
"golang.org/x/text/transform"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
ErrInputTooLarge = errors.New("input too large")
|
|
||||||
ErrUTF8Conversion = errors.New("UTF-8 conversion error")
|
|
||||||
)
|
|
||||||
|
|
||||||
func BytesToValidUTF8(input []byte) (string, error) {
|
|
||||||
if len(input) == 0 {
|
|
||||||
return "", nil
|
|
||||||
}
|
|
||||||
const maxSize = 10 * 1024 * 1024 // 10MB
|
|
||||||
if len(input) > maxSize {
|
|
||||||
return "", fmt.Errorf("%w: %d bytes (max %d)", ErrInputTooLarge, len(input), maxSize)
|
|
||||||
}
|
|
||||||
// Remove NULL byte 0x00 (ReplaceAll accepts slices)
|
|
||||||
inputNoNull := bytes.ReplaceAll(input, []byte{byte(0)}, []byte{})
|
|
||||||
if utf8.Valid(inputNoNull) {
|
|
||||||
return string(inputNoNull), nil
|
|
||||||
}
|
|
||||||
encodings := []transform.Transformer{
|
|
||||||
charmap.ISO8859_1.NewDecoder(),
|
|
||||||
charmap.ISO8859_7.NewDecoder(),
|
|
||||||
charmap.Windows1250.NewDecoder(), // Central European
|
|
||||||
charmap.Windows1251.NewDecoder(), // Cyrillic
|
|
||||||
charmap.Windows1252.NewDecoder(),
|
|
||||||
charmap.Windows1256.NewDecoder(), // Arabic
|
|
||||||
japanese.EUCJP.NewDecoder(), // Japanese
|
|
||||||
korean.EUCKR.NewDecoder(), // Korean
|
|
||||||
}
|
|
||||||
// First successful conversion wins.
|
|
||||||
var lastErr error
|
|
||||||
for _, encoding := range encodings {
|
|
||||||
reader := transform.NewReader(bytes.NewReader(inputNoNull), encoding)
|
|
||||||
result, err := io.ReadAll(reader)
|
|
||||||
if err != nil {
|
|
||||||
lastErr = err
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if utf8.Valid(result) {
|
|
||||||
return string(result), nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return "", fmt.Errorf("%w (tried %d encodings): %w", ErrUTF8Conversion, len(encodings), lastErr)
|
|
||||||
}
|
|
||||||
@@ -1,14 +0,0 @@
|
|||||||
package gemini
|
|
||||||
|
|
||||||
import "testing"
|
|
||||||
|
|
||||||
// Make sure NULL bytes are removed
|
|
||||||
func TestEnsureValidUTF8(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
// Create a string with a null byte
|
|
||||||
strWithNull := "Hello" + string('\x00') + "world"
|
|
||||||
result, _ := BytesToValidUTF8([]byte(strWithNull))
|
|
||||||
if result != "Helloworld" {
|
|
||||||
t.Errorf("Expected string without NULL byte, got %s", result)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,82 +0,0 @@
|
|||||||
package gemini
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
|
|
||||||
"gemini-grc/logging"
|
|
||||||
)
|
|
||||||
|
|
||||||
// RobotsCache is a map of blocked URLs
|
|
||||||
// key: URL
|
|
||||||
// value: []string list of disallowed URLs
|
|
||||||
// If a key has no blocked URLs, an empty
|
|
||||||
// list is stored for caching.
|
|
||||||
var RobotsCache sync.Map //nolint:gochecknoglobals
|
|
||||||
|
|
||||||
func populateBlacklist(key string) (entries []string) {
|
|
||||||
// We either store an empty list when
|
|
||||||
// no rules, or a list of disallowed URLs.
|
|
||||||
// This applies even if we have an error
|
|
||||||
// finding/downloading robots.txt
|
|
||||||
defer func() {
|
|
||||||
RobotsCache.Store(key, entries)
|
|
||||||
}()
|
|
||||||
url := fmt.Sprintf("gemini://%s/robots.txt", key)
|
|
||||||
robotsContent, err := ConnectAndGetData(url)
|
|
||||||
if err != nil {
|
|
||||||
logging.LogDebug("robots.txt error %s", err)
|
|
||||||
return []string{}
|
|
||||||
}
|
|
||||||
robotsData, err := processData(robotsContent)
|
|
||||||
if err != nil {
|
|
||||||
logging.LogDebug("robots.txt error %s", err)
|
|
||||||
return []string{}
|
|
||||||
}
|
|
||||||
if robotsData.ResponseCode != 20 {
|
|
||||||
logging.LogDebug("robots.txt error code %d, ignoring", robotsData.ResponseCode)
|
|
||||||
return []string{}
|
|
||||||
}
|
|
||||||
// Some return text/plain, others text/gemini.
|
|
||||||
// According to spec, the first is correct,
|
|
||||||
// however let's be lenient
|
|
||||||
var data string
|
|
||||||
switch {
|
|
||||||
case robotsData.MimeType == "text/plain":
|
|
||||||
data = string(robotsData.Data)
|
|
||||||
case robotsData.MimeType == "text/gemini":
|
|
||||||
data = robotsData.GemText
|
|
||||||
default:
|
|
||||||
return []string{}
|
|
||||||
}
|
|
||||||
entries = ParseRobotsTxt(data, key)
|
|
||||||
return entries
|
|
||||||
}
|
|
||||||
|
|
||||||
// RobotMatch checks if the snapshot URL matches
|
|
||||||
// a robots.txt allow rule.
|
|
||||||
func RobotMatch(url URL) bool {
|
|
||||||
key := strings.ToLower(fmt.Sprintf("%s:%d", url.Hostname, url.Port))
|
|
||||||
logging.LogDebug("Checking robots.txt cache for %s", key)
|
|
||||||
var disallowedURLs []string
|
|
||||||
cacheEntries, ok := RobotsCache.Load(key)
|
|
||||||
if !ok {
|
|
||||||
// First time check, populate robot cache
|
|
||||||
disallowedURLs = populateBlacklist(key)
|
|
||||||
logging.LogDebug("Added to robots.txt cache: %v => %v", key, disallowedURLs)
|
|
||||||
} else {
|
|
||||||
disallowedURLs, _ = cacheEntries.([]string)
|
|
||||||
}
|
|
||||||
return isURLblocked(disallowedURLs, url.Full)
|
|
||||||
}
|
|
||||||
|
|
||||||
func isURLblocked(disallowedURLs []string, input string) bool {
|
|
||||||
for _, url := range disallowedURLs {
|
|
||||||
if strings.HasPrefix(strings.ToLower(input), url) {
|
|
||||||
logging.LogDebug("robots.txt match: %s matches %s", input, url)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
@@ -1,31 +0,0 @@
|
|||||||
package gemini
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ParseRobotsTxt takes robots.txt content and a host, and
|
|
||||||
// returns a list of full URLs that shouldn't
|
|
||||||
// be visited.
|
|
||||||
// TODO Also take into account the user agent?
|
|
||||||
// Check gemini://geminiprotocol.net/docs/companion/robots.gmi
|
|
||||||
func ParseRobotsTxt(content string, host string) []string {
|
|
||||||
var disallowedPaths []string
|
|
||||||
for _, line := range strings.Split(content, "\n") {
|
|
||||||
line = strings.TrimSpace(line)
|
|
||||||
line = strings.ToLower(line)
|
|
||||||
if strings.HasPrefix(line, "disallow:") {
|
|
||||||
parts := strings.SplitN(line, ":", 2)
|
|
||||||
if len(parts) == 2 {
|
|
||||||
path := strings.TrimSpace(parts[1])
|
|
||||||
if path != "" {
|
|
||||||
// Construct full Gemini URL
|
|
||||||
disallowedPaths = append(disallowedPaths,
|
|
||||||
fmt.Sprintf("gemini://%s%s", host, path))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return disallowedPaths
|
|
||||||
}
|
|
||||||
@@ -1,55 +0,0 @@
|
|||||||
package gemini
|
|
||||||
|
|
||||||
import (
|
|
||||||
"reflect"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestParseRobotsTxt(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
input := `User-agent: *
|
|
||||||
Disallow: /cgi-bin/wp.cgi/view
|
|
||||||
Disallow: /cgi-bin/wp.cgi/media
|
|
||||||
User-agent: googlebot
|
|
||||||
Disallow: /admin/`
|
|
||||||
|
|
||||||
expected := []string{
|
|
||||||
"gemini://example.com/cgi-bin/wp.cgi/view",
|
|
||||||
"gemini://example.com/cgi-bin/wp.cgi/media",
|
|
||||||
"gemini://example.com/admin/",
|
|
||||||
}
|
|
||||||
|
|
||||||
result := ParseRobotsTxt(input, "example.com")
|
|
||||||
|
|
||||||
if !reflect.DeepEqual(result, expected) {
|
|
||||||
t.Errorf("ParseRobotsTxt() = %v, want %v", result, expected)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseRobotsTxtEmpty(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
input := ``
|
|
||||||
|
|
||||||
result := ParseRobotsTxt(input, "example.com")
|
|
||||||
|
|
||||||
if len(result) != 0 {
|
|
||||||
t.Errorf("ParseRobotsTxt() = %v, want empty []string", result)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIsURLblocked(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
disallowedURLs := []string{
|
|
||||||
"gemini://example.com/cgi-bin/wp.cgi/view",
|
|
||||||
"gemini://example.com/cgi-bin/wp.cgi/media",
|
|
||||||
"gemini://example.com/admin/",
|
|
||||||
}
|
|
||||||
url := "gemini://example.com/admin/index.html"
|
|
||||||
if !isURLblocked(disallowedURLs, url) {
|
|
||||||
t.Errorf("Expected %s to be blocked", url)
|
|
||||||
}
|
|
||||||
url = "gemini://example1.com/admin/index.html"
|
|
||||||
if isURLblocked(disallowedURLs, url) {
|
|
||||||
t.Errorf("expected %s to not be blocked", url)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,43 +0,0 @@
|
|||||||
package gemini
|
|
||||||
|
|
||||||
import (
|
|
||||||
"database/sql/driver"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/guregu/null/v5"
|
|
||||||
)
|
|
||||||
|
|
||||||
type LinkList []URL
|
|
||||||
|
|
||||||
func (l *LinkList) Value() (driver.Value, error) {
|
|
||||||
return json.Marshal(l)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l *LinkList) Scan(value interface{}) error {
|
|
||||||
if value == nil {
|
|
||||||
*l = nil
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
b, ok := value.([]byte) // Type assertion! Converts to []byte
|
|
||||||
if !ok {
|
|
||||||
return fmt.Errorf("failed to scan LinkList: expected []byte, got %T", value)
|
|
||||||
}
|
|
||||||
return json.Unmarshal(b, l)
|
|
||||||
}
|
|
||||||
|
|
||||||
type Snapshot struct {
|
|
||||||
ID int `db:"id" json:"id,omitempty"`
|
|
||||||
//UID string `db:"uid" json:"uid,omitempty"`
|
|
||||||
URL URL `db:"url" json:"url,omitempty"`
|
|
||||||
Host string `db:"host" json:"host,omitempty"`
|
|
||||||
Timestamp null.Time `db:"timestamp" json:"timestamp,omitempty"`
|
|
||||||
MimeType null.String `db:"mimetype" json:"mimetype,omitempty"`
|
|
||||||
Data null.Value[[]byte] `db:"data" json:"data,omitempty"` // For non text/gemini files.
|
|
||||||
GemText null.String `db:"gemtext" json:"gemtext,omitempty"` // For text/gemini files.
|
|
||||||
Header null.String `db:"header" json:"header,omitempty"` // Response header.
|
|
||||||
Links null.Value[LinkList] `db:"links" json:"links,omitempty"`
|
|
||||||
Lang null.String `db:"lang" json:"lang,omitempty"`
|
|
||||||
ResponseCode null.Int `db:"response_code" json:"code,omitempty"` // Gemini response status code.
|
|
||||||
Error null.String `db:"error" json:"error,omitempty"` // On network errors only
|
|
||||||
}
|
|
||||||
@@ -1,305 +0,0 @@
|
|||||||
package gemini
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gemini-grc/config"
|
|
||||||
"gemini-grc/logging"
|
|
||||||
"gemini-grc/util"
|
|
||||||
|
|
||||||
"github.com/guregu/null/v5"
|
|
||||||
"github.com/jmoiron/sqlx"
|
|
||||||
)
|
|
||||||
|
|
||||||
func SpawnWorkers(numOfWorkers int, db *sqlx.DB) {
|
|
||||||
logging.LogInfo("Spawning %d workers", numOfWorkers)
|
|
||||||
for i := range numOfWorkers {
|
|
||||||
go func(i int) {
|
|
||||||
for {
|
|
||||||
RunWorker(i, db, nil)
|
|
||||||
}
|
|
||||||
}(i)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func RunWorker(id int, db *sqlx.DB, url *string) {
|
|
||||||
// Each worker runs within a DB transaction.
|
|
||||||
tx, err := db.Beginx()
|
|
||||||
if err != nil {
|
|
||||||
logging.LogError("Failed to begin transaction: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Commit/rollback at the end
|
|
||||||
defer func() {
|
|
||||||
err = tx.Commit()
|
|
||||||
if err != nil {
|
|
||||||
logging.LogError("[%d] Failed to commit transaction: %w", id, err)
|
|
||||||
err := tx.Rollback()
|
|
||||||
if err != nil {
|
|
||||||
panic(fmt.Sprintf("[%d] Failed to roll back transaction: %v", id, err))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
var snapshots []Snapshot
|
|
||||||
|
|
||||||
// If not given a specific URL,
|
|
||||||
// get some random ones to visit from DB.
|
|
||||||
if url == nil {
|
|
||||||
snapshots, err = GetRandomSnapshotsDistinctHosts(tx)
|
|
||||||
if err != nil {
|
|
||||||
logging.LogError("[%d] GeminiError retrieving snapshot: %w", id, err)
|
|
||||||
panic("This should never happen")
|
|
||||||
} else if len(snapshots) == 0 {
|
|
||||||
logging.LogInfo("[%d] No snapshots to visit.", id)
|
|
||||||
time.Sleep(1 * time.Minute)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
snapshotURL, err := ParseURL(*url, "")
|
|
||||||
if err != nil {
|
|
||||||
logging.LogError("Invalid URL given: " + *url)
|
|
||||||
return
|
|
||||||
|
|
||||||
}
|
|
||||||
snapshots = []Snapshot{{
|
|
||||||
//UID: uid.UID(),
|
|
||||||
URL: *snapshotURL,
|
|
||||||
Host: snapshotURL.Hostname,
|
|
||||||
Timestamp: null.TimeFrom(time.Now()),
|
|
||||||
}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Start visiting URLs.
|
|
||||||
total := len(snapshots)
|
|
||||||
for i, s := range snapshots {
|
|
||||||
logging.LogInfo("[%d] Starting %d/%d %s", id, i+1, total, s.URL.String())
|
|
||||||
// We differentiate between errors:
|
|
||||||
// Unexpected errors are the ones returned from the following function.
|
|
||||||
// If an error is unexpected (which should never happen) we panic.
|
|
||||||
// Expected errors are stored as strings within the snapshot,
|
|
||||||
// so that they can also be stored in DB.
|
|
||||||
err = workOnSnapshot(id, tx, &s)
|
|
||||||
if err != nil {
|
|
||||||
logging.LogError("[%d] [%s] Unexpected GeminiError %w", id, s.URL.String(), err)
|
|
||||||
util.PrintStackAndPanic(err)
|
|
||||||
}
|
|
||||||
if s.Error.Valid {
|
|
||||||
logging.LogWarn("[%d] Error: %v", id, s.Error.String)
|
|
||||||
}
|
|
||||||
logging.LogDebug("[%d] Done %d/%d.", id, i+1, total)
|
|
||||||
}
|
|
||||||
logging.LogInfo("[%d] Worker done.", id)
|
|
||||||
}
|
|
||||||
|
|
||||||
// workOnSnapshot visits a URL and stores the result.
|
|
||||||
// errors should be returned only if they are unexpected.
|
|
||||||
func workOnSnapshot(id int, tx *sqlx.Tx, s *Snapshot) (err error) {
|
|
||||||
if IsBlacklisted(s.URL) {
|
|
||||||
logging.LogInfo("[%d] URL matches Blacklist, ignoring %s", id, s.URL.String())
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// If URL matches a robots.txt disallow line,
|
|
||||||
// add it as an error so next time it won't be
|
|
||||||
// crawled.
|
|
||||||
if RobotMatch(s.URL) {
|
|
||||||
s.Error = null.StringFrom(ErrGeminiRobotsDisallowed.Error())
|
|
||||||
err = UpsertSnapshot(id, tx, s)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("[%d] %w", id, err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Resolve IP address via DNS
|
|
||||||
IPs, err := getHostIPAddresses(s.Host)
|
|
||||||
if err != nil {
|
|
||||||
s.Error = null.StringFrom(err.Error())
|
|
||||||
err = UpsertSnapshot(id, tx, s)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("[%d] %w", id, err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// If the host's ip is in the connections pool we stop
|
|
||||||
IpPool.Lock.RLock()
|
|
||||||
logging.LogDebug("[%d] [%s] Checking pool for IP", id, s.URL.String())
|
|
||||||
for _, ip := range IPs {
|
|
||||||
_, ok := IpPool.IPs[ip]
|
|
||||||
if ok {
|
|
||||||
logging.LogDebug("[%d] Another worker is visiting this host: %s", id, s.URL.String())
|
|
||||||
IpPool.Lock.RUnlock()
|
|
||||||
time.Sleep(1 * time.Second) // Avoid flood-retrying
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
IpPool.Lock.RUnlock()
|
|
||||||
|
|
||||||
AddIPsToPool(IPs)
|
|
||||||
|
|
||||||
// After finishing, remove the host IPs from
|
|
||||||
// the connections pool, with a small delay
|
|
||||||
// to avoid potentially hitting the same IP quickly.
|
|
||||||
defer func() {
|
|
||||||
time.Sleep(5 * time.Second)
|
|
||||||
RemoveIPsFromPool(IPs)
|
|
||||||
}()
|
|
||||||
|
|
||||||
url := s.URL.String()
|
|
||||||
logging.LogDebug("[%d] Dialing %s", id, url)
|
|
||||||
|
|
||||||
err = Visit(s)
|
|
||||||
if err != nil {
|
|
||||||
if !IsKnownError(err) {
|
|
||||||
logging.LogError("[%d] Unknown error visiting %s: %w", id, url, err)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// Check if error is redirection, and handle it
|
|
||||||
s.Error = null.StringFrom(err.Error())
|
|
||||||
if errors.As(err, new(*GeminiError)) &&
|
|
||||||
err.(*GeminiError).Msg == "redirect" {
|
|
||||||
err = handleRedirection(id, tx, s)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
logging.LogInfo("[%d] Done, response code %d.", id, s.ResponseCode.ValueOrZero())
|
|
||||||
|
|
||||||
// If this is a gemini page, parse possible links inside
|
|
||||||
if !s.Error.Valid && s.MimeType.Valid && s.MimeType.String == "text/gemini" {
|
|
||||||
links := GetPageLinks(s.URL, s.GemText.String)
|
|
||||||
logging.LogDebug("[%d] Found %d links", id, len(links))
|
|
||||||
if len(links) > 0 {
|
|
||||||
s.Links = null.ValueFrom(links)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
logging.LogDebug("[%d] Not looking for page links", id)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = UpsertSnapshot(id, tx, s)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = storeLinks(tx, s)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func storeLinks(tx *sqlx.Tx, s *Snapshot) error {
|
|
||||||
if s.Links.Valid {
|
|
||||||
var batchSnapshots []*Snapshot
|
|
||||||
for _, link := range s.Links.ValueOrZero() {
|
|
||||||
if shouldPersistURL(link) {
|
|
||||||
newSnapshot := &Snapshot{
|
|
||||||
//UID: uid.UID(),
|
|
||||||
URL: link,
|
|
||||||
Host: link.Hostname,
|
|
||||||
Timestamp: null.TimeFrom(time.Now()),
|
|
||||||
}
|
|
||||||
batchSnapshots = append(batchSnapshots, newSnapshot)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(batchSnapshots) > 0 {
|
|
||||||
err := SaveLinksToDBinBatches(tx, batchSnapshots)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// shouldPersistURL returns true if we
|
|
||||||
// should save the URL in the DB.
|
|
||||||
// Only gemini:// urls are saved.
|
|
||||||
func shouldPersistURL(u URL) bool {
|
|
||||||
return strings.HasPrefix(u.String(), "gemini://")
|
|
||||||
}
|
|
||||||
|
|
||||||
func handleRedirection(id int, tx *sqlx.Tx, s *Snapshot) error {
|
|
||||||
newURL, err := extractRedirectTarget(s.URL, s.Error.ValueOrZero())
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
logging.LogDebug("[%d] Page redirects to %s", id, newURL)
|
|
||||||
// Insert fresh snapshot with new URL
|
|
||||||
snapshot := &Snapshot{
|
|
||||||
//UID: uid.UID(),
|
|
||||||
URL: *newURL,
|
|
||||||
Host: newURL.Hostname,
|
|
||||||
Timestamp: null.TimeFrom(time.Now()),
|
|
||||||
}
|
|
||||||
logging.LogDebug("[%d] Saving empty snapshot for %s", id, snapshot.URL.String())
|
|
||||||
err = SaveSnapshotIfNew(tx, snapshot)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func GetRandomSnapshotsDistinctHosts(tx *sqlx.Tx) ([]Snapshot, error) {
|
|
||||||
// Old, unoptimized query
|
|
||||||
|
|
||||||
// query := `
|
|
||||||
// SELECT DISTINCT ON (host) *
|
|
||||||
// FROM snapshots
|
|
||||||
// WHERE response_code IS NULL
|
|
||||||
// AND error IS NULL
|
|
||||||
// ORDER BY host, RANDOM()
|
|
||||||
// LIMIT $1
|
|
||||||
// `
|
|
||||||
query := `
|
|
||||||
SELECT *
|
|
||||||
FROM snapshots
|
|
||||||
WHERE response_code IS NULL
|
|
||||||
AND error IS NULL
|
|
||||||
ORDER BY RANDOM()
|
|
||||||
LIMIT $1
|
|
||||||
`
|
|
||||||
//query := `
|
|
||||||
// WITH RankedSnapshots AS (
|
|
||||||
// SELECT id, url, host, timestamp, mimetype, data, gemtext,
|
|
||||||
// links, lang, response_code, error,
|
|
||||||
// ROW_NUMBER() OVER (PARTITION BY host ORDER BY RANDOM()) as rn
|
|
||||||
// FROM snapshots
|
|
||||||
// WHERE response_code IS NULL
|
|
||||||
// AND error IS NULL
|
|
||||||
// )
|
|
||||||
// SELECT id, url, host, timestamp, mimetype, data, gemtext,
|
|
||||||
// links, lang, response_code, error
|
|
||||||
// FROM RankedSnapshots
|
|
||||||
// WHERE rn = 1
|
|
||||||
// LIMIT $1
|
|
||||||
//`
|
|
||||||
var snapshots []Snapshot
|
|
||||||
err := tx.Select(&snapshots, query, config.CONFIG.WorkerBatchSize)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return snapshots, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func GetSnapshotFromURL(tx *sqlx.Tx, url string) ([]Snapshot, error) {
|
|
||||||
query := `
|
|
||||||
SELECT *
|
|
||||||
FROM snapshots
|
|
||||||
WHERE url=$1
|
|
||||||
LIMIT 1
|
|
||||||
`
|
|
||||||
var snapshots []Snapshot
|
|
||||||
err := tx.Select(&snapshots, query, url)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return snapshots, nil
|
|
||||||
}
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
module gemini-grc
|
|
||||||
|
|
||||||
go 1.23.1
|
|
||||||
|
|
||||||
require (
|
|
||||||
github.com/guregu/null/v5 v5.0.0
|
|
||||||
github.com/jackc/pgx/v5 v5.7.1
|
|
||||||
github.com/jmoiron/sqlx v1.4.0
|
|
||||||
github.com/matoous/go-nanoid/v2 v2.1.0
|
|
||||||
github.com/rs/zerolog v1.33.0
|
|
||||||
github.com/stretchr/testify v1.9.0
|
|
||||||
golang.org/x/text v0.19.0
|
|
||||||
)
|
|
||||||
|
|
||||||
require (
|
|
||||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
|
||||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
|
||||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
|
||||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
|
||||||
github.com/kr/text v0.2.0 // indirect
|
|
||||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
|
||||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
|
||||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
|
||||||
github.com/rogpeppe/go-internal v1.13.1 // indirect
|
|
||||||
golang.org/x/crypto v0.27.0 // indirect
|
|
||||||
golang.org/x/sync v0.8.0 // indirect
|
|
||||||
golang.org/x/sys v0.25.0 // indirect
|
|
||||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
|
||||||
)
|
|
||||||
@@ -1,68 +0,0 @@
|
|||||||
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
|
|
||||||
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
|
||||||
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
|
|
||||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
|
||||||
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
|
|
||||||
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
|
|
||||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
|
||||||
github.com/guregu/null/v5 v5.0.0 h1:PRxjqyOekS11W+w/7Vfz6jgJE/BCwELWtgvOJzddimw=
|
|
||||||
github.com/guregu/null/v5 v5.0.0/go.mod h1:SjupzNy+sCPtwQTKWhUCqjhVCO69hpsl2QsZrWHjlwU=
|
|
||||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
|
||||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
|
||||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
|
||||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
|
||||||
github.com/jackc/pgx/v5 v5.7.1 h1:x7SYsPBYDkHDksogeSmZZ5xzThcTgRz++I5E+ePFUcs=
|
|
||||||
github.com/jackc/pgx/v5 v5.7.1/go.mod h1:e7O26IywZZ+naJtWWos6i6fvWK+29etgITqrqHLfoZA=
|
|
||||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
|
||||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
|
||||||
github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
|
|
||||||
github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
|
|
||||||
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
|
|
||||||
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
|
|
||||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
|
||||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
|
||||||
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
|
||||||
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
|
||||||
github.com/matoous/go-nanoid/v2 v2.1.0 h1:P64+dmq21hhWdtvZfEAofnvJULaRR1Yib0+PnU669bE=
|
|
||||||
github.com/matoous/go-nanoid/v2 v2.1.0/go.mod h1:KlbGNQ+FhrUNIHUxZdL63t7tl4LaPkZNpUULS8H4uVM=
|
|
||||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
|
||||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
|
||||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
|
||||||
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
|
||||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
|
||||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
|
||||||
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
|
|
||||||
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
|
||||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
|
||||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
|
||||||
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
|
|
||||||
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
|
|
||||||
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
|
||||||
github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
|
|
||||||
github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
|
|
||||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
|
||||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
|
||||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
|
||||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
|
||||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
|
||||||
golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
|
|
||||||
golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
|
|
||||||
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
|
|
||||||
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
|
||||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
|
||||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
|
||||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
|
||||||
golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
|
|
||||||
golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
|
||||||
golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
|
|
||||||
golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
|
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
|
||||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
|
||||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
|
||||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
|
||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
|
||||||
@@ -1,52 +0,0 @@
|
|||||||
package http
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"gemini-grc/logging"
|
|
||||||
_ "gemini-grc/logging"
|
|
||||||
"net/http"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func CreateServer(listenAddr string) *http.Server {
|
|
||||||
mux := http.NewServeMux()
|
|
||||||
mux.HandleFunc("GET /ping", wrapForError(getPing))
|
|
||||||
|
|
||||||
server := &http.Server{
|
|
||||||
Addr: listenAddr,
|
|
||||||
Handler: mux,
|
|
||||||
ReadHeaderTimeout: 10 * time.Second,
|
|
||||||
}
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
// Start the server. Blocking call.
|
|
||||||
logging.LogInfo("HTTP server listening on %s", listenAddr)
|
|
||||||
if err := server.ListenAndServe(); err != nil {
|
|
||||||
panic(fmt.Sprintf("Server failed to start: %s", err))
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
return server
|
|
||||||
}
|
|
||||||
|
|
||||||
func wrapForError(f func(http.ResponseWriter, *http.Request) error) http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
err := f(w, r)
|
|
||||||
if err != nil {
|
|
||||||
code := http.StatusInternalServerError
|
|
||||||
logging.LogWarn("Error while handling request: %d %s", code, err)
|
|
||||||
http.Error(w, http.StatusText(code), code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func getPing(w http.ResponseWriter, r *http.Request) error {
|
|
||||||
method := r.Method
|
|
||||||
url := r.URL.String()
|
|
||||||
path := r.URL.Path
|
|
||||||
response := fmt.Sprintf("Pong %s %s %s", method, url, path)
|
|
||||||
_, err := w.Write([]byte(response))
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to write response: %w", err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,23 +0,0 @@
|
|||||||
package logging
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
zlog "github.com/rs/zerolog/log"
|
|
||||||
)
|
|
||||||
|
|
||||||
func LogDebug(format string, args ...interface{}) {
|
|
||||||
zlog.Debug().Msg(fmt.Sprintf(format, args...))
|
|
||||||
}
|
|
||||||
|
|
||||||
func LogInfo(format string, args ...interface{}) {
|
|
||||||
zlog.Info().Msg(fmt.Sprintf(format, args...))
|
|
||||||
}
|
|
||||||
|
|
||||||
func LogWarn(format string, args ...interface{}) {
|
|
||||||
zlog.Warn().Msg(fmt.Sprintf(format, args...))
|
|
||||||
}
|
|
||||||
|
|
||||||
func LogError(format string, args ...interface{}) {
|
|
||||||
zlog.Error().Err(fmt.Errorf(format, args...)).Msg("")
|
|
||||||
}
|
|
||||||
@@ -1,63 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"gemini-grc/config"
|
|
||||||
"gemini-grc/gemini"
|
|
||||||
"gemini-grc/http"
|
|
||||||
"gemini-grc/logging"
|
|
||||||
"github.com/jmoiron/sqlx"
|
|
||||||
"github.com/rs/zerolog"
|
|
||||||
zlog "github.com/rs/zerolog/log"
|
|
||||||
"os"
|
|
||||||
"os/signal"
|
|
||||||
"syscall"
|
|
||||||
)
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
config.CONFIG = *config.GetConfig()
|
|
||||||
zerolog.TimeFieldFormat = zerolog.TimeFormatUnix
|
|
||||||
zerolog.SetGlobalLevel(config.CONFIG.LogLevel)
|
|
||||||
zlog.Logger = zlog.Output(zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: "[2006-01-02 15:04:05]"})
|
|
||||||
if err := runApp(); err != nil {
|
|
||||||
logging.LogError("Application error: %w", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func runApp() error {
|
|
||||||
logging.LogInfo("Starting up. Press Ctrl+C to exit")
|
|
||||||
signals := make(chan os.Signal, 1)
|
|
||||||
signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
|
|
||||||
|
|
||||||
server := http.CreateServer("localhost:8899")
|
|
||||||
db := gemini.ConnectToDB()
|
|
||||||
|
|
||||||
// !!! DANGER !!!
|
|
||||||
// Removes all rows and adds some seed URLs.
|
|
||||||
// populateDB(db)
|
|
||||||
|
|
||||||
defer func(db *sqlx.DB) {
|
|
||||||
err := db.Close()
|
|
||||||
if err != nil {
|
|
||||||
// TODO properly log & hangle error
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
}(db)
|
|
||||||
|
|
||||||
gemini.LoadBlacklist()
|
|
||||||
// If there's an argument, assume it's a URL
|
|
||||||
// to visit and ignore database state.
|
|
||||||
if len(os.Args) > 1 {
|
|
||||||
url := os.Args[1]
|
|
||||||
go gemini.RunWorker(0, db, &url)
|
|
||||||
} else {
|
|
||||||
go gemini.SpawnWorkers(config.CONFIG.NumOfWorkers, db)
|
|
||||||
}
|
|
||||||
|
|
||||||
<-signals
|
|
||||||
if err := server.Close(); err != nil {
|
|
||||||
logging.LogError("GeminiError during server shutdown: %s", err)
|
|
||||||
}
|
|
||||||
logging.LogInfo("Received SIGINT or SIGTERM signal, exiting")
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
use std::fs::File;
|
||||||
|
use std::io::{self, Read};
|
||||||
|
|
||||||
|
pub fn read_file_as_bytes(path: &str) -> io::Result<Vec<u8>> {
|
||||||
|
let mut file = File::open(path)?;
|
||||||
|
let mut buffer = Vec::new();
|
||||||
|
file.read_to_end(&mut buffer)?;
|
||||||
|
Ok(buffer)
|
||||||
|
}
|
||||||
@@ -0,0 +1,63 @@
|
|||||||
|
use std::fmt;
|
||||||
|
use std::fmt::Debug;
|
||||||
|
|
||||||
|
use anyhow::{anyhow, Context, Result};
|
||||||
|
use url::Url;
|
||||||
|
|
||||||
|
pub mod server;
|
||||||
|
pub mod tls;
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct GeminiUrl {
|
||||||
|
pub scheme: String,
|
||||||
|
pub hostname: String,
|
||||||
|
pub port: u16,
|
||||||
|
pub path: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl fmt::Display for GeminiUrl {
|
||||||
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
|
write!(
|
||||||
|
f,
|
||||||
|
"{}://{}:{}{}",
|
||||||
|
self.scheme, self.hostname, self.port, self.path
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl GeminiUrl {
|
||||||
|
/// Default scheme: "gemini://"
|
||||||
|
/// Port: 1965
|
||||||
|
/// Path: "/"
|
||||||
|
pub fn new(input: &str) -> Result<GeminiUrl> {
|
||||||
|
let input_with_scheme = if input.contains("://") {
|
||||||
|
input.to_string()
|
||||||
|
} else {
|
||||||
|
format!("gemini://{}", input)
|
||||||
|
};
|
||||||
|
let parsed = Url::parse(&input_with_scheme)
|
||||||
|
.with_context(|| format!("Invalid request URL {}", &input_with_scheme))?;
|
||||||
|
if parsed.scheme() != "gemini" {
|
||||||
|
return Err(anyhow!(
|
||||||
|
"Invalid request (URL protocol should be gemini://)"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let hostname = parsed
|
||||||
|
.host_str()
|
||||||
|
.ok_or_else(|| anyhow!("Invalid request (URL must have a host)"))?;
|
||||||
|
let mut parsed = GeminiUrl {
|
||||||
|
scheme: parsed.scheme().to_string(),
|
||||||
|
hostname: hostname.to_string(),
|
||||||
|
port: parsed.port().unwrap_or(1965),
|
||||||
|
path: parsed.path().to_string(),
|
||||||
|
};
|
||||||
|
if parsed.path.is_empty() {
|
||||||
|
parsed.path = String::from("/")
|
||||||
|
};
|
||||||
|
Ok(parsed)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn to_url(&self) -> Url {
|
||||||
|
Url::parse(&self.to_string()).unwrap()
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,233 @@
|
|||||||
|
use std::io::{Read, Write};
|
||||||
|
use std::net::{IpAddr, TcpListener, TcpStream};
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::thread;
|
||||||
|
|
||||||
|
use anyhow::{anyhow, Context, Result};
|
||||||
|
use log;
|
||||||
|
use log::{error, info};
|
||||||
|
use openssl::ssl::{SslAcceptor, SslStream};
|
||||||
|
use path_clean::PathClean;
|
||||||
|
|
||||||
|
use crate::fs::read_file_as_bytes;
|
||||||
|
use crate::gemini::{GeminiUrl, tls};
|
||||||
|
use crate::nanoid::nanoid;
|
||||||
|
use crate::time::now_unix_millis;
|
||||||
|
|
||||||
|
struct GeminiSession {
|
||||||
|
pub ip: IpAddr,
|
||||||
|
pub tls_id: Option<String>,
|
||||||
|
pub stream: SslStream<TcpStream>,
|
||||||
|
pub id: String,
|
||||||
|
pub timestamp: u128,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct GeminiResponse {
|
||||||
|
code: u8,
|
||||||
|
mime_type: Option<String>,
|
||||||
|
data: Option<Vec<u8>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl GeminiResponse {
|
||||||
|
pub fn new(
|
||||||
|
srv_root: &str,
|
||||||
|
gemini_session: &GeminiSession,
|
||||||
|
path: &String,
|
||||||
|
) -> Result<GeminiResponse> {
|
||||||
|
let filename = resolve_path(srv_root, path)?;
|
||||||
|
if !filename.is_file() {
|
||||||
|
return Ok(GeminiResponse {
|
||||||
|
code: 51, // NOT FOUND
|
||||||
|
mime_type: None,
|
||||||
|
data: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
let mime_type = Self::guess_mime_type(PathBuf::from(&filename));
|
||||||
|
if mime_type.is_none() {
|
||||||
|
log_info(
|
||||||
|
gemini_session,
|
||||||
|
&format!("Cannot guess mime type {:?}", filename),
|
||||||
|
);
|
||||||
|
return Ok(GeminiResponse {
|
||||||
|
code: 51,
|
||||||
|
mime_type: None,
|
||||||
|
data: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
log_info(gemini_session, &format!("Opening file {:?}", filename));
|
||||||
|
let file_str = filename
|
||||||
|
.to_str()
|
||||||
|
.ok_or_else(|| anyhow!("Failed to convert path to string"))?;
|
||||||
|
// let data = read_file_to_string(file_str)
|
||||||
|
// .with_context(|| format!("Failed to read file {}", path))?;
|
||||||
|
let data = read_file_as_bytes(file_str)
|
||||||
|
.with_context(|| format!("Failed to read file {}", path))?;
|
||||||
|
Ok(GeminiResponse {
|
||||||
|
code: 20,
|
||||||
|
mime_type,
|
||||||
|
data: Some(data),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn guess_mime_type(path: PathBuf) -> Option<String> {
|
||||||
|
match path.extension() {
|
||||||
|
Some(e) => match e.to_str().unwrap_or("") {
|
||||||
|
"gmi" => Some(String::from("text/gemini")),
|
||||||
|
"jpg" => Some(String::from("image/jpeg")),
|
||||||
|
"jpeg" => Some(String::from("image/jpeg")),
|
||||||
|
"png" => Some(String::from("image/png")),
|
||||||
|
_ => None,
|
||||||
|
},
|
||||||
|
None => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_bytes(&self) -> Vec<u8> {
|
||||||
|
let header = match &self.mime_type {
|
||||||
|
Some(e) => format!("{}\t{}\r\n", self.code, e),
|
||||||
|
None => format!("{}\r\n", self.code),
|
||||||
|
};
|
||||||
|
// Convert the header to bytes and prepare the response vector
|
||||||
|
let mut response_bytes = header.into_bytes();
|
||||||
|
|
||||||
|
// Extend the response with the data if it exists
|
||||||
|
if let Some(e) = &self.data {
|
||||||
|
response_bytes.extend(e);
|
||||||
|
}
|
||||||
|
response_bytes
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn start_server(
|
||||||
|
bind_address: &str,
|
||||||
|
key_file: &str,
|
||||||
|
cert_file: &str,
|
||||||
|
srv_root: &str,
|
||||||
|
) -> Result<()> {
|
||||||
|
let acceptor = tls::create_tls_acceptor(key_file, cert_file).context("TLS error")?;
|
||||||
|
let listener = TcpListener::bind(bind_address).context("TCP bind error")?;
|
||||||
|
info!("Gemini server listening to {}", &bind_address);
|
||||||
|
|
||||||
|
for stream in listener.incoming() {
|
||||||
|
// The stream will be dropped at the end of
|
||||||
|
// each loop iteration, which will close
|
||||||
|
// the stream automatically. No housekeeping
|
||||||
|
// necessary.
|
||||||
|
// Each stream is handled in a new thread.
|
||||||
|
// Async is under consideration for now.
|
||||||
|
match stream {
|
||||||
|
Err(e) => {
|
||||||
|
error!("Failed to accept connection: {:?}", e);
|
||||||
|
}
|
||||||
|
Ok(stream) => {
|
||||||
|
let acceptor = acceptor.clone();
|
||||||
|
let srv_root = srv_root.to_string();
|
||||||
|
|
||||||
|
let handle = thread::spawn(move || {
|
||||||
|
if let Err(e) = initiate_connection(&acceptor, stream, &srv_root) {
|
||||||
|
error!("Connection handling error: {:?}", e);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
if let Err(e) = handle.join() {
|
||||||
|
error!("Thread join error while handling stream: {:?}", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn log_info(session: &GeminiSession, message: &str) {
|
||||||
|
info!("{} {} {}", session.id, session.ip.to_string(), message);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resolve final absolute path
|
||||||
|
/// of the file to serve.
|
||||||
|
/// Final path must always begin with
|
||||||
|
/// root path (no escape outside root!).
|
||||||
|
/// If final path is a directory, append
|
||||||
|
/// index.gmi.
|
||||||
|
fn resolve_path(root_path: &str, input: &String) -> Result<PathBuf> {
|
||||||
|
let path1 = PathBuf::from(root_path);
|
||||||
|
let mut path2 = PathBuf::from(input);
|
||||||
|
if path2.is_absolute() {
|
||||||
|
path2 = PathBuf::from(path2.strip_prefix("/")?);
|
||||||
|
}
|
||||||
|
let mut final_path = path1.join(path2).clean();
|
||||||
|
if !final_path.starts_with(path1) {
|
||||||
|
return Err(anyhow!("Invalid path {:?} -> {:?}", input, final_path));
|
||||||
|
}
|
||||||
|
if final_path.is_dir() {
|
||||||
|
final_path = final_path.join("index.gmi");
|
||||||
|
}
|
||||||
|
Ok(final_path)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn handle_gemini_session(srv_root: &str, gemini_session: &mut GeminiSession) -> Result<()> {
|
||||||
|
let mut buffer = [0; 1024];
|
||||||
|
let bytes_read = gemini_session
|
||||||
|
.stream
|
||||||
|
.read(&mut buffer)
|
||||||
|
.context("Invalid request (failed to read input stream)")?;
|
||||||
|
let received = String::from_utf8(Vec::from(&buffer[..bytes_read]))
|
||||||
|
.context("Invalid request (failed to convert input to UTF8)")?;
|
||||||
|
let request_data = received.trim();
|
||||||
|
let url = GeminiUrl::new(request_data)?;
|
||||||
|
log_info(
|
||||||
|
gemini_session,
|
||||||
|
&format!(
|
||||||
|
"New request from {}{} {}",
|
||||||
|
gemini_session.ip,
|
||||||
|
match &gemini_session.tls_id {
|
||||||
|
Some(x) => format!(" TLS digest {}", x),
|
||||||
|
None => String::from(""),
|
||||||
|
},
|
||||||
|
request_data
|
||||||
|
),
|
||||||
|
);
|
||||||
|
let response = GeminiResponse::new(srv_root, gemini_session, &url.path)?;
|
||||||
|
let response_raw = response.get_bytes();
|
||||||
|
log_info(
|
||||||
|
gemini_session,
|
||||||
|
&format!(
|
||||||
|
"Reply Code {} Response length {} bytes",
|
||||||
|
response.code,
|
||||||
|
response_raw.len()
|
||||||
|
),
|
||||||
|
);
|
||||||
|
gemini_session
|
||||||
|
.stream
|
||||||
|
.write_all(&response_raw)
|
||||||
|
.context("Failed to write response to stream")?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn initiate_connection(acceptor: &SslAcceptor, stream: TcpStream, srv_root: &str) -> Result<()> {
|
||||||
|
let timestamp1 = now_unix_millis();
|
||||||
|
|
||||||
|
let ip = stream
|
||||||
|
.peer_addr()
|
||||||
|
.context("Failed to get peer IP address from TCP stream")?
|
||||||
|
.ip();
|
||||||
|
|
||||||
|
let stream = acceptor
|
||||||
|
.accept(stream)
|
||||||
|
.with_context(|| format!("{} Failed to establish SSL connection", ip))?;
|
||||||
|
|
||||||
|
let mut gemini_session = GeminiSession {
|
||||||
|
ip,
|
||||||
|
tls_id: tls::get_peer_certificate_digest(&stream),
|
||||||
|
stream,
|
||||||
|
id: nanoid(),
|
||||||
|
timestamp: now_unix_millis(),
|
||||||
|
};
|
||||||
|
|
||||||
|
handle_gemini_session(srv_root, &mut gemini_session)?;
|
||||||
|
|
||||||
|
log_info(
|
||||||
|
&gemini_session,
|
||||||
|
&format!("Finished ({}ms)", now_unix_millis() - timestamp1),
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
use std::net::TcpStream;
|
||||||
|
|
||||||
|
use hex::ToHex;
|
||||||
|
use openssl::hash::MessageDigest;
|
||||||
|
use openssl::ssl::{SslAcceptor, SslFiletype, SslMethod, SslStream, SslVerifyMode};
|
||||||
|
|
||||||
|
pub fn create_tls_acceptor(key_filename: &str, cert_filename: &str) -> anyhow::Result<SslAcceptor> {
|
||||||
|
let mut acceptor = SslAcceptor::mozilla_intermediate(SslMethod::tls())?;
|
||||||
|
acceptor.set_private_key_file(key_filename, SslFiletype::PEM)?;
|
||||||
|
acceptor.set_certificate_chain_file(cert_filename)?;
|
||||||
|
|
||||||
|
// Enable peer verification
|
||||||
|
acceptor.set_verify(SslVerifyMode::PEER);
|
||||||
|
|
||||||
|
// Custom verification callback
|
||||||
|
acceptor.set_verify_callback(SslVerifyMode::PEER, |preverify_ok, x509_ctx| {
|
||||||
|
if preverify_ok {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
// Check if it's a self-signed certificate
|
||||||
|
let cert = match x509_ctx.current_cert() {
|
||||||
|
Some(cert) => cert,
|
||||||
|
None => return false,
|
||||||
|
};
|
||||||
|
|
||||||
|
let issuer_name = cert.issuer_name().to_der();
|
||||||
|
let subject_name = cert.subject_name().to_der();
|
||||||
|
|
||||||
|
let is_self_signed = match (issuer_name, subject_name) {
|
||||||
|
(Ok(issuer), Ok(subject)) => issuer == subject,
|
||||||
|
_ => return false,
|
||||||
|
};
|
||||||
|
|
||||||
|
if is_self_signed {
|
||||||
|
let public_key = match cert.public_key() {
|
||||||
|
Ok(key) => key,
|
||||||
|
Err(_) => return false,
|
||||||
|
};
|
||||||
|
|
||||||
|
if cert.verify(&public_key).is_ok() {
|
||||||
|
let now = match openssl::asn1::Asn1Time::days_from_now(0) {
|
||||||
|
Ok(time) => time,
|
||||||
|
Err(_) => return false,
|
||||||
|
};
|
||||||
|
|
||||||
|
if cert.not_before() <= now && cert.not_after() >= now {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
false
|
||||||
|
});
|
||||||
|
// Initialize the session ID context
|
||||||
|
let context = &crate::nanoid::nanoid().into_bytes();
|
||||||
|
acceptor.set_session_id_context(context)?;
|
||||||
|
acceptor.check_private_key()?;
|
||||||
|
Ok(acceptor.build())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_peer_certificate_digest(stream: &SslStream<TcpStream>) -> Option<String> {
|
||||||
|
if let Some(cert) = stream.ssl().peer_certificate() {
|
||||||
|
if let Ok(digest) = cert.digest(MessageDigest::ripemd160()) {
|
||||||
|
Some(digest.encode_hex::<String>())
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
pub mod fs;
|
||||||
|
pub mod gemini;
|
||||||
|
pub mod nanoid;
|
||||||
|
pub mod time;
|
||||||
+61
@@ -0,0 +1,61 @@
|
|||||||
|
use std::env;
|
||||||
|
|
||||||
|
use anyhow::{Context, Result};
|
||||||
|
use log::error;
|
||||||
|
|
||||||
|
use gemini_grs::gemini;
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct EnvConfig {
|
||||||
|
pub hostname: String,
|
||||||
|
pub key_path: String,
|
||||||
|
pub cert_path: String,
|
||||||
|
pub server_root: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl EnvConfig {
|
||||||
|
pub fn from_env() -> Result<Self> {
|
||||||
|
let hostname = env::var("GEMINI_SERVER_HOSTNAME").unwrap_or("127.0.0.1:1965".to_string());
|
||||||
|
let key_path = env::var("GEMINI_SERVER_TLS_KEY_FILENAME").with_context(|| {
|
||||||
|
"Missing environment variable GEMINI_SERVER_TLS_KEY_FILENAME".to_string()
|
||||||
|
})?;
|
||||||
|
let cert_path = env::var("GEMINI_SERVER_TLS_CERT_FILENAME").with_context(|| {
|
||||||
|
"Missing environment variable GEMINI_SERVER_TLS_CERT_FILENAME".to_string()
|
||||||
|
})?;
|
||||||
|
let server_root = env::var("GEMINI_SERVER_ROOT_DIRECTORY").with_context(|| {
|
||||||
|
"Missing environment variable GEMINI_SERVER_ROOT_DIRECTORY".to_string()
|
||||||
|
})?;
|
||||||
|
Ok(Self {
|
||||||
|
hostname,
|
||||||
|
key_path,
|
||||||
|
cert_path,
|
||||||
|
server_root,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn main() {
|
||||||
|
let exit_code = run();
|
||||||
|
std::process::exit(exit_code);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run() -> i32 {
|
||||||
|
env_logger::init();
|
||||||
|
let env_config = match EnvConfig::from_env() {
|
||||||
|
Err(e) => {
|
||||||
|
error!("{:#?}", e);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
Ok(x) => x,
|
||||||
|
};
|
||||||
|
if let Err(e) = gemini::server::start_server(
|
||||||
|
&env_config.hostname,
|
||||||
|
&env_config.key_path,
|
||||||
|
&env_config.cert_path,
|
||||||
|
&env_config.server_root,
|
||||||
|
) {
|
||||||
|
error!("{:#?}", e);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
0
|
||||||
|
}
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
use nanoid::nanoid;
|
||||||
|
|
||||||
|
pub fn nanoid() -> String {
|
||||||
|
let alphabet = [
|
||||||
|
'1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'a', 'b', 'c', 'd', 'e', 'f',
|
||||||
|
'g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z',
|
||||||
|
'A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T',
|
||||||
|
'U','V','W','X','Y','Z',
|
||||||
|
];
|
||||||
|
nanoid!(16, &alphabet)
|
||||||
|
}
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
|
// returns the time since the epoch in milliseconds
|
||||||
|
pub fn now_unix_millis() -> u128 {
|
||||||
|
SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.unwrap()
|
||||||
|
.as_millis()
|
||||||
|
}
|
||||||
-14
@@ -1,14 +0,0 @@
|
|||||||
package uid
|
|
||||||
|
|
||||||
import (
|
|
||||||
nanoid "github.com/matoous/go-nanoid/v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
func UID() string {
|
|
||||||
// No 'o','O' and 'l'
|
|
||||||
id, err := nanoid.Generate("abcdefghijkmnpqrstuvwxyzABCDEFGHIJKLMNPQRSTUVWXYZ0123456789", 20)
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
return id
|
|
||||||
}
|
|
||||||
@@ -1,11 +0,0 @@
|
|||||||
package util
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"runtime/debug"
|
|
||||||
)
|
|
||||||
|
|
||||||
func PrintStackAndPanic(err error) {
|
|
||||||
fmt.Printf("Error %s Stack trace:\n%s", err, debug.Stack())
|
|
||||||
panic("PANIC")
|
|
||||||
}
|
|
||||||
Reference in New Issue
Block a user