Compare commits
18 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 antanst
|
ef3f009709 | ||
|
antanst
|
c49a69728a | ||
|
antanst
|
8d9ea6cdec | ||
|
antanst
|
f9b5fd5e7f | ||
|
antanst
|
62369d90ae | ||
|
antanst
|
e51d84cad8 | ||
|
antanst
|
17ef03d621 | ||
|
antanst
|
a2a6bd200a | ||
|
antanst
|
3c5206ae43 | ||
|
antanst
|
cd60c1363b | ||
|
antanst
|
212345764b | ||
|
antanst
|
8278f2b204 | ||
|
antanst
|
91f8e69fdf | ||
|
antanst
|
7e849feffe | ||
|
antanst
|
c3d6481de0 | ||
|
antanst
|
74e9327b0b | ||
|
antanst
|
74be6b4d0d | ||
|
antanst
|
eb963542b7 |
+9
-9
@@ -1,9 +1,9 @@
|
||||
/.idea
|
||||
/target
|
||||
|
||||
|
||||
# Added by cargo
|
||||
#
|
||||
# already existing elements were commented out
|
||||
|
||||
#/target
|
||||
.idea
|
||||
**/.#*
|
||||
**/*~
|
||||
/cmd
|
||||
/db/initdb.sql
|
||||
/db/*sh
|
||||
/run*.sh
|
||||
/gemini-grc
|
||||
/snaps
|
||||
|
||||
Generated
-551
@@ -1,551 +0,0 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 3
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "1.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstream"
|
||||
version = "0.6.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"anstyle-parse",
|
||||
"anstyle-query",
|
||||
"anstyle-wincon",
|
||||
"colorchoice",
|
||||
"is_terminal_polyfill",
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle"
|
||||
version = "1.0.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1"
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-parse"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb"
|
||||
dependencies = [
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-query"
|
||||
version = "1.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a"
|
||||
dependencies = [
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-wincon"
|
||||
version = "3.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.86"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.1.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72db2f7947ecee9b03b510377e8bb9077afa27176fdbff55c51027e976fdcc48"
|
||||
dependencies = [
|
||||
"shlex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||
|
||||
[[package]]
|
||||
name = "colorchoice"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0"
|
||||
|
||||
[[package]]
|
||||
name = "env_filter"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4f2c92ceda6ceec50f43169f9ee8424fe2db276791afde7b2cd8bc084cb376ab"
|
||||
dependencies = [
|
||||
"log",
|
||||
"regex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "env_logger"
|
||||
version = "0.11.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e13fa619b91fb2381732789fc5de83b45675e882f66623b7d8cb4f643017018d"
|
||||
dependencies = [
|
||||
"anstream",
|
||||
"anstyle",
|
||||
"env_filter",
|
||||
"humantime",
|
||||
"log",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "foreign-types"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
|
||||
dependencies = [
|
||||
"foreign-types-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "foreign-types-shared"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
|
||||
|
||||
[[package]]
|
||||
name = "form_urlencoded"
|
||||
version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456"
|
||||
dependencies = [
|
||||
"percent-encoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gemini-grs"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"env_logger",
|
||||
"hex",
|
||||
"log",
|
||||
"nanoid",
|
||||
"openssl",
|
||||
"path-clean",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"wasi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hex"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
|
||||
|
||||
[[package]]
|
||||
name = "humantime"
|
||||
version = "2.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6"
|
||||
dependencies = [
|
||||
"unicode-bidi",
|
||||
"unicode-normalization",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "is_terminal_polyfill"
|
||||
version = "1.70.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.156"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a5f43f184355eefb8d17fc948dbecf6c13be3c141f20d834ae842193a448c72a"
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.22"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.7.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
|
||||
|
||||
[[package]]
|
||||
name = "nanoid"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3ffa00dec017b5b1a8b7cf5e2c008bfda1aa7e0697ac1508b491fdf2622fb4d8"
|
||||
dependencies = [
|
||||
"rand",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.19.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
|
||||
|
||||
[[package]]
|
||||
name = "openssl"
|
||||
version = "0.10.66"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"cfg-if",
|
||||
"foreign-types",
|
||||
"libc",
|
||||
"once_cell",
|
||||
"openssl-macros",
|
||||
"openssl-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "openssl-macros"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "openssl-sys"
|
||||
version = "0.9.103"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
"pkg-config",
|
||||
"vcpkg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "path-clean"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "17359afc20d7ab31fdb42bb844c8b3bb1dabd7dcf7e68428492da7f16966fcef"
|
||||
|
||||
[[package]]
|
||||
name = "percent-encoding"
|
||||
version = "2.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
|
||||
|
||||
[[package]]
|
||||
name = "pkg-config"
|
||||
version = "0.3.30"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04"
|
||||
dependencies = [
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.86"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.36"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.8.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"rand_chacha",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_chacha"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
|
||||
dependencies = [
|
||||
"ppv-lite86",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.10.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-automata",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-automata"
|
||||
version = "0.4.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.8.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.74"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fceb41e3d546d0bd83421d3409b1460cc7444cd389341a4c880fe7a042cb3d7"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinyvec"
|
||||
version = "1.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938"
|
||||
dependencies = [
|
||||
"tinyvec_macros",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinyvec_macros"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-bidi"
|
||||
version = "0.3.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-normalization"
|
||||
version = "0.1.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5"
|
||||
dependencies = [
|
||||
"tinyvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "url"
|
||||
version = "2.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c"
|
||||
dependencies = [
|
||||
"form_urlencoded",
|
||||
"idna",
|
||||
"percent-encoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "utf8parse"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "vcpkg"
|
||||
version = "0.2.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.11.0+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.52.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
|
||||
dependencies = [
|
||||
"windows-targets",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-targets"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
|
||||
dependencies = [
|
||||
"windows_aarch64_gnullvm",
|
||||
"windows_aarch64_msvc",
|
||||
"windows_i686_gnu",
|
||||
"windows_i686_gnullvm",
|
||||
"windows_i686_msvc",
|
||||
"windows_x86_64_gnu",
|
||||
"windows_x86_64_gnullvm",
|
||||
"windows_x86_64_msvc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnullvm"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnullvm"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.7.35"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.7.35"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
-14
@@ -1,14 +0,0 @@
|
||||
[package]
|
||||
name = "gemini-grs"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
anyhow = "1.0.86"
|
||||
env_logger = "0.11.5"
|
||||
hex = "0.4.3"
|
||||
log = "0.4.22"
|
||||
nanoid = "0.4.0"
|
||||
openssl = "0.10.66"
|
||||
path-clean = "1.0.1"
|
||||
url = "2.5.2"
|
||||
@@ -1,51 +1,31 @@
|
||||
# gemini-grs
|
||||
# gemini-grc
|
||||
|
||||
A Gemini protocol server written in Rust.
|
||||
A Gemini crawler.
|
||||
|
||||
## Done
|
||||
- [x] Concurrent downloading with workers
|
||||
- [x] Concurrent connection limit per host
|
||||
- [x] URL Blacklist
|
||||
- [x] Save image/* and text/* files
|
||||
- [x] Configuration via environment variables
|
||||
- [x] Storing snapshots in PostgreSQL
|
||||
- [x] Proper response header & body UTF-8 and format validation
|
||||
- [x] Follow robots.txt
|
||||
|
||||
## TODO
|
||||
- [ ] Take into account gemini://geminiprotocol.net/docs/companion/robots.gmi
|
||||
- [ ] Proper handling of all response codes
|
||||
- [ ] Handle 3X redirects properly
|
||||
- [ ] Handle URLs that need presentation of a TLS cert, like astrobotany
|
||||
+ [ ] Probably have a common "grc" cert for all?
|
||||
- [ ] Proper input and response validations:
|
||||
+ [ ] When making a request, the URI MUST NOT exceed 1024 bytes
|
||||
- [ ] Subscriptions to gemini pages? gemini://geminiprotocol.net/docs/companion/
|
||||
|
||||
- [x] Configuration via environment variables
|
||||
- [x] Proper logging
|
||||
- [x] Serve images
|
||||
- [ ] Read and send file in chunks instead of serving whole file from memory
|
||||
- [ ] Expand to serve other protocols (spartan, scroll, titan etc.)
|
||||
- [ ] Async I/O?
|
||||
|
||||
## External dependencies
|
||||
|
||||
OpenSSL
|
||||
|
||||
## Building
|
||||
|
||||
```shell
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
## Running
|
||||
|
||||
Generate your TLS keys:
|
||||
|
||||
```shell
|
||||
openssl genrsa -out key.pem 2048
|
||||
openssl req -new -key key.pem -out request.pem
|
||||
openssl x509 -req -days 36500 -in request.pem -signkey key.pem -out cert.pem
|
||||
```
|
||||
|
||||
Environment variables with examples:
|
||||
|
||||
- GEMINI_SERVER_HOSTNAME="0.0.0.0:1965"
|
||||
- GEMINI_SERVER_TLS_KEY_FILENAME="/where/you/put/key.pem"
|
||||
- GEMINI_SERVER_TLS_CERT_FILENAME="/where/you/put/cert.pem"
|
||||
- GEMINI_SERVER_ROOT_DIRECTORY="/files/to/serve"
|
||||
- RUST_LOG="debug"
|
||||
|
||||
Example command:
|
||||
|
||||
```shell
|
||||
GEMINI_SERVER_TLS_KEY_FILENAME=/home/user/server/key.pem \
|
||||
GEMINI_SERVER_TLS_CERT_FILENAME=/home/user/server/cert.pem \
|
||||
GEMINI_SERVER_ROOT_DIRECTORY=/gemini-root \
|
||||
RUST_LOG=debug \
|
||||
./target/release/gemini-grs
|
||||
```
|
||||
|
||||
## TODO for later
|
||||
- [ ] Add other protocols
|
||||
+ [ ] Gopher
|
||||
+ [ ] Scroll gemini://auragem.letz.dev/devlog/20240316.gmi
|
||||
+ [ ] Spartan
|
||||
+ [ ] Nex
|
||||
+ [ ] SuperTXT https://supertxt.net/00-intro.html
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
LogLevel zerolog.Level
|
||||
rootPath string
|
||||
MaxResponseSize int
|
||||
NumOfWorkers int
|
||||
ResponseTimeout int
|
||||
WorkerBatchSize int
|
||||
}
|
||||
|
||||
var CONFIG Config
|
||||
|
||||
func GetConfig() *Config {
|
||||
var config Config
|
||||
for _, envVar := range []string{
|
||||
"LOG_LEVEL",
|
||||
"ROOT_PATH",
|
||||
"NUM_OF_WORKERS",
|
||||
"WORKER_BATCH_SIZE",
|
||||
"MAX_RESPONSE_SIZE",
|
||||
"RESPONSE_TIMEOUT",
|
||||
} {
|
||||
if env, ok := os.LookupEnv(envVar); !ok {
|
||||
fmt.Fprintf(os.Stderr, "Missing env var %s\n", envVar)
|
||||
os.Exit(1)
|
||||
} else {
|
||||
switch envVar {
|
||||
case "LOG_LEVEL":
|
||||
{
|
||||
logLevel, err := zerolog.ParseLevel(env)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Invalid LOG_LEVEL value\n")
|
||||
os.Exit(1)
|
||||
}
|
||||
config.LogLevel = logLevel
|
||||
}
|
||||
case "ROOT_PATH":
|
||||
{
|
||||
config.rootPath = env
|
||||
}
|
||||
case "NUM_OF_WORKERS":
|
||||
{
|
||||
if numOfWorkers, err := strconv.Atoi(env); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Invalid NUM_OF_WORKERS value\n")
|
||||
os.Exit(1)
|
||||
} else {
|
||||
config.NumOfWorkers = numOfWorkers
|
||||
}
|
||||
}
|
||||
case "WORKER_BATCH_SIZE":
|
||||
{
|
||||
if workerBatchSize, err := strconv.Atoi(env); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Invalid WORKER_BATCH_SIZE value\n")
|
||||
os.Exit(1)
|
||||
} else {
|
||||
config.WorkerBatchSize = workerBatchSize
|
||||
}
|
||||
}
|
||||
case "MAX_RESPONSE_SIZE":
|
||||
{
|
||||
if maxResponseSize, err := strconv.Atoi(env); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Invalid MAX_RESPONSE_SIZE value\n")
|
||||
os.Exit(1)
|
||||
} else {
|
||||
config.MaxResponseSize = maxResponseSize
|
||||
}
|
||||
}
|
||||
case "RESPONSE_TIMEOUT":
|
||||
{
|
||||
if val, err := strconv.Atoi(env); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Invalid RESPONSE_TIMEOUT value\n")
|
||||
os.Exit(1)
|
||||
} else {
|
||||
config.ResponseTimeout = val
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return &config
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
BEGIN;
|
||||
|
||||
-- Increase statement timeout
|
||||
SET statement_timeout = '10min';
|
||||
|
||||
-- Step 1: Create a new table with the same schema
|
||||
CREATE TABLE backup (LIKE snapshots INCLUDING ALL);
|
||||
|
||||
-- Step 2: Copy data from the old table to the new one
|
||||
INSERT INTO backup SELECT * FROM snapshots;
|
||||
|
||||
-- (Optional) Step 3: Truncate the original table if you are moving the data
|
||||
-- TRUNCATE TABLE snapshots;
|
||||
|
||||
-- Commit the transaction if everything went well
|
||||
COMMIT;
|
||||
@@ -0,0 +1,26 @@
|
||||
-- Explanation:
|
||||
|
||||
-- WITH DuplicateSnapshots AS:
|
||||
-- This is a Common Table Expression (CTE) that selects all rows from the snapshots table.
|
||||
-- ROW_NUMBER() OVER (PARTITION BY url ORDER BY id): This assigns a unique row number to each row with the same url. The PARTITION BY url groups the rows by url, and ORDER BY id ensures that the row with the smallest id is given row_num = 1.
|
||||
-- DELETE FROM snapshots WHERE id IN:
|
||||
-- The DELETE statement deletes rows from the snapshots table where the id is in the result of the subquery.
|
||||
-- WHERE row_num > 1:
|
||||
-- In the subquery, we select only rows where row_num > 1, which means only the duplicate rows (since row_num = 1 is the one row we want to keep).
|
||||
|
||||
-- Result:
|
||||
|
||||
-- This query will delete all duplicate rows from the snapshots table, keeping only the row with the smallest id for each url.
|
||||
-- If multiple rows share the same url, only the first one (based on id) will be retained.
|
||||
|
||||
WITH DuplicateSnapshots AS (
|
||||
SELECT id,
|
||||
ROW_NUMBER() OVER (PARTITION BY url ORDER BY id) AS row_num
|
||||
FROM snapshots
|
||||
)
|
||||
DELETE FROM snapshots
|
||||
WHERE id IN (
|
||||
SELECT id
|
||||
FROM DuplicateSnapshots
|
||||
WHERE row_num > 1
|
||||
);
|
||||
@@ -0,0 +1,5 @@
|
||||
SELECT host, COUNT(*) AS row_count
|
||||
FROM snapshots
|
||||
GROUP BY host
|
||||
ORDER BY row_count DESC
|
||||
LIMIT 10;
|
||||
@@ -0,0 +1,48 @@
|
||||
-- DB creation and users
|
||||
CREATE USER gemini;
|
||||
ALTER USER gemini WITH PASSWORD 'gemini';
|
||||
CREATE DATABASE gemini;
|
||||
GRANT ALL PRIVILEGES ON DATABASE gemini TO gemini;
|
||||
ALTER DATABASE gemini OWNER TO gemini;
|
||||
GRANT ALL PRIVILEGES ON SCHEMA public TO gemini;
|
||||
GRANT ALL PRIVILEGES ON gemini TO gemini;
|
||||
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO gemini;
|
||||
|
||||
-- Extensions
|
||||
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||
|
||||
|
||||
\c gemini
|
||||
|
||||
-- Tables
|
||||
DROP TABLE IF EXISTS snapshots;
|
||||
|
||||
CREATE TABLE snapshots (
|
||||
id SERIAL PRIMARY KEY,
|
||||
uid TEXT NOT NULL UNIQUE,
|
||||
url TEXT NOT NULL,
|
||||
host TEXT NOT NULL,
|
||||
timestamp TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
mimetype TEXT,
|
||||
data BYTEA,
|
||||
gemtext TEXT,
|
||||
links JSONB,
|
||||
lang TEXT,
|
||||
response_code INTEGER,
|
||||
error TEXT
|
||||
);
|
||||
|
||||
ALTER TABLE snapshots OWNER TO gemini;
|
||||
|
||||
CREATE INDEX idx_uid ON snapshots (uid);
|
||||
CREATE INDEX idx_url ON snapshots (url);
|
||||
CREATE INDEX idx_timestamp ON snapshots (timestamp);
|
||||
CREATE INDEX idx_mimetype ON snapshots (mimetype);
|
||||
CREATE INDEX idx_lang ON snapshots (lang);
|
||||
CREATE INDEX idx_response_code ON snapshots (response_code);
|
||||
CREATE INDEX idx_error ON snapshots (error);
|
||||
CREATE INDEX idx_host ON snapshots (host);
|
||||
CREATE INDEX idx_snapshots_unprocessed_no_data ON snapshots (host)
|
||||
WHERE response_code IS NULL AND error IS NULL
|
||||
INCLUDE (id, uid, url, timestamp, mimetype, gemtext, links, lang);
|
||||
CREATE INDEX idx_response_code_error_nulls ON snapshots (response_code, error) WHERE response_code IS NULL AND error IS NULL;
|
||||
@@ -0,0 +1,99 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"gemini-grc/gemini"
|
||||
"os"
|
||||
|
||||
_ "github.com/jackc/pgx/v5/stdlib" // PGX driver for PostgreSQL
|
||||
"github.com/jmoiron/sqlx"
|
||||
)
|
||||
|
||||
func checkIfDone() bool { return true }
|
||||
|
||||
// Populates the `host` field
|
||||
func main() {
|
||||
db := connectToDB()
|
||||
|
||||
if checkIfDone() {
|
||||
fmt.Println("Migration already applied")
|
||||
return
|
||||
}
|
||||
|
||||
count := 0
|
||||
for {
|
||||
// Start the transaction
|
||||
tx, err := db.Beginx()
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return
|
||||
}
|
||||
|
||||
query := `
|
||||
SELECT * FROM snapshots
|
||||
WHERE host IS NULL
|
||||
LIMIT 5000
|
||||
`
|
||||
var snapshots []gemini.Snapshot
|
||||
err = tx.Select(&snapshots, query)
|
||||
if len(snapshots) == 0 {
|
||||
fmt.Println("Done!")
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
err := tx.Rollback()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
for _, s := range snapshots {
|
||||
s.Host = s.URL.Hostname
|
||||
fmt.Println(count, s.UID, s.URL.Hostname)
|
||||
err := gemini.SaveSnapshotToDB(tx, &s)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
err := tx.Rollback()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
count += 1
|
||||
}
|
||||
|
||||
err = tx.Commit()
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
err := tx.Rollback()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func connectToDB() *sqlx.DB {
|
||||
connStr := fmt.Sprintf("postgres://%s:%s@%s:%s/%s",
|
||||
os.Getenv("PG_USER"),
|
||||
os.Getenv("PG_PASSWORD"),
|
||||
os.Getenv("PG_HOST"),
|
||||
os.Getenv("PG_PORT"),
|
||||
os.Getenv("PG_DATABASE"),
|
||||
)
|
||||
|
||||
// Create a connection pool
|
||||
db, err := sqlx.Open("pgx", connStr)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("Unable to connect to database with URL %s: %v\n", connStr, err))
|
||||
}
|
||||
db.SetMaxOpenConns(20)
|
||||
err = db.Ping()
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("Unable to ping database: %v\n", err))
|
||||
}
|
||||
|
||||
fmt.Println("Connected to database")
|
||||
return db
|
||||
}
|
||||
Executable
+14
@@ -0,0 +1,14 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
MAX_RESPONSE_SIZE=10485760 \
|
||||
LOG_LEVEL=info \
|
||||
ROOT_PATH=./snaps \
|
||||
RESPONSE_TIMEOUT=10 \
|
||||
NUM_OF_WORKERS=5 \
|
||||
PG_DATABASE=gemini \
|
||||
PG_HOST=127.0.0.1 \
|
||||
PG_PORT=5433 \
|
||||
PG_USER=gemini \
|
||||
PG_PASSWORD=gemini \
|
||||
go run ./migrate1_host.go
|
||||
@@ -0,0 +1,12 @@
|
||||
SELECT
|
||||
query,
|
||||
total_exec_time AS total_time, -- total time spent on the query execution
|
||||
calls, -- number of times the query has been called
|
||||
mean_exec_time AS mean_time -- average time per execution
|
||||
-- max_exec_time AS max_time -- maximum time taken for any single execution
|
||||
FROM
|
||||
pg_stat_statements
|
||||
ORDER BY
|
||||
total_exec_time DESC -- order by total execution time
|
||||
LIMIT 5;
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
SELECT pg_stat_statements_reset();
|
||||
@@ -0,0 +1,20 @@
|
||||
// func PopulateDB(db *sqlx.DB) {
|
||||
// // Delete all rows in the snapshots table
|
||||
// db.MustExec("TRUNCATE snapshots;")
|
||||
|
||||
// // Prepare the query for inserting a snapshot with uid, url, and timestamp
|
||||
// query := `INSERT INTO snapshots(uid, url, timestamp)
|
||||
// VALUES ($1, $2, $3)`
|
||||
|
||||
// // Calculate the timestamp for 2 days ago
|
||||
// timestamp := time.Now().Add(-48 * time.Hour)
|
||||
|
||||
// db.MustExec(query, uid.UID(), "gemini://geminiprotocol.net/", timestamp)
|
||||
// db.MustExec(query, uid.UID(), "gemini://warmedal.se/~antenna", timestamp)
|
||||
// db.MustExec(query, uid.UID(), "gemini://skyjake.fi/~Cosmos/", timestamp)
|
||||
// db.MustExec(query, uid.UID(), "gemini://gemini.circumlunar.space/capcom/", timestamp)
|
||||
// db.MustExec(query, uid.UID(), "gemini://auragem.letz.dev/", timestamp)
|
||||
// db.MustExec(query, uid.UID(), "gemini://gemplex.space/", timestamp)
|
||||
// db.MustExec(query, uid.UID(), "gemini://kennedy.gemi.dev/", timestamp)
|
||||
// db.MustExec(query, uid.UID(), "gemini://tlgs.one/", timestamp)
|
||||
// }
|
||||
@@ -0,0 +1,9 @@
|
||||
BEGIN;
|
||||
|
||||
SET statement_timeout = '10min';
|
||||
|
||||
TRUNCATE TABLE snapshots;
|
||||
|
||||
INSERT INTO snapshots SELECT * FROM backup;
|
||||
|
||||
COMMIT;
|
||||
@@ -0,0 +1,9 @@
|
||||
WITH DuplicateSnapshots AS (
|
||||
SELECT id,
|
||||
url,
|
||||
ROW_NUMBER() OVER (PARTITION BY url ORDER BY id) AS row_num
|
||||
FROM snapshots
|
||||
)
|
||||
SELECT *
|
||||
FROM DuplicateSnapshots
|
||||
WHERE row_num > 1;
|
||||
@@ -0,0 +1,5 @@
|
||||
SELECT
|
||||
COUNT(CASE WHEN response_code IS NOT NULL AND error IS NULL THEN 1 END) AS "Visited",
|
||||
COUNT(CASE WHEN response_code IS NULL AND error IS NULL THEN 1 END) AS "Pending",
|
||||
COUNT(CASE WHEN error IS NOT NULL THEN 1 END) AS "Errors"
|
||||
FROM snapshots;
|
||||
@@ -0,0 +1,8 @@
|
||||
// body with null byte
|
||||
gemini://kennedy.gemi.dev/archive/cached?url=gemini://spam.works/mirrors/textfiles/fun/consult.how&t=638427244900000000&raw=False
|
||||
|
||||
// has invalid url
|
||||
gemini://tlgs.one/known-hosts
|
||||
|
||||
// Needs SNI TLS info (our bug)
|
||||
gemini://hanzbrix.pollux.casa/gemlog/20241002.gmi
|
||||
@@ -0,0 +1,32 @@
|
||||
package gemini
|
||||
|
||||
import (
|
||||
"gemini-grc/logging"
|
||||
)
|
||||
|
||||
var IpPool IpAddressPool = IpAddressPool{IPs: make(map[string]int)}
|
||||
|
||||
func AddIPsToPool(IPs []string) {
|
||||
IpPool.Lock.Lock()
|
||||
for _, ip := range IPs {
|
||||
logging.LogDebug("Adding %s to pool", ip)
|
||||
IpPool.IPs[ip]++
|
||||
}
|
||||
IpPool.Lock.Unlock()
|
||||
}
|
||||
|
||||
func RemoveIPsFromPool(IPs []string) {
|
||||
IpPool.Lock.Lock()
|
||||
for _, ip := range IPs {
|
||||
_, ok := IpPool.IPs[ip]
|
||||
if ok {
|
||||
logging.LogDebug("Removing %s from pool", ip)
|
||||
if IpPool.IPs[ip] == 1 {
|
||||
delete(IpPool.IPs, ip)
|
||||
} else {
|
||||
IpPool.IPs[ip]--
|
||||
}
|
||||
}
|
||||
}
|
||||
IpPool.Lock.Unlock()
|
||||
}
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
package gemini
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"gemini-grc/logging"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// sanitizePath encodes invalid filesystem characters using URL encoding.
|
||||
// Example:
|
||||
// /example/path/to/page?query=param&another=value
|
||||
// would become
|
||||
// example/path/to/page%3Fquery%3Dparam%26another%3Dvalue
|
||||
func sanitizePath(p string) string {
|
||||
// Split the path into its components
|
||||
components := strings.Split(p, "/")
|
||||
|
||||
// Encode each component separately
|
||||
for i, component := range components {
|
||||
// Decode any existing percent-encoded characters
|
||||
decodedComponent, err := url.PathUnescape(component)
|
||||
if err != nil {
|
||||
decodedComponent = component // Fallback to original if unescape fails
|
||||
}
|
||||
|
||||
// Encode the component to escape invalid filesystem characters
|
||||
encodedComponent := url.QueryEscape(decodedComponent)
|
||||
|
||||
// Replace '+' (from QueryEscape) with '%20' to handle spaces correctly
|
||||
encodedComponent = strings.ReplaceAll(encodedComponent, "+", "%20")
|
||||
|
||||
components[i] = encodedComponent
|
||||
}
|
||||
|
||||
// Rejoin the components into a sanitized path
|
||||
safe := filepath.Join(components...)
|
||||
|
||||
return safe
|
||||
}
|
||||
|
||||
// getFilePath constructs a safe file path from the root path and URL path.
|
||||
// It URL-encodes invalid filesystem characters to ensure the path is valid.
|
||||
func calcFilePath(rootPath, urlPath string) (string, error) {
|
||||
// Normalize the URL path
|
||||
cleanPath := filepath.Clean(urlPath)
|
||||
|
||||
// Safe check to prevent directory traversal
|
||||
if strings.Contains(cleanPath, "..") {
|
||||
return "", fmt.Errorf("Invalid URL path: contains directory traversal")
|
||||
}
|
||||
|
||||
// Sanitize the path by encoding invalid characters
|
||||
safePath := sanitizePath(cleanPath)
|
||||
|
||||
// Join the root path and the sanitized URL path
|
||||
finalPath := filepath.Join(rootPath, safePath)
|
||||
|
||||
return finalPath, nil
|
||||
}
|
||||
|
||||
func SaveToFile(rootPath string, s *Snapshot, done chan struct{}) {
|
||||
parentPath := path.Join(rootPath, s.URL.Hostname)
|
||||
urlPath := s.URL.Path
|
||||
// If path is empty, add `index.gmi` as the file to save
|
||||
if urlPath == "" || urlPath == "." {
|
||||
urlPath = fmt.Sprintf("index.gmi")
|
||||
}
|
||||
// If path ends with '/' then add index.gmi for the
|
||||
// directory to be created.
|
||||
if strings.HasSuffix(urlPath, "/") {
|
||||
urlPath = strings.Join([]string{urlPath, "index.gmi"}, "")
|
||||
}
|
||||
|
||||
finalPath, err := calcFilePath(parentPath, urlPath)
|
||||
if err != nil {
|
||||
logging.LogError("Error saving %s: %w", s.URL, err)
|
||||
return
|
||||
}
|
||||
// Ensure the directory exists
|
||||
dir := filepath.Dir(finalPath)
|
||||
if err := os.MkdirAll(dir, os.ModePerm); err != nil {
|
||||
logging.LogError("Failed to create directory: %w", err)
|
||||
return
|
||||
}
|
||||
if s.MimeType.Valid && s.MimeType.String == "text/gemini" {
|
||||
err = os.WriteFile(finalPath, (*s).Data.V, 0666)
|
||||
} else {
|
||||
err = os.WriteFile(finalPath, []byte((*s).GemText.String), 0666)
|
||||
}
|
||||
if err != nil {
|
||||
logging.LogError("Error saving %s: %w", s.URL.Full, err)
|
||||
}
|
||||
close(done)
|
||||
}
|
||||
|
||||
func ReadLines(path string) []string {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("Failed to read file: %s", err))
|
||||
}
|
||||
lines := strings.Split(string(data), "\n")
|
||||
// Remove last line if empty
|
||||
// (happens when file ends with '\n')
|
||||
if lines[len(lines)-1] == "" {
|
||||
lines = lines[:len(lines)-1]
|
||||
}
|
||||
return lines
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
package gemini
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"gemini-grc/logging"
|
||||
"net/url"
|
||||
gourl "net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func isGeminiURL(url string) bool {
|
||||
_, err := gourl.Parse(url)
|
||||
if err != nil {
|
||||
logging.LogWarn("[%s] Invalid URL: %v", url, err)
|
||||
return false
|
||||
}
|
||||
return strings.HasPrefix(url, "gemini://")
|
||||
}
|
||||
|
||||
func parseLinks(s Snapshot, queue chan string) {
|
||||
for _, link := range *s.Links {
|
||||
if strings.HasPrefix(link.Full, "gemini://") {
|
||||
go func(link GeminiUrl) {
|
||||
// fmt.Printf("LINK: %s\n", link)
|
||||
queue <- link.Full
|
||||
}(link)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func checkGeminiStatusCode(code int) error {
|
||||
switch {
|
||||
case code == 20:
|
||||
return nil
|
||||
case code >= 10 && code < 20:
|
||||
return fmt.Errorf("gemini response %d needs data input", code)
|
||||
case code >= 30 && code < 40:
|
||||
return fmt.Errorf("gemini response %d redirect", code)
|
||||
case code >= 40 && code < 50:
|
||||
return fmt.Errorf("gemini response %d server error", code)
|
||||
case code >= 50 && code < 60:
|
||||
return fmt.Errorf("gemini response %d server permanent error", code)
|
||||
case code >= 60 && code < 70:
|
||||
return fmt.Errorf("gemini response %d certificate error", code)
|
||||
default:
|
||||
return fmt.Errorf("unexpected/unhandled Gemini response %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
func ProcessGemini(snapshot *Snapshot) *Snapshot {
|
||||
// Grab link lines
|
||||
linkLines := ExtractLinkLines(snapshot.GemText.String)
|
||||
logging.LogDebug("[%s] Found %d links", snapshot.URL.String(), len(linkLines))
|
||||
|
||||
// Normalize URLs in links, and store them in snapshot
|
||||
for _, line := range linkLines {
|
||||
normalizedLink, descr, err := NormalizeLink(line, snapshot.URL.String())
|
||||
if err != nil {
|
||||
logging.LogDebug("Cannot normalize URL in line '%s': %v", line, err)
|
||||
continue
|
||||
}
|
||||
geminiUrl, err := ParseUrl(normalizedLink, descr)
|
||||
if err != nil {
|
||||
logging.LogDebug("Cannot parse URL in link '%s': %v", line, err)
|
||||
continue
|
||||
}
|
||||
if snapshot.Links == nil {
|
||||
snapshot.Links = &LinkList{*geminiUrl}
|
||||
} else {
|
||||
*snapshot.Links = append(*snapshot.Links, *geminiUrl)
|
||||
}
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
|
||||
func ParseUrl(input string, descr string) (*GeminiUrl, error) {
|
||||
u, err := url.Parse(input)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error parsing URL %s: %w", input, err)
|
||||
}
|
||||
protocol := u.Scheme
|
||||
hostname := u.Hostname()
|
||||
strPort := u.Port()
|
||||
path := u.Path
|
||||
if strPort == "" {
|
||||
strPort = "1965"
|
||||
}
|
||||
port, err := strconv.Atoi(strPort)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error parsing URL %s: %w", input, err)
|
||||
}
|
||||
return &GeminiUrl{Protocol: protocol, Hostname: hostname, Port: port, Path: path, Descr: descr, Full: u.String()}, nil
|
||||
}
|
||||
|
||||
// ExtractLinkLines takes a Gemtext document as a string and returns all lines that are link lines
|
||||
func ExtractLinkLines(gemtext string) []string {
|
||||
// Define the regular expression pattern to match link lines
|
||||
re := regexp.MustCompile(`(?m)^=>[ \t]+.*`)
|
||||
|
||||
// Find all matches using the regular expression
|
||||
matches := re.FindAllString(gemtext, -1)
|
||||
|
||||
return matches
|
||||
}
|
||||
|
||||
// NormalizeLink takes a single link line and the current URL,
|
||||
// return the URL converted to an absolute URL
|
||||
// and its description.
|
||||
func NormalizeLink(linkLine string, currentURL string) (link string, descr string, err error) {
|
||||
// Parse the current URL
|
||||
baseURL, err := url.Parse(currentURL)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("invalid current URL: %v", err)
|
||||
}
|
||||
|
||||
// Regular expression to extract the URL part from a link line
|
||||
re := regexp.MustCompile(`^=>[ \t]+(\S+)([ \t]+.*)?`)
|
||||
|
||||
// Use regex to extract the URL and the rest of the line
|
||||
matches := re.FindStringSubmatch(linkLine)
|
||||
if len(matches) == 0 {
|
||||
// If the line doesn't match the expected format, return it unchanged
|
||||
return "", "", fmt.Errorf("not a link line: %v", linkLine)
|
||||
}
|
||||
|
||||
originalURLStr := matches[1]
|
||||
_, err = url.QueryUnescape(originalURLStr)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("error decoding URL: %w", err)
|
||||
}
|
||||
|
||||
restOfLine := ""
|
||||
if len(matches) > 2 {
|
||||
restOfLine = matches[2]
|
||||
}
|
||||
|
||||
// Parse the URL from the link line
|
||||
parsedURL, err := url.Parse(originalURLStr)
|
||||
if err != nil {
|
||||
// If URL parsing fails, return an error
|
||||
return "", "", fmt.Errorf("invalid URL '%s': %v", originalURLStr, err)
|
||||
}
|
||||
|
||||
// Resolve relative URLs against the base URL
|
||||
if !parsedURL.IsAbs() {
|
||||
parsedURL = baseURL.ResolveReference(parsedURL)
|
||||
}
|
||||
|
||||
// Construct the canonicalized link line
|
||||
canonicalURLStr := parsedURL.String()
|
||||
|
||||
// Remove usual first space from URL description:
|
||||
// => URL description
|
||||
// ^^^^^^^^^^^^
|
||||
if len(restOfLine) > 0 && restOfLine[0] == ' ' {
|
||||
restOfLine = restOfLine[1:]
|
||||
}
|
||||
|
||||
return canonicalURLStr, restOfLine, nil
|
||||
// canonicalizedLine := fmt.Sprintf("=> %s%s", canonicalURLStr, restOfLine)
|
||||
// return canonicalizedLine, nil
|
||||
}
|
||||
|
||||
// ParseFirstTwoDigits takes a string and returns the first one or two digits as an int.
|
||||
// If no valid digits are found, it returns an error.
|
||||
func ParseFirstTwoDigits(input string) (int, error) {
|
||||
// Define the regular expression pattern to match one or two leading digits
|
||||
re := regexp.MustCompile(`^(\d{1,2})`)
|
||||
|
||||
// Find the first match in the string
|
||||
matches := re.FindStringSubmatch(input)
|
||||
if len(matches) == 0 {
|
||||
return 0, errors.New("no digits found at the beginning of the string")
|
||||
}
|
||||
|
||||
// Parse the captured match as an integer
|
||||
snapshot, err := strconv.Atoi(matches[1])
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to convert matched digits to int: %v", err)
|
||||
}
|
||||
|
||||
return snapshot, nil
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package gemini
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"gemini-grc/logging"
|
||||
)
|
||||
|
||||
type GeminiUrl struct {
|
||||
Protocol string `json:"protocol,omitempty"`
|
||||
Hostname string `json:"hostname,omitempty"`
|
||||
Port int `json:"port,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Descr string `json:"descr,omitempty"`
|
||||
Full string `json:"full,omitempty"`
|
||||
}
|
||||
|
||||
func (g *GeminiUrl) Scan(value interface{}) error {
|
||||
if value == nil {
|
||||
// Clear the fields in the current GeminiUrl object (not the pointer itself)
|
||||
*g = GeminiUrl{}
|
||||
return nil
|
||||
}
|
||||
b, ok := value.(string)
|
||||
if !ok {
|
||||
return fmt.Errorf("failed to scan GeminiUrl: expected string, got %T", value)
|
||||
}
|
||||
parsedUrl, err := ParseUrl(b, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*g = *parsedUrl
|
||||
return nil
|
||||
}
|
||||
|
||||
func (u GeminiUrl) String() string {
|
||||
return u.Full
|
||||
// return fmt.Sprintf("%s://%s:%d%s", u.Protocol, u.Hostname, u.Port, u.Path)
|
||||
}
|
||||
|
||||
func GeminiUrltoJSON(g GeminiUrl) string {
|
||||
// Serialize the Person struct to JSON
|
||||
jsonData, err := json.Marshal(g)
|
||||
if err != nil {
|
||||
logging.LogError("Error serializing to JSON: %w", err)
|
||||
}
|
||||
return string(jsonData)
|
||||
}
|
||||
|
||||
func GeminiUrlFromJSON(input string) GeminiUrl {
|
||||
var geminiUrl GeminiUrl
|
||||
err := json.Unmarshal([]byte(input), &geminiUrl)
|
||||
if err != nil {
|
||||
logging.LogError("Error deserializing from JSON: %w", err)
|
||||
}
|
||||
return geminiUrl
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package gemini
|
||||
|
||||
import "sync"
|
||||
|
||||
// Used to limit requests per
|
||||
// IP address. Maps IP address
|
||||
// to number of active connections.
|
||||
type IpAddressPool struct {
|
||||
IPs map[string]int
|
||||
Lock sync.RWMutex
|
||||
}
|
||||
|
||||
func (p *IpAddressPool) Set(key string, value int) {
|
||||
p.Lock.Lock() // Lock for writing
|
||||
defer p.Lock.Unlock() // Ensure mutex is unlocked after the write
|
||||
p.IPs[key] = value
|
||||
}
|
||||
|
||||
func (p *IpAddressPool) Get(key string) int {
|
||||
p.Lock.RLock() // Lock for reading
|
||||
defer p.Lock.RUnlock() // Ensure mutex is unlocked after reading
|
||||
if value, ok := p.IPs[key]; !ok {
|
||||
return 0
|
||||
} else {
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
func (p *IpAddressPool) Delete(key string) {
|
||||
p.Lock.Lock()
|
||||
defer p.Lock.Unlock()
|
||||
delete(p.IPs, key)
|
||||
}
|
||||
|
||||
func (p *IpAddressPool) Incr(key string) {
|
||||
p.Lock.Lock()
|
||||
defer p.Lock.Unlock()
|
||||
if _, ok := p.IPs[key]; !ok {
|
||||
p.IPs[key] = 1
|
||||
} else {
|
||||
p.IPs[key] = p.IPs[key] + 1
|
||||
}
|
||||
}
|
||||
|
||||
func (p *IpAddressPool) Decr(key string) {
|
||||
p.Lock.Lock()
|
||||
defer p.Lock.Unlock()
|
||||
if val, ok := p.IPs[key]; ok {
|
||||
p.IPs[key] = val - 1
|
||||
if p.IPs[key] == 0 {
|
||||
delete(p.IPs, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
package gemini
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"gemini-grc/config"
|
||||
"io"
|
||||
"net"
|
||||
go_url "net/url"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/guregu/null/v5"
|
||||
)
|
||||
|
||||
type GeminiPageData struct {
|
||||
ResponseCode int
|
||||
MimeType string
|
||||
Lang string
|
||||
GemText string
|
||||
Data []byte
|
||||
}
|
||||
|
||||
// Resolve the URL hostname and
|
||||
// check if we already have an open
|
||||
// connection to this host.
|
||||
// If we can connect, return a list
|
||||
// of the resolved IPs.
|
||||
func getHostIPAddresses(hostname string) ([]string, error) {
|
||||
addrs, err := net.LookupHost(hostname)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
IpPool.Lock.RLock()
|
||||
defer func() {
|
||||
IpPool.Lock.RUnlock()
|
||||
}()
|
||||
return addrs, nil
|
||||
}
|
||||
|
||||
func ConnectAndGetData(url string) ([]byte, error) {
|
||||
parsedUrl, err := go_url.Parse(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Could not parse URL, error %w", err)
|
||||
}
|
||||
hostname := parsedUrl.Hostname()
|
||||
port := parsedUrl.Port()
|
||||
if port == "" {
|
||||
port = "1965"
|
||||
}
|
||||
host := fmt.Sprintf("%s:%s", hostname, port)
|
||||
// Establish the underlying TCP connection.
|
||||
dialer := &net.Dialer{
|
||||
Timeout: time.Duration(config.CONFIG.ResponseTimeout) * time.Second,
|
||||
KeepAlive: 10 * time.Second,
|
||||
}
|
||||
conn, err := dialer.Dial("tcp", host)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("TCP connection failed: %w", err)
|
||||
}
|
||||
// Make sure we always close the connection.
|
||||
defer func() {
|
||||
err := conn.Close()
|
||||
if err != nil {
|
||||
// Do nothing! Connection will timeout eventually if still open somehow.
|
||||
}
|
||||
}()
|
||||
|
||||
// Set read and write timeouts on the TCP connection.
|
||||
err = conn.SetReadDeadline(time.Now().Add(time.Duration(config.CONFIG.ResponseTimeout) * time.Second))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Error setting connection deadline: %w", err)
|
||||
}
|
||||
err = conn.SetWriteDeadline(time.Now().Add(time.Duration(config.CONFIG.ResponseTimeout) * time.Second))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Error setting connection deadline: %w", err)
|
||||
}
|
||||
|
||||
// Perform the TLS handshake
|
||||
tlsConfig := &tls.Config{
|
||||
InsecureSkipVerify: true, // Accept all TLS certs, even if insecure.
|
||||
ServerName: parsedUrl.Hostname(), // SNI should not include port
|
||||
// MinVersion: tls.VersionTLS12, // Use a minimum TLS version. Warning breaks a lot of sites.
|
||||
}
|
||||
tlsConn := tls.Client(conn, tlsConfig)
|
||||
if err := tlsConn.Handshake(); err != nil {
|
||||
return nil, fmt.Errorf("TLS handshake error: %w", err)
|
||||
}
|
||||
|
||||
// We read `buf`-sized chunks and add data to `data`.
|
||||
buf := make([]byte, 4096)
|
||||
var data []byte
|
||||
|
||||
// Send Gemini request to trigger server response.
|
||||
_, err = tlsConn.Write([]byte(fmt.Sprintf("%s\r\n", url)))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Error sending network request: %w", err)
|
||||
}
|
||||
// Read response bytes in len(buf) byte chunks
|
||||
for {
|
||||
n, err := tlsConn.Read(buf)
|
||||
if n > 0 {
|
||||
data = append(data, buf[:n]...)
|
||||
}
|
||||
if len(data) > config.CONFIG.MaxResponseSize {
|
||||
data = []byte{}
|
||||
return nil, fmt.Errorf("Response size exceeded maximum of %d bytes", config.CONFIG.MaxResponseSize)
|
||||
}
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
} else {
|
||||
return nil, fmt.Errorf("Network error: %s", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// Connect to given URL, using the Gemini protocol.
|
||||
// Mutate given Snapshot with the data or the error.
|
||||
func Visit(s *Snapshot) {
|
||||
data, err := ConnectAndGetData(s.URL.String())
|
||||
if err != nil {
|
||||
s.Error = null.StringFrom(err.Error())
|
||||
return
|
||||
}
|
||||
pageData, err := processData(data)
|
||||
if err != nil {
|
||||
s.Error = null.StringFrom(err.Error())
|
||||
return
|
||||
}
|
||||
s.ResponseCode = null.IntFrom(int64(pageData.ResponseCode))
|
||||
s.MimeType = null.StringFrom(pageData.MimeType)
|
||||
s.Lang = null.StringFrom(pageData.Lang)
|
||||
if pageData.GemText != "" {
|
||||
s.GemText = null.StringFrom(string(pageData.GemText))
|
||||
}
|
||||
if pageData.Data != nil {
|
||||
s.Data = null.ValueFrom(pageData.Data)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Update given snapshot with the
|
||||
// Gemini header data: response code,
|
||||
// mime type and lang (optional)
|
||||
func processData(data []byte) (*GeminiPageData, error) {
|
||||
headers, body, err := getHeadersAndData(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
code, mimeType, lang := getMimeTypeAndLang(headers)
|
||||
geminiError := checkGeminiStatusCode(code)
|
||||
if geminiError != nil {
|
||||
return nil, geminiError
|
||||
}
|
||||
pageData := GeminiPageData{
|
||||
ResponseCode: code,
|
||||
MimeType: mimeType,
|
||||
Lang: lang,
|
||||
}
|
||||
// If we've got a Gemini document, populate
|
||||
// `GemText` field, otherwise raw data goes to `Data`.
|
||||
if mimeType == "text/gemini" {
|
||||
validBody, err := EnsureValidUTF8(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("UTF-8 error: %w", err)
|
||||
}
|
||||
pageData.GemText = validBody
|
||||
} else {
|
||||
pageData.Data = body
|
||||
}
|
||||
return &pageData, nil
|
||||
}
|
||||
|
||||
// Checks for a Gemini header, which is
|
||||
// basically the first line of the response
|
||||
// and should contain the response code,
|
||||
// mimeType and language.
|
||||
func getHeadersAndData(data []byte) (firstLine string, rest []byte, err error) {
|
||||
firstLineEnds := slices.Index(data, '\n')
|
||||
if firstLineEnds == -1 {
|
||||
return "", nil, fmt.Errorf("Could not parse response header")
|
||||
}
|
||||
firstLine = string(data[:firstLineEnds])
|
||||
rest = data[firstLineEnds+1:]
|
||||
return string(firstLine), rest, nil
|
||||
}
|
||||
|
||||
// Parses code, mime type and language
|
||||
// from a Gemini header.
|
||||
// Examples:
|
||||
// `20 text/gemini lang=en` (code, mimetype, lang)
|
||||
// `20 text/gemini` (code, mimetype)
|
||||
// `31 gemini://redirected.to/other/site` (code)
|
||||
func getMimeTypeAndLang(headers string) (code int, mimeType string, lang string) {
|
||||
// Regex that parses code, mimetype & lang
|
||||
re := regexp.MustCompile(`^(\d+)\s+([a-zA-Z0-9/\-+]+)(?:[;\s]+(lang=([a-zA-Z0-9-]+)))?\s*$`)
|
||||
matches := re.FindStringSubmatch(headers)
|
||||
if matches == nil || len(matches) <= 1 {
|
||||
// Try to get code at least.
|
||||
re := regexp.MustCompile(`^(\d+)\s+`)
|
||||
matches := re.FindStringSubmatch(headers)
|
||||
if matches == nil || len(matches) <= 1 {
|
||||
return 0, "", ""
|
||||
}
|
||||
code, err := strconv.Atoi(matches[1])
|
||||
if err != nil {
|
||||
return 0, "", ""
|
||||
}
|
||||
return code, "", ""
|
||||
}
|
||||
code, err := strconv.Atoi(matches[1])
|
||||
if err != nil {
|
||||
return 0, "", ""
|
||||
}
|
||||
mimeType = matches[2]
|
||||
lang = matches[4]
|
||||
return code, mimeType, lang
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package gemini
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Test for input: `20 text/gemini`
|
||||
func TestGetMimeTypeAndLang1(t *testing.T) {
|
||||
code, mimeType, lang := getMimeTypeAndLang("20 text/gemini")
|
||||
if code != 20 || mimeType != "text/gemini" || lang != "" {
|
||||
t.Errorf("Expected (20, 'text/gemini', ''), got (%d, '%s', '%s')", code, mimeType, lang)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetMimeTypeAndLang11(t *testing.T) {
|
||||
code, mimeType, lang := getMimeTypeAndLang("20 text/gemini\n")
|
||||
if code != 20 || mimeType != "text/gemini" || lang != "" {
|
||||
t.Errorf("Expected (20, 'text/gemini', ''), got (%d, '%s', '%s')", code, mimeType, lang)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetTypeAndLang2(t *testing.T) {
|
||||
code, mimeType, lang := getMimeTypeAndLang("20 text/gemini lang=en")
|
||||
if code != 20 || mimeType != "text/gemini" || lang != "en" {
|
||||
t.Errorf("Expected (20, 'text/gemini', 'en'), got (%d, '%s', '%s')", code, mimeType, lang)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetMimeTypeAndLang3(t *testing.T) {
|
||||
code, mimeType, lang := getMimeTypeAndLang("31 gemini://redirect.to/page")
|
||||
if code != 31 || mimeType != "" || lang != "" {
|
||||
t.Errorf("Expected (20, '', ''), got (%d, '%s', '%s')", code, mimeType, lang)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetMimeTypeAndLang4(t *testing.T) {
|
||||
code, mimeType, lang := getMimeTypeAndLang("aaafdasdasd")
|
||||
if code != 0 || mimeType != "" || lang != "" {
|
||||
t.Errorf("Expected (0, '', ''), got (%d, '%s', '%s')", code, mimeType, lang)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetMimeTypeAndLang5(t *testing.T) {
|
||||
code, mimeType, lang := getMimeTypeAndLang("")
|
||||
if code != 0 || mimeType != "" || lang != "" {
|
||||
t.Errorf("Expected (0, '', ''), got (%d, '%s', '%s')", code, mimeType, lang)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package gemini
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"gemini-grc/logging"
|
||||
"os"
|
||||
|
||||
_ "github.com/jackc/pgx/v5/stdlib" // PGX driver for PostgreSQL
|
||||
"github.com/jmoiron/sqlx"
|
||||
)
|
||||
|
||||
func ConnectToDB() *sqlx.DB {
|
||||
connStr := fmt.Sprintf("postgres://%s:%s@%s:%s/%s",
|
||||
os.Getenv("PG_USER"),
|
||||
os.Getenv("PG_PASSWORD"),
|
||||
os.Getenv("PG_HOST"),
|
||||
os.Getenv("PG_PORT"),
|
||||
os.Getenv("PG_DATABASE"),
|
||||
)
|
||||
|
||||
// Create a connection pool
|
||||
db, err := sqlx.Open("pgx", connStr)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("Unable to connect to database with URL %s: %v\n", connStr, err))
|
||||
}
|
||||
db.SetMaxOpenConns(20)
|
||||
err = db.Ping()
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("Unable to ping database: %v\n", err))
|
||||
}
|
||||
|
||||
logging.LogDebug("Connected to database")
|
||||
return db
|
||||
}
|
||||
|
||||
func SaveSnapshotToDB(tx *sqlx.Tx, s *Snapshot) error {
|
||||
query := `
|
||||
INSERT INTO snapshots (uid, url, host, timestamp, mimetype, data, gemtext, links, lang, response_code, error)
|
||||
VALUES (:uid, :url, :host, :timestamp, :mimetype, :data, :gemtext, :links, :lang, :response_code, :error)
|
||||
ON CONFLICT (uid) DO UPDATE SET
|
||||
url = EXCLUDED.url,
|
||||
host = EXCLUDED.host,
|
||||
timestamp = EXCLUDED.timestamp,
|
||||
mimetype = EXCLUDED.mimetype,
|
||||
data = EXCLUDED.data,
|
||||
gemtext = EXCLUDED.gemtext,
|
||||
links = EXCLUDED.links,
|
||||
lang = EXCLUDED.lang,
|
||||
response_code = EXCLUDED.response_code,
|
||||
error = EXCLUDED.error
|
||||
`
|
||||
_, err := tx.NamedExec(query, s)
|
||||
if err != nil {
|
||||
logging.LogError("[%s] [%s] Error upserting snapshot: %w", s.URL, s.MimeType.String, err)
|
||||
return fmt.Errorf("DB error: %w", err) // Return the error instead of panicking
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func SaveLinksToDBinBatches(tx *sqlx.Tx, snapshots []*Snapshot) error {
|
||||
// Approximately 5,957 rows maximum (65535/11 parameters), use 5000 to be safe
|
||||
const batchSize = 5000
|
||||
|
||||
query := `
|
||||
INSERT INTO snapshots (uid, url, host, timestamp, mimetype, data, gemtext, links, lang, response_code, error)
|
||||
VALUES (:uid, :url, :host, :timestamp, :mimetype, :data, :gemtext, :links, :lang, :response_code, :error)
|
||||
ON CONFLICT (uid) DO NOTHING
|
||||
`
|
||||
|
||||
for i := 0; i < len(snapshots); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(snapshots) {
|
||||
end = len(snapshots)
|
||||
}
|
||||
|
||||
batch := snapshots[i:end]
|
||||
|
||||
_, err := tx.NamedExec(query, batch)
|
||||
if err != nil {
|
||||
logging.LogError("Error batch inserting snapshots: %w", err)
|
||||
return fmt.Errorf("DB error: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func SaveLinksToDB(tx *sqlx.Tx, snapshots []*Snapshot) error {
|
||||
query := `
|
||||
INSERT INTO snapshots (uid, url, host, timestamp, mimetype, data, gemtext, links, lang, response_code, error)
|
||||
VALUES (:uid, :url, :host, :timestamp, :mimetype, :data, :gemtext, :links, :lang, :response_code, :error)
|
||||
ON CONFLICT (uid) DO NOTHING
|
||||
`
|
||||
_, err := tx.NamedExec(query, snapshots)
|
||||
if err != nil {
|
||||
logging.LogError("Error batch inserting snapshots: %w", err)
|
||||
return fmt.Errorf("DB error: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package gemini
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"unicode/utf8"
|
||||
|
||||
"golang.org/x/text/encoding/charmap"
|
||||
"golang.org/x/text/transform"
|
||||
)
|
||||
|
||||
func EnsureValidUTF8(input []byte) (string, error) {
|
||||
// Remove NULL byte 0x00
|
||||
inputNoNull := bytes.ReplaceAll(input, []byte{0}, nil)
|
||||
isValidUTF8 := utf8.Valid(inputNoNull)
|
||||
if !isValidUTF8 {
|
||||
encodings := []transform.Transformer{
|
||||
charmap.ISO8859_1.NewDecoder(), // First try ISO8859-1
|
||||
charmap.Windows1252.NewDecoder(), // Then try Windows-1252, etc
|
||||
// TODO: Try more encodings?
|
||||
}
|
||||
for _, encoding := range encodings {
|
||||
reader := transform.NewReader(bytes.NewReader(inputNoNull), encoding)
|
||||
result, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("UTF-8 error: %w", err)
|
||||
}
|
||||
return string(result), nil
|
||||
}
|
||||
}
|
||||
return string(inputNoNull), nil
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package gemini
|
||||
|
||||
import "testing"
|
||||
|
||||
// Make sure NULL bytes are removed
|
||||
func TestEnsureValidUTF8(t *testing.T) {
|
||||
// Create a string with a null byte
|
||||
strWithNull := "Hello" + string('\x00') + "world"
|
||||
result, _ := EnsureValidUTF8([]byte(strWithNull))
|
||||
if result != "Helloworld" {
|
||||
t.Errorf("Expected string without NULL byte, got %s", result)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package gemini
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"gemini-grc/logging"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// key: "host:port" (string)
|
||||
// value:
|
||||
// empty []string if no robots data, or
|
||||
// list of URL prefixes ([]string) in robots
|
||||
var RobotsCache sync.Map
|
||||
|
||||
func populateBlacklist(key string) (entries []string) {
|
||||
// We either store an empty list when
|
||||
// no rules, or a list of disallowed URLs.
|
||||
// This applies even if we have an error
|
||||
// finding/downloading robots.txt
|
||||
defer func() {
|
||||
RobotsCache.Store(key, entries)
|
||||
}()
|
||||
url := fmt.Sprintf("gemini://%s/robots.txt", key)
|
||||
robotsContent, err := ConnectAndGetData(url)
|
||||
if err != nil {
|
||||
logging.LogDebug("robots.txt error %s", err)
|
||||
return []string{}
|
||||
}
|
||||
robotsData, err := processData(robotsContent)
|
||||
if err != nil {
|
||||
logging.LogDebug("robots.txt error %s", err)
|
||||
return []string{}
|
||||
}
|
||||
if robotsData.ResponseCode != 20 {
|
||||
logging.LogDebug("robots.txt error code %d, ignoring", robotsData.ResponseCode)
|
||||
return []string{}
|
||||
}
|
||||
// Some return text/plain, others text/gemini.
|
||||
// According to spec, the first is correct,
|
||||
// however let's be lenient
|
||||
var data string
|
||||
if robotsData.MimeType == "text/plain" {
|
||||
data = string(robotsData.Data)
|
||||
} else if robotsData.MimeType == "text/gemini" {
|
||||
data = robotsData.GemText
|
||||
} else {
|
||||
return []string{}
|
||||
}
|
||||
entries = ParseRobotsTxt(string(data), key)
|
||||
return entries
|
||||
}
|
||||
|
||||
// Check if the snapshot URL matches
|
||||
// a robots.txt allow rule.
|
||||
func RobotMatch(s *Snapshot) bool {
|
||||
logging.LogDebug("Checking robots.txt cache for %s", s.URL.String())
|
||||
key := fmt.Sprintf("%s:%d", s.Host, s.URL.Port)
|
||||
v, ok := RobotsCache.Load(key)
|
||||
if ok == false {
|
||||
// First time check, populate robot cache
|
||||
logging.LogDebug("No robots.txt entry, populating cache for %s", s.URL.String())
|
||||
disallowedURLs := populateBlacklist(key)
|
||||
for _, url := range disallowedURLs {
|
||||
if strings.HasPrefix(s.URL.String(), url) {
|
||||
logging.LogDebug("robots.txt match: %s %s", s.URL.String(), url)
|
||||
return true
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if len(v.([]string)) == 0 {
|
||||
logging.LogDebug("No robots.txt or no rules, allowed")
|
||||
return false
|
||||
}
|
||||
for _, url := range v.([]string) {
|
||||
if strings.HasPrefix(s.URL.String(), url) {
|
||||
logging.LogDebug("robots.txt match: %s %s", s.URL.String(), url)
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package gemini
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Takes robots.txt content and a host, and
|
||||
// returns a list of full URLs that shouldn't
|
||||
// be visited.
|
||||
// TODO Also take into account the user agent?
|
||||
// Check gemini://geminiprotocol.net/docs/companion/robots.gmi
|
||||
func ParseRobotsTxt(content string, host string) []string {
|
||||
var disallowedPaths []string
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
line = strings.ToLower(line)
|
||||
if strings.HasPrefix(line, "disallow:") {
|
||||
parts := strings.SplitN(line, ":", 2)
|
||||
if len(parts) == 2 {
|
||||
path := strings.TrimSpace(parts[1])
|
||||
if path != "" {
|
||||
// Construct full Gemini URL
|
||||
disallowedPaths = append(disallowedPaths,
|
||||
fmt.Sprintf("gemini://%s%s", host, path))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return disallowedPaths
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package gemini
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseRobotsTxt(t *testing.T) {
|
||||
input := `User-agent: *
|
||||
Disallow: /cgi-bin/wp.cgi/view
|
||||
Disallow: /cgi-bin/wp.cgi/media
|
||||
User-agent: googlebot
|
||||
Disallow: /admin/`
|
||||
|
||||
expected := []string{
|
||||
"gemini://example.com/cgi-bin/wp.cgi/view",
|
||||
"gemini://example.com/cgi-bin/wp.cgi/media",
|
||||
"gemini://example.com/admin/",
|
||||
}
|
||||
|
||||
result := ParseRobotsTxt(input, "example.com")
|
||||
|
||||
if !reflect.DeepEqual(result, expected) {
|
||||
t.Errorf("ParseRobotsTxt() = %v, want %v", result, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRobotsTxtEmpty(t *testing.T) {
|
||||
input := ``
|
||||
|
||||
result := ParseRobotsTxt(input, "example.com")
|
||||
|
||||
if len(result) != 0 {
|
||||
t.Errorf("ParseRobotsTxt() = %v, want empty []string", result)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package gemini
|
||||
|
||||
import (
|
||||
"database/sql/driver"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"gemini-grc/logging"
|
||||
"strings"
|
||||
|
||||
"github.com/guregu/null/v5"
|
||||
)
|
||||
|
||||
type LinkList []GeminiUrl
|
||||
|
||||
func (l LinkList) Value() (driver.Value, error) {
|
||||
return json.Marshal(l)
|
||||
}
|
||||
|
||||
func (l *LinkList) Scan(value interface{}) error {
|
||||
if value == nil {
|
||||
*l = nil
|
||||
return nil
|
||||
}
|
||||
b, ok := value.([]byte) // Type assertion! Converts to []byte
|
||||
if !ok {
|
||||
return fmt.Errorf("failed to scan LinkList: expected []byte, got %T", value)
|
||||
}
|
||||
return json.Unmarshal(b, l)
|
||||
}
|
||||
|
||||
type Snapshot struct {
|
||||
ID int `db:"id" json:"id,omitempty"`
|
||||
UID string `db:"uid" json:"uid,omitempty"`
|
||||
URL GeminiUrl `db:"url" json:"url,omitempty"`
|
||||
Host string `db:"host" json:"host,omitempty"`
|
||||
Timestamp null.Time `db:"timestamp" json:"timestamp,omitempty"`
|
||||
MimeType null.String `db:"mimetype" json:"mimetype,omitempty"`
|
||||
Data null.Value[[]byte] `db:"data" json:"data,omitempty"` // For non text/gemini files.
|
||||
GemText null.String `db:"gemtext" json:"gemtext,omitempty"` // For text/gemini files.
|
||||
Links *LinkList `db:"links" json:"links,omitempty"`
|
||||
Lang null.String `db:"lang" json:"lang,omitempty"`
|
||||
ResponseCode null.Int `db:"response_code" json:"code,omitempty"` // Gemini response status code.
|
||||
Error null.String `db:"error" json:"error,omitempty"` // On network errors only
|
||||
}
|
||||
|
||||
func SnapshotToJSON(g Snapshot) string {
|
||||
// Serialize the Person struct to JSON
|
||||
jsonData, err := json.MarshalIndent(g, "", "\t")
|
||||
if err != nil {
|
||||
logging.LogError("Error serializing to JSON: %w", err)
|
||||
}
|
||||
return string(jsonData)
|
||||
}
|
||||
|
||||
func SnapshotFromJSON(input string) Snapshot {
|
||||
var snapshot Snapshot
|
||||
err := json.Unmarshal([]byte(input), &snapshot)
|
||||
if err != nil {
|
||||
logging.LogError("Error deserializing from JSON: %w", err)
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
|
||||
func ShouldPersistSnapshot(result *Snapshot) bool {
|
||||
if !result.MimeType.Valid {
|
||||
return false
|
||||
}
|
||||
if result.MimeType.String == "text/gemini" ||
|
||||
strings.HasPrefix(result.MimeType.String, "image/") ||
|
||||
strings.HasPrefix(result.MimeType.String, "text/") {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
package gemini
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"gemini-grc/config"
|
||||
"gemini-grc/logging"
|
||||
"gemini-grc/uid"
|
||||
"gemini-grc/util"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/guregu/null/v5"
|
||||
"github.com/jmoiron/sqlx"
|
||||
)
|
||||
|
||||
func SpawnWorkers(numOfWorkers int, db *sqlx.DB) {
|
||||
logging.LogInfo("Spawning %d workers", numOfWorkers)
|
||||
for i := 0; i < numOfWorkers; i++ {
|
||||
go func(i int) {
|
||||
for {
|
||||
runWorker(i, db)
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
}
|
||||
|
||||
func runWorker(id int, db *sqlx.DB) {
|
||||
// Start the DB transaction
|
||||
tx, err := db.Beginx()
|
||||
if err != nil {
|
||||
logging.LogError("Failed to begin transaction: %w", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
err = tx.Commit()
|
||||
if err != nil {
|
||||
logging.LogError("[%d] Failed to commit transaction: %w", id, err)
|
||||
err := tx.Rollback()
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("[%d] Failed to roll back transaction: %v", id, err))
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
snapshots, err := GetRandomSnapshotsDistinctHosts(tx)
|
||||
|
||||
if err != nil {
|
||||
logging.LogError("[%d] Error retrieving snapshot: %w", id, err)
|
||||
time.Sleep(10 * time.Second)
|
||||
return
|
||||
} else if len(snapshots) == 0 {
|
||||
logging.LogInfo("[%d] No remaining snapshots to visit.", id)
|
||||
time.Sleep(1 * time.Minute)
|
||||
return
|
||||
}
|
||||
total := len(snapshots)
|
||||
for i, s := range snapshots {
|
||||
logging.LogInfo("[%d] Starting %d/%d %s", id, i+1, total, s.URL)
|
||||
err = workOnSnapshot(id, tx, &s)
|
||||
if err != nil {
|
||||
logging.LogError("[%d] [%s] Error %w", id, s.URL, err)
|
||||
util.PrintStackAndPanic(err)
|
||||
}
|
||||
if s.Error.Valid {
|
||||
logging.LogWarn("[%d] [%s] Error: %v", id, s.URL, fmt.Errorf(s.Error.String))
|
||||
}
|
||||
logging.LogDebug("[%d] Done %d/%d.", id, i, total)
|
||||
}
|
||||
logging.LogInfo("[%d] Worker done.", id)
|
||||
}
|
||||
|
||||
func workOnSnapshot(id int, tx *sqlx.Tx, s *Snapshot) (err error) {
|
||||
// If URL matches a robots.txt disallow line,
|
||||
// add it as an error so next time it won't be
|
||||
// crawled.
|
||||
if RobotMatch(s) {
|
||||
s.Error = null.StringFrom("robots.txt disallow match")
|
||||
err = SaveSnapshotToDB(tx, s)
|
||||
if err != nil {
|
||||
return fmt.Errorf("[%d] DB Error: %w", id, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
IPs, err := getHostIPAddresses(s.Host)
|
||||
if err != nil {
|
||||
s.Error = null.StringFrom("DNS Resolve error")
|
||||
err = SaveSnapshotToDB(tx, s)
|
||||
if err != nil {
|
||||
return fmt.Errorf("[%d] DB Error: %w", id, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// If the host's ip is in the connections pool,
|
||||
// stop and add the url in the queue later.
|
||||
IpPool.Lock.RLock()
|
||||
logging.LogDebug("[%d] [%s] Checking pool for IP", id, s.URL)
|
||||
for _, ip := range IPs {
|
||||
_, ok := IpPool.IPs[ip]
|
||||
if ok {
|
||||
logging.LogDebug("[%d] Another worker is visiting this host: %s", id, s.URL)
|
||||
IpPool.Lock.RUnlock()
|
||||
time.Sleep(1 * time.Second) // Avoid flood-retrying when few URLs remain
|
||||
return nil
|
||||
}
|
||||
}
|
||||
IpPool.Lock.RUnlock()
|
||||
|
||||
AddIPsToPool(IPs)
|
||||
|
||||
url := s.URL.String()
|
||||
logging.LogDebug("[%d] Dialing %s", id, url)
|
||||
Visit(s)
|
||||
logging.LogDebug("[%d] Finished dialing.", id)
|
||||
|
||||
go func() {
|
||||
time.Sleep(5 * time.Second)
|
||||
RemoveIPsFromPool(IPs)
|
||||
}()
|
||||
|
||||
if s.MimeType.Valid && s.MimeType.String == "text/gemini" {
|
||||
logging.LogDebug("[%d] [%s] Processing", id, url)
|
||||
s = ProcessGemini(s)
|
||||
}
|
||||
logging.LogDebug("[%d] Saving", id)
|
||||
err = SaveSnapshotToDB(tx, s)
|
||||
if err != nil {
|
||||
return fmt.Errorf("[%d] DB Error: %w", id, err)
|
||||
}
|
||||
|
||||
// Store links in batch
|
||||
if s.Links != nil {
|
||||
var batchSnapshots []*Snapshot
|
||||
timestamp := null.TimeFrom(time.Now())
|
||||
|
||||
for _, link := range *s.Links {
|
||||
if shouldPersistURL(tx, link) {
|
||||
newSnapshot := &Snapshot{
|
||||
UID: uid.UID(),
|
||||
URL: link,
|
||||
Host: link.Hostname,
|
||||
Timestamp: timestamp,
|
||||
}
|
||||
batchSnapshots = append(batchSnapshots, newSnapshot)
|
||||
}
|
||||
}
|
||||
|
||||
if len(batchSnapshots) > 0 {
|
||||
logging.LogDebug("[%d] Batch saving %d links", id, len(batchSnapshots))
|
||||
err = SaveLinksToDBinBatches(tx, batchSnapshots)
|
||||
if err != nil {
|
||||
return fmt.Errorf("[%d] DB Error: %w", id, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Should we save the given URL for crawling?
|
||||
func shouldPersistURL(tx *sqlx.Tx, u GeminiUrl) bool {
|
||||
if !strings.HasPrefix(u.String(), "gemini://") {
|
||||
return false
|
||||
}
|
||||
query := `SELECT EXISTS(SELECT 1 FROM snapshots WHERE URL=$1)`
|
||||
var exists bool
|
||||
err := tx.Get(&exists, query, u.String())
|
||||
if err != nil {
|
||||
fmt.Println("Error executing query:", err)
|
||||
return false
|
||||
}
|
||||
return !exists
|
||||
}
|
||||
|
||||
func GetRandomSnapshotsDistinctHosts(tx *sqlx.Tx) ([]Snapshot, error) {
|
||||
// Old, unoptimized query
|
||||
//
|
||||
// query := `
|
||||
// SELECT DISTINCT ON (host) *
|
||||
// FROM snapshots
|
||||
// WHERE response_code IS NULL
|
||||
// AND error IS NULL
|
||||
// ORDER BY host, RANDOM()
|
||||
// LIMIT $1
|
||||
// `
|
||||
query := `
|
||||
WITH RankedSnapshots AS (
|
||||
SELECT id, uid, url, host, timestamp, mimetype, data, gemtext,
|
||||
links, lang, response_code, error,
|
||||
ROW_NUMBER() OVER (PARTITION BY host ORDER BY RANDOM()) as rn
|
||||
FROM snapshots
|
||||
WHERE response_code IS NULL
|
||||
AND error IS NULL
|
||||
)
|
||||
SELECT id, uid, url, host, timestamp, mimetype, data, gemtext,
|
||||
links, lang, response_code, error
|
||||
FROM RankedSnapshots
|
||||
WHERE rn = 1
|
||||
LIMIT $1
|
||||
`
|
||||
var snapshots []Snapshot
|
||||
err := tx.Select(&snapshots, query, config.CONFIG.WorkerBatchSize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return snapshots, nil
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
module gemini-grc
|
||||
|
||||
go 1.23.1
|
||||
|
||||
require (
|
||||
github.com/jaevor/go-nanoid v1.4.0
|
||||
github.com/rs/zerolog v1.33.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/guregu/null/v5 v5.0.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/pgx/v5 v5.7.1 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/jmoiron/sqlx v1.4.0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
golang.org/x/crypto v0.27.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20241004190924-225e2abe05e6 // indirect
|
||||
golang.org/x/net v0.27.0 // indirect
|
||||
golang.org/x/sync v0.8.0 // indirect
|
||||
golang.org/x/sys v0.25.0 // indirect
|
||||
golang.org/x/text v0.18.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,54 @@
|
||||
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
||||
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/gabriel-vasile/mimetype v1.4.5 h1:J7wGKdGu33ocBOhGy0z653k/lFKLFDPJMG8Gql0kxn4=
|
||||
github.com/gabriel-vasile/mimetype v1.4.5/go.mod h1:ibHel+/kbxn9x2407k1izTA1S81ku1z/DlgOW2QE0M4=
|
||||
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
|
||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||
github.com/guregu/null/v5 v5.0.0 h1:PRxjqyOekS11W+w/7Vfz6jgJE/BCwELWtgvOJzddimw=
|
||||
github.com/guregu/null/v5 v5.0.0/go.mod h1:SjupzNy+sCPtwQTKWhUCqjhVCO69hpsl2QsZrWHjlwU=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.7.1 h1:x7SYsPBYDkHDksogeSmZZ5xzThcTgRz++I5E+ePFUcs=
|
||||
github.com/jackc/pgx/v5 v5.7.1/go.mod h1:e7O26IywZZ+naJtWWos6i6fvWK+29etgITqrqHLfoZA=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/jaevor/go-nanoid v1.4.0 h1:mPz0oi3CrQyEtRxeRq927HHtZCJAAtZ7zdy7vOkrvWs=
|
||||
github.com/jaevor/go-nanoid v1.4.0/go.mod h1:GIpPtsvl3eSBsjjIEFQdzzgpi50+Bo1Luk+aYlbJzlc=
|
||||
github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
|
||||
github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
|
||||
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
||||
github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
|
||||
github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
|
||||
golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
|
||||
golang.org/x/exp v0.0.0-20241004190924-225e2abe05e6 h1:1wqE9dj9NpSm04INVsJhhEUzhuDVjbcyKH91sVyPATw=
|
||||
golang.org/x/exp v0.0.0-20241004190924-225e2abe05e6/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8=
|
||||
golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
|
||||
golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
|
||||
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
|
||||
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
|
||||
golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
|
||||
golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -0,0 +1,23 @@
|
||||
package logging
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
zlog "github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
func LogDebug(format string, args ...interface{}) {
|
||||
zlog.Debug().Msg(fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
func LogInfo(format string, args ...interface{}) {
|
||||
zlog.Info().Msg(fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
func LogWarn(format string, args ...interface{}) {
|
||||
zlog.Warn().Msg(fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
func LogError(format string, args ...interface{}) {
|
||||
zlog.Error().Err(fmt.Errorf(format, args...)).Msg("")
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"gemini-grc/config"
|
||||
"gemini-grc/gemini"
|
||||
"gemini-grc/logging"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
zlog "github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
func main() {
|
||||
config.CONFIG = *config.GetConfig()
|
||||
zerolog.TimeFieldFormat = zerolog.TimeFormatUnix
|
||||
zerolog.SetGlobalLevel(config.CONFIG.LogLevel)
|
||||
zlog.Logger = zlog.Output(zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: "[2006-01-02 15:04:05]"})
|
||||
if err := runApp(); err != nil {
|
||||
logging.LogError("Application error: %w", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func runApp() error {
|
||||
logging.LogInfo("Starting up. Press Ctrl+C to exit")
|
||||
sigs := make(chan os.Signal, 1)
|
||||
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
|
||||
|
||||
db := gemini.ConnectToDB()
|
||||
|
||||
// !!! DANGER !!!
|
||||
// Removes all rows and adds some seed URLs.
|
||||
// populateDB(db)
|
||||
|
||||
defer func(db *sqlx.DB) {
|
||||
err := db.Close()
|
||||
if err != nil {
|
||||
// TODO properly log & hangle error
|
||||
panic(err)
|
||||
}
|
||||
}(db)
|
||||
|
||||
go gemini.SpawnWorkers(config.CONFIG.NumOfWorkers, db)
|
||||
|
||||
<-sigs
|
||||
logging.LogInfo("Received SIGINT or SIGTERM signal, exiting")
|
||||
return nil
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
use std::fs::File;
|
||||
use std::io::{self, Read};
|
||||
|
||||
pub fn read_file_as_bytes(path: &str) -> io::Result<Vec<u8>> {
|
||||
let mut file = File::open(path)?;
|
||||
let mut buffer = Vec::new();
|
||||
file.read_to_end(&mut buffer)?;
|
||||
Ok(buffer)
|
||||
}
|
||||
@@ -1,63 +0,0 @@
|
||||
use std::fmt;
|
||||
use std::fmt::Debug;
|
||||
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use url::Url;
|
||||
|
||||
pub mod server;
|
||||
pub mod tls;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct GeminiUrl {
|
||||
pub scheme: String,
|
||||
pub hostname: String,
|
||||
pub port: u16,
|
||||
pub path: String,
|
||||
}
|
||||
|
||||
impl fmt::Display for GeminiUrl {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(
|
||||
f,
|
||||
"{}://{}:{}{}",
|
||||
self.scheme, self.hostname, self.port, self.path
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl GeminiUrl {
|
||||
/// Default scheme: "gemini://"
|
||||
/// Port: 1965
|
||||
/// Path: "/"
|
||||
pub fn new(input: &str) -> Result<GeminiUrl> {
|
||||
let input_with_scheme = if input.contains("://") {
|
||||
input.to_string()
|
||||
} else {
|
||||
format!("gemini://{}", input)
|
||||
};
|
||||
let parsed = Url::parse(&input_with_scheme)
|
||||
.with_context(|| format!("Invalid request URL {}", &input_with_scheme))?;
|
||||
if parsed.scheme() != "gemini" {
|
||||
return Err(anyhow!(
|
||||
"Invalid request (URL protocol should be gemini://)"
|
||||
));
|
||||
}
|
||||
let hostname = parsed
|
||||
.host_str()
|
||||
.ok_or_else(|| anyhow!("Invalid request (URL must have a host)"))?;
|
||||
let mut parsed = GeminiUrl {
|
||||
scheme: parsed.scheme().to_string(),
|
||||
hostname: hostname.to_string(),
|
||||
port: parsed.port().unwrap_or(1965),
|
||||
path: parsed.path().to_string(),
|
||||
};
|
||||
if parsed.path.is_empty() {
|
||||
parsed.path = String::from("/")
|
||||
};
|
||||
Ok(parsed)
|
||||
}
|
||||
|
||||
pub fn to_url(&self) -> Url {
|
||||
Url::parse(&self.to_string()).unwrap()
|
||||
}
|
||||
}
|
||||
@@ -1,233 +0,0 @@
|
||||
use std::io::{Read, Write};
|
||||
use std::net::{IpAddr, TcpListener, TcpStream};
|
||||
use std::path::PathBuf;
|
||||
use std::thread;
|
||||
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use log;
|
||||
use log::{error, info};
|
||||
use openssl::ssl::{SslAcceptor, SslStream};
|
||||
use path_clean::PathClean;
|
||||
|
||||
use crate::fs::read_file_as_bytes;
|
||||
use crate::gemini::{GeminiUrl, tls};
|
||||
use crate::nanoid::nanoid;
|
||||
use crate::time::now_unix_millis;
|
||||
|
||||
struct GeminiSession {
|
||||
pub ip: IpAddr,
|
||||
pub tls_id: Option<String>,
|
||||
pub stream: SslStream<TcpStream>,
|
||||
pub id: String,
|
||||
pub timestamp: u128,
|
||||
}
|
||||
|
||||
struct GeminiResponse {
|
||||
code: u8,
|
||||
mime_type: Option<String>,
|
||||
data: Option<Vec<u8>>,
|
||||
}
|
||||
|
||||
impl GeminiResponse {
|
||||
pub fn new(
|
||||
srv_root: &str,
|
||||
gemini_session: &GeminiSession,
|
||||
path: &String,
|
||||
) -> Result<GeminiResponse> {
|
||||
let filename = resolve_path(srv_root, path)?;
|
||||
if !filename.is_file() {
|
||||
return Ok(GeminiResponse {
|
||||
code: 51, // NOT FOUND
|
||||
mime_type: None,
|
||||
data: None,
|
||||
});
|
||||
}
|
||||
let mime_type = Self::guess_mime_type(PathBuf::from(&filename));
|
||||
if mime_type.is_none() {
|
||||
log_info(
|
||||
gemini_session,
|
||||
&format!("Cannot guess mime type {:?}", filename),
|
||||
);
|
||||
return Ok(GeminiResponse {
|
||||
code: 51,
|
||||
mime_type: None,
|
||||
data: None,
|
||||
});
|
||||
}
|
||||
log_info(gemini_session, &format!("Opening file {:?}", filename));
|
||||
let file_str = filename
|
||||
.to_str()
|
||||
.ok_or_else(|| anyhow!("Failed to convert path to string"))?;
|
||||
// let data = read_file_to_string(file_str)
|
||||
// .with_context(|| format!("Failed to read file {}", path))?;
|
||||
let data = read_file_as_bytes(file_str)
|
||||
.with_context(|| format!("Failed to read file {}", path))?;
|
||||
Ok(GeminiResponse {
|
||||
code: 20,
|
||||
mime_type,
|
||||
data: Some(data),
|
||||
})
|
||||
}
|
||||
|
||||
fn guess_mime_type(path: PathBuf) -> Option<String> {
|
||||
match path.extension() {
|
||||
Some(e) => match e.to_str().unwrap_or("") {
|
||||
"gmi" => Some(String::from("text/gemini")),
|
||||
"jpg" => Some(String::from("image/jpeg")),
|
||||
"jpeg" => Some(String::from("image/jpeg")),
|
||||
"png" => Some(String::from("image/png")),
|
||||
_ => None,
|
||||
},
|
||||
None => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_bytes(&self) -> Vec<u8> {
|
||||
let header = match &self.mime_type {
|
||||
Some(e) => format!("{}\t{}\r\n", self.code, e),
|
||||
None => format!("{}\r\n", self.code),
|
||||
};
|
||||
// Convert the header to bytes and prepare the response vector
|
||||
let mut response_bytes = header.into_bytes();
|
||||
|
||||
// Extend the response with the data if it exists
|
||||
if let Some(e) = &self.data {
|
||||
response_bytes.extend(e);
|
||||
}
|
||||
response_bytes
|
||||
}
|
||||
}
|
||||
|
||||
pub fn start_server(
|
||||
bind_address: &str,
|
||||
key_file: &str,
|
||||
cert_file: &str,
|
||||
srv_root: &str,
|
||||
) -> Result<()> {
|
||||
let acceptor = tls::create_tls_acceptor(key_file, cert_file).context("TLS error")?;
|
||||
let listener = TcpListener::bind(bind_address).context("TCP bind error")?;
|
||||
info!("Gemini server listening to {}", &bind_address);
|
||||
|
||||
for stream in listener.incoming() {
|
||||
// The stream will be dropped at the end of
|
||||
// each loop iteration, which will close
|
||||
// the stream automatically. No housekeeping
|
||||
// necessary.
|
||||
// Each stream is handled in a new thread.
|
||||
// Async is under consideration for now.
|
||||
match stream {
|
||||
Err(e) => {
|
||||
error!("Failed to accept connection: {:?}", e);
|
||||
}
|
||||
Ok(stream) => {
|
||||
let acceptor = acceptor.clone();
|
||||
let srv_root = srv_root.to_string();
|
||||
|
||||
let handle = thread::spawn(move || {
|
||||
if let Err(e) = initiate_connection(&acceptor, stream, &srv_root) {
|
||||
error!("Connection handling error: {:?}", e);
|
||||
}
|
||||
});
|
||||
if let Err(e) = handle.join() {
|
||||
error!("Thread join error while handling stream: {:?}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn log_info(session: &GeminiSession, message: &str) {
|
||||
info!("{} {} {}", session.id, session.ip.to_string(), message);
|
||||
}
|
||||
|
||||
/// Resolve final absolute path
|
||||
/// of the file to serve.
|
||||
/// Final path must always begin with
|
||||
/// root path (no escape outside root!).
|
||||
/// If final path is a directory, append
|
||||
/// index.gmi.
|
||||
fn resolve_path(root_path: &str, input: &String) -> Result<PathBuf> {
|
||||
let path1 = PathBuf::from(root_path);
|
||||
let mut path2 = PathBuf::from(input);
|
||||
if path2.is_absolute() {
|
||||
path2 = PathBuf::from(path2.strip_prefix("/")?);
|
||||
}
|
||||
let mut final_path = path1.join(path2).clean();
|
||||
if !final_path.starts_with(path1) {
|
||||
return Err(anyhow!("Invalid path {:?} -> {:?}", input, final_path));
|
||||
}
|
||||
if final_path.is_dir() {
|
||||
final_path = final_path.join("index.gmi");
|
||||
}
|
||||
Ok(final_path)
|
||||
}
|
||||
|
||||
fn handle_gemini_session(srv_root: &str, gemini_session: &mut GeminiSession) -> Result<()> {
|
||||
let mut buffer = [0; 1024];
|
||||
let bytes_read = gemini_session
|
||||
.stream
|
||||
.read(&mut buffer)
|
||||
.context("Invalid request (failed to read input stream)")?;
|
||||
let received = String::from_utf8(Vec::from(&buffer[..bytes_read]))
|
||||
.context("Invalid request (failed to convert input to UTF8)")?;
|
||||
let request_data = received.trim();
|
||||
let url = GeminiUrl::new(request_data)?;
|
||||
log_info(
|
||||
gemini_session,
|
||||
&format!(
|
||||
"New request from {}{} {}",
|
||||
gemini_session.ip,
|
||||
match &gemini_session.tls_id {
|
||||
Some(x) => format!(" TLS digest {}", x),
|
||||
None => String::from(""),
|
||||
},
|
||||
request_data
|
||||
),
|
||||
);
|
||||
let response = GeminiResponse::new(srv_root, gemini_session, &url.path)?;
|
||||
let response_raw = response.get_bytes();
|
||||
log_info(
|
||||
gemini_session,
|
||||
&format!(
|
||||
"Reply Code {} Response length {} bytes",
|
||||
response.code,
|
||||
response_raw.len()
|
||||
),
|
||||
);
|
||||
gemini_session
|
||||
.stream
|
||||
.write_all(&response_raw)
|
||||
.context("Failed to write response to stream")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn initiate_connection(acceptor: &SslAcceptor, stream: TcpStream, srv_root: &str) -> Result<()> {
|
||||
let timestamp1 = now_unix_millis();
|
||||
|
||||
let ip = stream
|
||||
.peer_addr()
|
||||
.context("Failed to get peer IP address from TCP stream")?
|
||||
.ip();
|
||||
|
||||
let stream = acceptor
|
||||
.accept(stream)
|
||||
.with_context(|| format!("{} Failed to establish SSL connection", ip))?;
|
||||
|
||||
let mut gemini_session = GeminiSession {
|
||||
ip,
|
||||
tls_id: tls::get_peer_certificate_digest(&stream),
|
||||
stream,
|
||||
id: nanoid(),
|
||||
timestamp: now_unix_millis(),
|
||||
};
|
||||
|
||||
handle_gemini_session(srv_root, &mut gemini_session)?;
|
||||
|
||||
log_info(
|
||||
&gemini_session,
|
||||
&format!("Finished ({}ms)", now_unix_millis() - timestamp1),
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,70 +0,0 @@
|
||||
use std::net::TcpStream;
|
||||
|
||||
use hex::ToHex;
|
||||
use openssl::hash::MessageDigest;
|
||||
use openssl::ssl::{SslAcceptor, SslFiletype, SslMethod, SslStream, SslVerifyMode};
|
||||
|
||||
pub fn create_tls_acceptor(key_filename: &str, cert_filename: &str) -> anyhow::Result<SslAcceptor> {
|
||||
let mut acceptor = SslAcceptor::mozilla_intermediate(SslMethod::tls())?;
|
||||
acceptor.set_private_key_file(key_filename, SslFiletype::PEM)?;
|
||||
acceptor.set_certificate_chain_file(cert_filename)?;
|
||||
|
||||
// Enable peer verification
|
||||
acceptor.set_verify(SslVerifyMode::PEER);
|
||||
|
||||
// Custom verification callback
|
||||
acceptor.set_verify_callback(SslVerifyMode::PEER, |preverify_ok, x509_ctx| {
|
||||
if preverify_ok {
|
||||
return true;
|
||||
}
|
||||
// Check if it's a self-signed certificate
|
||||
let cert = match x509_ctx.current_cert() {
|
||||
Some(cert) => cert,
|
||||
None => return false,
|
||||
};
|
||||
|
||||
let issuer_name = cert.issuer_name().to_der();
|
||||
let subject_name = cert.subject_name().to_der();
|
||||
|
||||
let is_self_signed = match (issuer_name, subject_name) {
|
||||
(Ok(issuer), Ok(subject)) => issuer == subject,
|
||||
_ => return false,
|
||||
};
|
||||
|
||||
if is_self_signed {
|
||||
let public_key = match cert.public_key() {
|
||||
Ok(key) => key,
|
||||
Err(_) => return false,
|
||||
};
|
||||
|
||||
if cert.verify(&public_key).is_ok() {
|
||||
let now = match openssl::asn1::Asn1Time::days_from_now(0) {
|
||||
Ok(time) => time,
|
||||
Err(_) => return false,
|
||||
};
|
||||
|
||||
if cert.not_before() <= now && cert.not_after() >= now {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
});
|
||||
// Initialize the session ID context
|
||||
let context = &crate::nanoid::nanoid().into_bytes();
|
||||
acceptor.set_session_id_context(context)?;
|
||||
acceptor.check_private_key()?;
|
||||
Ok(acceptor.build())
|
||||
}
|
||||
|
||||
pub fn get_peer_certificate_digest(stream: &SslStream<TcpStream>) -> Option<String> {
|
||||
if let Some(cert) = stream.ssl().peer_certificate() {
|
||||
if let Ok(digest) = cert.digest(MessageDigest::ripemd160()) {
|
||||
Some(digest.encode_hex::<String>())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
pub mod fs;
|
||||
pub mod gemini;
|
||||
pub mod nanoid;
|
||||
pub mod time;
|
||||
-61
@@ -1,61 +0,0 @@
|
||||
use std::env;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use log::error;
|
||||
|
||||
use gemini_grs::gemini;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct EnvConfig {
|
||||
pub hostname: String,
|
||||
pub key_path: String,
|
||||
pub cert_path: String,
|
||||
pub server_root: String,
|
||||
}
|
||||
|
||||
impl EnvConfig {
|
||||
pub fn from_env() -> Result<Self> {
|
||||
let hostname = env::var("GEMINI_SERVER_HOSTNAME").unwrap_or("127.0.0.1:1965".to_string());
|
||||
let key_path = env::var("GEMINI_SERVER_TLS_KEY_FILENAME").with_context(|| {
|
||||
"Missing environment variable GEMINI_SERVER_TLS_KEY_FILENAME".to_string()
|
||||
})?;
|
||||
let cert_path = env::var("GEMINI_SERVER_TLS_CERT_FILENAME").with_context(|| {
|
||||
"Missing environment variable GEMINI_SERVER_TLS_CERT_FILENAME".to_string()
|
||||
})?;
|
||||
let server_root = env::var("GEMINI_SERVER_ROOT_DIRECTORY").with_context(|| {
|
||||
"Missing environment variable GEMINI_SERVER_ROOT_DIRECTORY".to_string()
|
||||
})?;
|
||||
Ok(Self {
|
||||
hostname,
|
||||
key_path,
|
||||
cert_path,
|
||||
server_root,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let exit_code = run();
|
||||
std::process::exit(exit_code);
|
||||
}
|
||||
|
||||
fn run() -> i32 {
|
||||
env_logger::init();
|
||||
let env_config = match EnvConfig::from_env() {
|
||||
Err(e) => {
|
||||
error!("{:#?}", e);
|
||||
return 1;
|
||||
}
|
||||
Ok(x) => x,
|
||||
};
|
||||
if let Err(e) = gemini::server::start_server(
|
||||
&env_config.hostname,
|
||||
&env_config.key_path,
|
||||
&env_config.cert_path,
|
||||
&env_config.server_root,
|
||||
) {
|
||||
error!("{:#?}", e);
|
||||
return 1;
|
||||
}
|
||||
0
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
use nanoid::nanoid;
|
||||
|
||||
pub fn nanoid() -> String {
|
||||
let alphabet = [
|
||||
'1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'a', 'b', 'c', 'd', 'e', 'f',
|
||||
'g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z',
|
||||
'A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T',
|
||||
'U','V','W','X','Y','Z',
|
||||
];
|
||||
nanoid!(16, &alphabet)
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
// returns the time since the epoch in milliseconds
|
||||
pub fn now_unix_millis() -> u128 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_millis()
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
package uid
|
||||
|
||||
import (
|
||||
nanoid "github.com/jaevor/go-nanoid"
|
||||
)
|
||||
|
||||
func UID() string {
|
||||
// Missing o,O and l
|
||||
uid, err := nanoid.CustomASCII("abcdefghijkmnpqrstuvwxyzABCDEFGHIJKLMNPQRSTUVWXYZ0123456789", 20)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return uid()
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime/debug"
|
||||
)
|
||||
|
||||
func PrintStackAndPanic(err error) {
|
||||
fmt.Printf("Error %s Stack trace:\n%s", err, debug.Stack())
|
||||
panic("PANIC")
|
||||
}
|
||||
Reference in New Issue
Block a user