Add robots.txt checking

Still needs periodic cache refresh
2024-10-23 14:24:10 +03:00
parent c49a69728a
commit ef3f009709
12 changed files with 225 additions and 156 deletions
--- a/util/util.go
+++ b/util/util.go
@@ -0,0 +1,11 @@
+package util
+
+import (
+	"fmt"
+	"runtime/debug"
+)
+
+func PrintStackAndPanic(err error) {
+	fmt.Printf("Error %s Stack trace:\n%s", err, debug.Stack())
+	panic("PANIC")
+}