54 lines
1.3 KiB
Go
54 lines
1.3 KiB
Go
package gemini
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
)
|
|
|
|
func ParseRobotsTxt(content string) *RobotsData {
|
|
data := &RobotsData{}
|
|
var currentUserAgent string
|
|
|
|
for _, line := range strings.Split(content, "\n") {
|
|
line = strings.TrimSpace(line)
|
|
if line == "" || strings.HasPrefix(line, "#") {
|
|
continue
|
|
}
|
|
|
|
parts := strings.SplitN(line, ":", 2)
|
|
if len(parts) != 2 {
|
|
continue
|
|
}
|
|
|
|
directive := strings.TrimSpace(strings.ToLower(parts[0]))
|
|
value := strings.TrimSpace(parts[1])
|
|
|
|
switch directive {
|
|
case "user-agent":
|
|
currentUserAgent = value
|
|
case "allow":
|
|
if value != "" {
|
|
data.Rules = append(data.Rules, RobotRule{
|
|
UserAgent: currentUserAgent,
|
|
Allow: true,
|
|
Path: value,
|
|
})
|
|
}
|
|
case "disallow":
|
|
if value != "" {
|
|
data.Rules = append(data.Rules, RobotRule{
|
|
UserAgent: currentUserAgent,
|
|
Allow: false,
|
|
Path: value,
|
|
})
|
|
}
|
|
case "crawl-delay":
|
|
if delay, err := strconv.Atoi(value); err == nil {
|
|
data.CrawlDelay = delay
|
|
}
|
|
}
|
|
}
|
|
|
|
return data
|
|
}
|