package gemini

import (
	"fmt"
	"strings"
)

func ParseRobotsTxt(content string) *RobotsData {
    data := &RobotsData{}
    var currentUserAgent string

    for _, line := range strings.Split(content, "\n") {
        line = strings.TrimSpace(line)
        if line == "" || strings.HasPrefix(line, "#") {
            continue
        }

        parts := strings.SplitN(line, ":", 2)
        if len(parts) != 2 {
            continue
        }

        directive := strings.TrimSpace(strings.ToLower(parts[0]))
        value := strings.TrimSpace(parts[1])

        switch directive {
        case "user-agent":
            currentUserAgent = value
        case "allow":
            if value != "" {
                data.Rules = append(data.Rules, RobotRule{
                    UserAgent: currentUserAgent,
                    Allow:     true,
                    Path:      value,
                })
            }
        case "disallow":
            if value != "" {
                data.Rules = append(data.Rules, RobotRule{
                    UserAgent: currentUserAgent,
                    Allow:     false,
                    Path:      value,
                })
            }
        case "crawl-delay":
            if delay, err := strconv.Atoi(value); err == nil {
                data.CrawlDelay = delay
            }
        }
    }

    return data
}