package security import ( "bufio" "log" "net/http" "os" "path/filepath" "strings" "sync" "time" ) type BotBlocker struct { mu sync.RWMutex badUAs map[string]struct{} blocklistURLs []string cacheFile string } func NewBotBlocker(blocklistPath string) *BotBlocker { bb := &BotBlocker{ badUAs: make(map[string]struct{}), cacheFile: ".cache/bad-user-agents.txt", } if blocklistPath != "" { // #nosec G304 if file, err := os.Open(filepath.Clean(blocklistPath)); err == nil { scanner := bufio.NewScanner(file) for scanner.Scan() { url := strings.TrimSpace(scanner.Text()) if url != "" && !strings.HasPrefix(url, "#") { bb.blocklistURLs = append(bb.blocklistURLs, url) } } _ = file.Close() } } // Load existing cache if available bb.loadFromCache() // If we have URLs, start background updater if len(bb.blocklistURLs) > 0 { go bb.startUpdater() } return bb } func (bb *BotBlocker) loadFromCache() { if _, err := os.Stat(bb.cacheFile); err == nil { if file, err := os.Open(bb.cacheFile); err == nil { bb.mu.Lock() scanner := bufio.NewScanner(file) for scanner.Scan() { ua := strings.TrimSpace(scanner.Text()) if ua != "" { bb.badUAs[strings.ToLower(ua)] = struct{}{} } } bb.mu.Unlock() _ = file.Close() } } } func (bb *BotBlocker) startUpdater() { // Immediate fetch on start bb.fetchAndRefresh() ticker := time.NewTicker(24 * time.Hour) for range ticker.C { bb.fetchAndRefresh() } } func (bb *BotBlocker) fetchAndRefresh() { newUAs := make(map[string]struct{}) client := &http.Client{Timeout: 30 * time.Second} for _, url := range bb.blocklistURLs { resp, err := client.Get(url) if err != nil { log.Printf("Error fetching bot blocklist from %s: %v", url, err) continue } scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { ua := strings.TrimSpace(scanner.Text()) if ua != "" && !strings.HasPrefix(ua, "#") { newUAs[strings.ToLower(ua)] = struct{}{} } } _ = resp.Body.Close() } if len(newUAs) > 0 { bb.mu.Lock() bb.badUAs = newUAs bb.mu.Unlock() // Save to cache _ = os.MkdirAll(".cache", 0750) if file, err := os.Create(bb.cacheFile); err == nil { writer := bufio.NewWriter(file) for ua := range newUAs { _, _ = writer.WriteString(ua + "\n") } _ = writer.Flush() _ = file.Close() } log.Printf("Bot blocklist updated with %d entries", len(newUAs)) } } func (bb *BotBlocker) IsBot(ua string) bool { if ua == "" { return false } uaLower := strings.ToLower(ua) // Check static list first (fast) for _, bot := range BotUserAgents { if strings.Contains(uaLower, bot) { return true } } // Check dynamic list bb.mu.RLock() defer bb.mu.RUnlock() // Some lists contain partial strings, some contain exact matches. // We'll do a partial match check for each entry in our dynamic list. // This might be slow if the list is huge. // Optimization: check exact match first, then partial if needed. if _, ok := bb.badUAs[uaLower]; ok { return true } for badUA := range bb.badUAs { if strings.Contains(uaLower, badUA) { return true } } return false }