# =============================================================================
# CFM WebDetector — Challenge Exclude Rules
# =============================================================================
#
# Purpose
#   Suppress *challenge enforcement* for known-good / verified crawlers so they
#   don't get caught in challenge loops by accident.
#
# Activation
#   This file is loaded automatically when it exists at the path configured by
#   CHALLENGE_EXCLUDE_FILE in detectors.conf (default
#   /etc/cfm/webdetector_challenge_exclude.txt). There is no on/off knob —
#   delete the file (or empty it) to disable, edit it to update. Changes are
#   picked up on the next detectors reload.
#
# File format
#   One rule per line:    key=value; key=value; ...
#   Blank lines and lines starting with '#' or ';' are ignored.
#   Keys/values are case-insensitive.
#
# Keys
#   ua=PATTERN        User-Agent glob (matches against the request's UA)
#   asn=AS12345       ASN glob (numeric form auto-prefixed with "as"; e.g.
#                     asn=15169 == asn=as15169). Globs allowed: asn=as1516?
#   ptr=PATTERN       Reverse-DNS hostname glob (e.g. *.googlebot.com)
#   host=PATTERN      Request vhost glob (e.g. *.example.com)
#   verify_fcrdns=1   After PTR matches, require forward-confirmed rDNS — the
#                     PTR hostname must resolve back to the request IP. This is
#                     the spoof-proof check; recommended for all PTR rules.
#   mode=all|any      Combination mode (default: all).
#                       all → every specified field must match (safer)
#                       any → at least one field matching is enough (riskier)
#   action=skip | skip_vhost_only
#                       skip            → suppress ALL challenges for this hit
#                       skip_vhost_only → only suppress vhost-wide rules
#                                          (CHALLENGE_VHOST, CHALLENGE_SUSPICIOUS_VHOST_SCORE)
#
# Glob semantics
#   '*' matches any run of characters (including the empty string). It is NOT
#       a path separator — it freely crosses '/' inside UA strings.
#   '?' matches exactly one character.
#   A pattern with no '*' or '?' is treated as a substring match.
#
# Safety guidance
#   - UA alone is trivially spoofable. Anyone can set User-Agent: AhrefsBot.
#     For verified crawlers, always combine with ptr=...; verify_fcrdns=1.
#   - ASN alone is broad (clouds host attackers too). Prefer asn + ua, or PTR.
#   - "mode=any" is rarely what you want; default "all" is safer.
#   - Test new rules with a small action first (skip_vhost_only) before
#     promoting to action=skip.
#
# Worked examples
#   # Verified crawler — PTR + FCrDNS (gold standard).
#   ptr=*.googlebot.com; verify_fcrdns=1; action=skip
#
#   # UA + PTR + FCrDNS — narrows match to the bot you expect on that PTR.
#   ua=*ahrefs*; ptr=*.ahrefs.com; verify_fcrdns=1; action=skip
#
#   # ASN + UA combo (no FCrDNS available) — semi-strict.
#   asn=as32934; ua=*facebookexternalhit*; action=skip
#
#   # Trust a partner ASN, but ONLY for vhost-wide panic rules.
#   asn=as64500; action=skip_vhost_only
#
#   # Free pass for a specific vhost (e.g. status page).
#   host=status.example.com; action=skip
#
# Operator workflow
#   tail -f /var/log/cfm/detectors.log | grep "challenge exclude"
#       → confirms parsed rules at load and matches at request time.
# =============================================================================

# -----------------------------------------------------------------------------
# 1) Googlebot — verified via FCrDNS.
#    Fake "Googlebot" UAs from random IPs do NOT pass.
# -----------------------------------------------------------------------------
ptr=*.googlebot.com; verify_fcrdns=1; action=skip
ptr=*.google.com;    verify_fcrdns=1; action=skip

# -----------------------------------------------------------------------------
# 2) Bingbot — verified via FCrDNS (PTRs under search.msn.com).
# -----------------------------------------------------------------------------
ptr=*.search.msn.com; verify_fcrdns=1; action=skip

# -----------------------------------------------------------------------------
# 3) Meta / Facebook crawler.
#    Meta doesn't publish a stable FCrDNS scheme for all crawlers, so we
#    combine their ASN with a UA substring. Adjust the UA glob to match what
#    you actually see in your logs (meta*, facebookexternalhit*, etc).
# -----------------------------------------------------------------------------
asn=as32934; ua=*facebookexternalhit*; action=skip
asn=as32934; ua=*meta*;                action=skip

# -----------------------------------------------------------------------------
# 4) Trusted partner ASNs — suppress only vhost-wide panic rules.
#    Per-IP / per-path rules still apply. Replace ASxxxx with your partner ASN.
# -----------------------------------------------------------------------------
#asn=asXXXX; action=skip_vhost_only

# -----------------------------------------------------------------------------
# 5) Skroutz crawler (ASN-only).
# -----------------------------------------------------------------------------
asn=as202042; action=skip

# -----------------------------------------------------------------------------
# 6) Apple (e.g. WKWebView / Apple Bot ASN).
# -----------------------------------------------------------------------------
asn=as714; action=skip

# -----------------------------------------------------------------------------
# 7) AhrefsBot — verified via FCrDNS (PTR ends in .ahrefs.com).
#    Spoofed "AhrefsBot" UAs from non-Ahrefs IPs are still challenged.
# -----------------------------------------------------------------------------
ptr=*.ahrefs.com; verify_fcrdns=1; action=skip

# -----------------------------------------------------------------------------
# 8) SemrushBot — verified via FCrDNS.
#    Semrush publishes PTRs under .semrush.com.
# -----------------------------------------------------------------------------
ptr=*.semrush.com; verify_fcrdns=1; action=skip

# -----------------------------------------------------------------------------
# 9) VPN by Google / Chrome private prefetch-proxy egress.
#    Why: this is SHARED egress. One /24 (e.g. 162.120.188.0/24, PTRs under
#    *.fetch.tunnel.googlezip.net, AS15169) carries MANY real users, so the
#    per-subnet behavioural heuristic CHALLENGE_SUBNET reads "many IPs + many
#    unique paths from one /24" as a scanner and challenges the whole subnet.
#    The prefetch proxy fetches on behalf of users and does NOT run JS, so it
#    can never solve the challenge — it just loops (issued-but-never-solved),
#    and the request UA is the end-user's own browser (no bot-UA to match on).
#    This is challenge-suppression ONLY: the WAF rule engine (SQLi/RCE/upload/
#    webshell -> 403) stays fully armed for these IPs, so a real exploit is
#    still blocked; we only stop the pointless behavioural challenge loop.
#    Two rules, belt-and-suspenders:
#      - PTR + FCrDNS (gold standard): matches only if the googlezip PTR
#        forward-confirms back to the request IP. Harmless if it doesn't — it
#        simply won't match and the second rule carries it.
#      - ASN + PTR (robust): the ASN comes from BGP routing (not client-
#        spoofable) and a *.googlezip.net PTR needs Google's reverse DNS, so the
#        pair means "a real Google IP" even without FCrDNS. mode=all (default)
#        requires BOTH to match.
# -----------------------------------------------------------------------------
ptr=*.googlezip.net; verify_fcrdns=1; action=skip
asn=as15169; ptr=*.googlezip.net;     action=skip
