#!/usr/bin/env python3
"""
reach-scan.py — "Can agents reach you?" reachability probe.

Point it at any site or API. It asks the one question a conformance grade does
not: does the SAME request get the SAME answer from the HTTP clients real
agents actually use? A door can look flawless to curl and a browser and be shut
to Python's urllib or an AI crawler, because a WAF/CDN rule refuses that client
BEFORE any terms are served. Coppice found this class of defect on its own
infrastructure twice and across 40 top-graded strangers' endpoints once; this
is that instrument, generalised off x402.

  usage:  reach-scan.py <url-or-domain> [--json] [--post]

Not x402-specific. No payment, no account, no key. Read-only GETs by default
(--post also probes an http->https POST redirect for body loss). Stdlib +
curl + node only. Every request is one this box could make of a public URL;
nothing is fuzzed, flooded, or authenticated.
"""
import sys, json, subprocess, urllib.request, urllib.error, urllib.parse, ssl, socket, time, re

TIMEOUT = 15

# Real client stacks (the honest probe: different TLS/HTTP fingerprints, not
# just a spoofed header) plus UA variants that name the agents a WAF rule most
# often singles out. curl and node are invoked as real processes; urllib is a
# real stdlib request. The UA-only rows ride curl's stack but carry the agent's
# name, which is what a User-Agent WAF rule keys on.
UA_CURL    = 'curl/8.5.0'
UA_NODE    = None  # node/undici default
UexA_BROWS = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36'
UA_LIBWWW  = 'libwww-perl/6.67'
UA_PYREQ   = 'python-requests/2.31.0'
UA_GPTBOT  = 'Mozilla/5.0 (compatible; GPTBot/1.1; +https://openai.com/gptbot)'
UA_CLAUDE  = 'Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)'
UA_GOOGLE  = 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'


def norm(u):
    u = u.strip()
    if not re.match(r'^https?://', u):
        u = 'https://' + u
    return u


def curl_get(url, ua, method='GET', body=None):
    a = ['curl', '-sS', '-o', '/dev/null', '-m', str(TIMEOUT),
         '-w', '%{http_code} %{num_redirects} %{url_effective} %{size_download}',
         '-X', method, '-L', '--max-redirs', '5']
    if ua:
        a += ['-A', ua]
    if body is not None:
        a += ['-H', 'Content-Type: application/json', '-d', body]
    try:
        r = subprocess.run(a + [url], capture_output=True, text=True, timeout=TIMEOUT + 5)
        out = r.stdout.strip().split()
        if len(out) >= 4:
            return {'status': int(out[0]), 'redirects': int(out[1]),
                    'final': out[2], 'bytes': int(out[3])}
        return {'status': 0, 'err': (r.stderr or r.stdout)[:80]}
    except Exception as e:
        return {'status': 0, 'err': str(e)[:80]}


def curl_no_redirect(url, ua, method='GET', body=None):
    """One hop only: reveals the redirect status code (301/302 drop POST body,
    307/308 preserve it) and whether http even upgrades to https."""
    a = ['curl', '-sS', '-o', '/dev/null', '-m', str(TIMEOUT),
         '-w', '%{http_code} %{redirect_url}', '-X', method]
    if ua:
        a += ['-A', ua]
    if body is not None:
        a += ['-H', 'Content-Type: application/json', '-d', body]
    try:
        r = subprocess.run(a + [url], capture_output=True, text=True, timeout=TIMEOUT + 5)
        p = r.stdout.strip().split(None, 1)
        return {'status': int(p[0]) if p and p[0].isdigit() else 0,
                'location': p[1] if len(p) > 1 else ''}
    except Exception as e:
        return {'status': 0, 'err': str(e)[:80]}


def urllib_get(url, method='GET', body=None):
    data = body.encode() if body is not None else None
    headers = {'Content-Type': 'application/json'} if body is not None else {}
    req = urllib.request.Request(url, data=data, method=method, headers=headers)
    ctx = ssl.create_default_context()
    try:
        x = urllib.request.urlopen(req, timeout=TIMEOUT, context=ctx)
        return {'status': x.status, 'final': x.geturl(), 'bytes': len(x.read(1 << 20))}
    except urllib.error.HTTPError as e:
        return {'status': e.code, 'err': 'HTTPError'}
    except ssl.SSLCertVerificationError as e:
        return {'status': 0, 'err': 'TLS:' + str(e)[:60]}
    except Exception as e:
        return {'status': 0, 'err': str(e)[:80]}


def node_get(url, method='GET', body=None):
    b = ("body:%r,headers:{'content-type':'application/json'}," % body) if body is not None else ''
    js = ("fetch(%r,{method:%r,%sredirect:'follow'})"
          ".then(async r=>{const t=await r.text();"
          "console.log(JSON.stringify({status:r.status,final:r.url,bytes:t.length}));})"
          ".catch(e=>console.log(JSON.stringify({status:0,err:String(e.cause?e.cause.code||e.cause:e).slice(0,80)})));"
          % (url, method, b))
    try:
        r = subprocess.run(['node', '-e', js], capture_output=True, text=True, timeout=TIMEOUT + 10)
        return json.loads(r.stdout.strip() or '{"status":0,"err":"no-output"}')
    except Exception as e:
        return {'status': 0, 'err': str(e)[:80]}


def fetch_text(url):
    """Returns (ok, body). ok means HTTP 200; body is the response text."""
    try:
        r = subprocess.run(['curl', '-sS', '-m', str(TIMEOUT), '-A', UA_CURL,
                            '-w', '\n%{http_code}', url],
                           capture_output=True, text=True, timeout=TIMEOUT + 5)
        out = r.stdout
        code = out.rsplit('\n', 1)[-1].strip()
        body = out[:out.rfind('\n')] if '\n' in out else out
        return (r.returncode == 0 and code == '200'), body
    except Exception:
        return False, ''


def looks_like_html(body):
    head = body.lstrip()[:200].lower()
    return head.startswith('<!doctype') or head.startswith('<html') or '<head' in head or '<body' in head


def scan(url, do_post=False):
    url = norm(url)
    parsed = urllib.parse.urlparse(url)
    origin = f'{parsed.scheme}://{parsed.netloc}'
    R = {'target': url, 'origin': origin, 'checked_at': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}

    # --- multi-client GET on the target ---
    clients = [
        ('curl',            lambda: curl_get(url, UA_CURL)),
        ('node/undici',     lambda: node_get(url)),
        ('python-urllib',   lambda: urllib_get(url)),
        ('python-requests', lambda: curl_get(url, UA_PYREQ)),
        ('libwww-perl',     lambda: curl_get(url, UA_LIBWWW)),
        ('browser(Chrome)', lambda: curl_get(url, UexA_BROWS)),
        ('GPTBot',          lambda: curl_get(url, UA_GPTBOT)),
        ('ClaudeBot',       lambda: curl_get(url, UA_CLAUDE)),
        ('Googlebot',       lambda: curl_get(url, UA_GOOGLE)),
    ]
    rows = {}
    for name, fn in clients:
        rows[name] = fn()
    R['clients'] = rows

    # A client "reaches" the door if it gets a non-network answer with a body,
    # and that answer is not a hard refusal (401/403/429/451) or a 5xx.
    def reached(r):
        s = r.get('status', 0)
        if s == 0:
            return False
        if s in (401, 403, 429, 451) or s >= 500:
            return False
        return True
    reach = {n: reached(r) for n, r in rows.items()}
    R['reach'] = reach
    ok = [n for n, v in reach.items() if v]
    bad = [n for n, v in reach.items() if not v]

    # --- robots.txt: does it disallow AI agents / does the served file differ ---
    findings = []
    got_robots, robots = fetch_text(origin + '/robots.txt')
    R['robots_present'] = got_robots and 'Disallow' in robots
    ai_blocked = []
    if got_robots:
        blocks = re.split(r'(?im)^user-agent:', robots)
        for blk in blocks:
            names = re.findall(r'^\s*([^\n]+)', blk)
            agent = names[0].strip().lower() if names else ''
            disallows = re.findall(r'(?im)^\s*disallow:\s*(\S+)', blk)
            if any(d == '/' for d in disallows):
                if any(a in agent for a in ('gptbot', 'claudebot', 'ccbot', 'google-extended',
                                            'anthropic', 'perplexity', 'bytespider', 'amazonbot')):
                    ai_blocked.append(agent.split()[0] if agent else '?')
                elif agent == '*':
                    ai_blocked.append('* (all)')
    R['robots_disallow_ai'] = sorted(set(ai_blocked))

    # --- llms.txt presence (a 200 catch-all HTML page for every path is NOT
    # an llms.txt; require a real, non-HTML, non-empty body) ---
    got_llms, llms = fetch_text(origin + '/llms.txt')
    R['llms_txt'] = bool(got_llms and llms.strip() and not looks_like_html(llms))

    # --- http -> https upgrade + POST-safety of the redirect ---
    http_origin = 'http://' + parsed.netloc
    hop = curl_no_redirect(http_origin + (parsed.path or '/'), UA_CURL)
    R['http_redirect'] = hop
    redirect_drops_post = False
    if 300 <= hop.get('status', 0) < 400 and hop['status'] in (301, 302):
        redirect_drops_post = True

    # --- POST body loss (optional, only when the caller asks) ---
    if do_post:
        p = curl_no_redirect(http_origin + (parsed.path or '/'), UA_CURL, method='POST', body='{}')
        R['http_post_redirect'] = p
        if 300 <= p.get('status', 0) < 400 and p['status'] in (301, 302):
            redirect_drops_post = True

    # ---------- findings ----------
    if bad and ok:
        findings.append({
            'severity': 'high',
            'code': 'client-asymmetry',
            'title': 'Some HTTP clients are refused where others get through',
            'detail': f"reached by {', '.join(ok)}; refused/failed for {', '.join(bad)}"
                      f" (statuses: " + ', '.join(f'{n}={rows[n].get('status')}' for n in bad) + ")"
        })
    if not ok:
        findings.append({'severity': 'high', 'code': 'unreachable',
                         'title': 'No probed client could read this URL',
                         'detail': 'every client returned a refusal, error, or network failure'})
    if R['robots_disallow_ai']:
        findings.append({'severity': 'medium', 'code': 'robots-blocks-ai',
                         'title': 'robots.txt tells AI agents to stay out',
                         'detail': 'Disallow: / for ' + ', '.join(R['robots_disallow_ai'])})
    if redirect_drops_post:
        findings.append({'severity': 'medium', 'code': 'redirect-drops-post',
                         'title': 'The http->https redirect is a 301/302, which turns a paying POST into a GET',
                         'detail': f"http hop returned {R['http_redirect'].get('status')}; "
                                   '307/308 would preserve the method and body'})
    if not R['llms_txt']:
        findings.append({'severity': 'low', 'code': 'no-llms-txt',
                         'title': 'No /llms.txt', 'detail': 'agents have no machine-readable guide to your site'})
    R['findings'] = findings

    # ---------- the one true sentence ----------
    if not ok:
        s = f"Not one of nine common HTTP clients could read {origin}: it is closed to agents."
    elif bad:
        worst = [n for n in bad if n in ('python-urllib', 'python-requests', 'libwww-perl',
                                         'GPTBot', 'ClaudeBot')]
        who = worst or bad
        s = (f"{origin} answers curl and a browser, but {', '.join(who)} "
             f"{'get' if len(who) > 1 else 'gets'} refused before any content is served — "
             "a rule someone switched on is shadowing your site from those agents.")
    elif R['robots_disallow_ai']:
        s = f"{origin} reaches every client I probed, but your robots.txt tells {', '.join(R['robots_disallow_ai'])} to stay out."
    elif redirect_drops_post:
        s = f"{origin} is reachable, but its http->https redirect would turn a paying agent's POST into a GET and lose the body."
    else:
        s = f"{origin} answered all nine clients I probed the same way — it reads clean to agents."
    R['sentence'] = s
    R['grade'] = 'FAIL' if (not ok or (bad and ok)) else ('WARN' if findings else 'PASS')
    return R


def human(R):
    print(f"\n  reach-scan  {R['target']}   ({R['checked_at']})")
    print(f"  {'-'*68}")
    for n, r in R['clients'].items():
        mark = 'OK ' if R['reach'][n] else 'XX '
        s = r.get('status', 0)
        extra = r.get('err', '') or (f"{r.get('bytes','?')}b" if R['reach'][n] else '')
        print(f"    {mark} {n:18} status={s:<5} {extra}")
    print(f"  {'-'*68}")
    print(f"    robots blocks AI : {R['robots_disallow_ai'] or 'no'}")
    print(f"    /llms.txt        : {'present' if R['llms_txt'] else 'MISSING'}")
    print(f"    http->https hop  : {R['http_redirect'].get('status')} {R['http_redirect'].get('location','')[:40]}")
    print(f"  {'-'*68}")
    for f in R['findings']:
        print(f"    [{f['severity'].upper():6}] {f['title']}")
        print(f"             {f['detail']}")
    print(f"  {'-'*68}")
    print(f"    GRADE: {R['grade']}")
    print(f"    {R['sentence']}\n")


if __name__ == '__main__':
    args = [a for a in sys.argv[1:] if not a.startswith('--')]
    flags = set(a for a in sys.argv[1:] if a.startswith('--'))
    if not args:
        print(__doc__)
        sys.exit(1)
    R = scan(args[0], do_post='--post' in flags)
    if '--json' in flags:
        print(json.dumps(R, indent=2))
    else:
        human(R)