# -*- coding: utf-8 -*-
"""
tier_gate.py — a claimed source tier must be supported by the source cited.

The tier ladder is the index's central evidentiary claim: 1 = court or government
record, 2 = named independent outlet (national or regional), 3 = local or trade, 4 = single-source or partisan.
Round-six review found 24 rows claiming Tier 1 or Tier 2 whose only cited URL is
Wikipedia — including the row carrying the index's most reputationally loaded
characterisation, claiming Tier 1, "Court/govt record", sourced to a wiki article.

Wikipedia is neither a court record nor an independent outlet. It is a tertiary source
that may be sourced to good ones, which is why the master coding sometimes writes
"(per Wikipedia, sourced)" — an honest note that does not change what was actually
cited. The index already knows how to fix this: one row was explicitly "SOURCING
UPGRADED OFF WIKIPEDIA" to two named outlets. It was not applied consistently.

This gate refuses a Tier 1 or Tier 2 claim whose citations are wiki-only. The
remedy is to cite the underlying source, or to state the tier the citation supports.

Run: python3 scripts/tier_gate.py
"""
import os, re, sys

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
import phase2  # noqa: E402

# PHASE A, 2026-08-30 — the 30 rows that once claimed Tier 1/2 on a wiki are RESOLVED, so
# this set is empty. It is kept (empty) because the roster's tier_verified column, the
# figures count, and the G2 basis all read it, and an empty set says "nothing outstanding"
# in the one place they look. How the 30 were resolved, under the hybrid decision:
#   * Costa and Perry (the two carrying adverse findings) were VERIFIED off Wikipedia to
#     named contemporary outlets and court records, in phase2.py, by hand.
#   * five rows whose only non-tertiary citation is the member's own official House
#     biography — a .gov government record — were reclassified to a Government-record tier,
#     legitimate for NON-ADVERSE biography and enforced below to be non-adverse.
#   * the remaining rows, cited to nothing but a wiki, were honestly DOWNGRADED to a
#     tertiary tier. They no longer claim a tier their citation cannot show.
# A row that reintroduces the defect — any Tier 1/2 claim its citations do not support —
# fails the build. This is now enforced by CLAIM TYPE, not by a blanket wiki test, because
# a media lawyer pointed out the blanket test could not tell a bio fact (which a government
# record can support) from an adverse finding (which it cannot).
TIER_UNVERIFIED = set()

TERTIARY = ("wikipedia.org", "ballotpedia.org")
NEEDS_PRIMARY = ("1 - ", "2 - ")
GOV_BIO_TIER = "1 - Government record"   # a .gov/court record, non-adverse claims only
# What is NOT an adverse finding. A red-teamer defeated the old approach — a closed set of
# nine exact strings {Conviction, Charge filed, ...} — by coding "Ethics violation" (a
# synonym in none of them), so is_adverse() returned False and the finding shipped on the
# subject's own House bio as if it were benign biography. The rule is now DEFAULT-DENY: any
# non-empty legal value that is not one of these exact benign markers is an adverse finding
# and must be sourced as one. A synonym cannot slip the net because there is no net of
# adverse words to slip — everything that is not explicitly "nothing found" is a finding.
BENIGN_LEGAL = {"", "none found", "none", "n/a", "no matter surfaced"}
# The independent outlets and public records this index has ACCEPTED as sourcing for an
# adverse finding about a named person. It is a positive allowlist, on purpose: the old gate
# called any host that was not a wiki, not a .gov, and did not contain the subject's surname
# a "named independent outlet", so a red-teamer sourced a conviction to the subject's own PR
# firm (checkmatestrategies.com) and it counted. An adverse finding is a reputational and
# legal act; the sources that may carry one are named here, and adding one is a deliberate
# edit. Captured from the sources already backing the index's adverse rows.
OUTLET_ALLOW = {
    "aol.com", "ca.news.yahoo.com", "capradio.org", "carolinajournal.com", "cnn.com",
    "engadget.com", "lasvegassun.com", "lehighvalleypublicmedia.org", "myfox28columbus.com",
    "newspapers.com", "pressreader.com", "santafenewmexican.com", "spotlightpa.org",
    "texasborderbusiness.com", "texastribune.org", "thebusinessjournal.com", "thehill.com",
    "thelantern.com", "washingtonpost.com", "wcpo.com", "wfmz.com", "wisconsinwatch.org",
    "forbes.com",
    # added 2026-09-02: the two outlets that actually report the Buckhout conviction,
    # verified against the articles (Daily Advance 2024-02-07; WRAL 2024-02-23)
    "dailyadvance.com", "wral.com",
}
# A citation is an absolute http(s) URL. Splitting the field on ";" and calling every
# fragment a citation is how "; see also" -- and, in one live row, the "06/23/" left over
# from splitting a date out of a path -- came to count as corroborating sources.
URL = re.compile(r"https?://[^\s;,]+")
GOVISH = re.compile(r"(\.gov$|\.gov/|pacer|courtlistener|courts\.|\.uscourts\.)", re.I)


def _host(url):
    return re.sub(r"^https?://(www\.)?", "", url).split("/")[0].lower()


def citations(rec):
    return URL.findall(rec.get("url") or "")


def is_adverse(rec):
    """DEFAULT-DENY: anything that is not an explicit benign marker is an adverse finding."""
    return (rec.get("legal") or "").strip().lower() not in BENIGN_LEGAL


def adverse_ok(name, rec):
    """An adverse finding is adequately sourced ONLY by a court/enforcement government record
    (a .gov that is not the subject's own office) or a citation to an accepted outlet. The
    subject's own bio, a campaign or PR site, a wiki, or an unlisted host cannot carry one."""
    for u in citations(rec):
        if classify(name, u) == "GOV":                    # court / DOJ / enforcement .gov
            return True
        if any(h == _host(u) or _host(u).endswith("." + h) for h in OUTLET_ALLOW):
            return True
    return False


def self_host(name, url):
    """The subject's own campaign site or own House office — the surname is in the host.

    Deliberately narrow (surname in host) so it names what it catches rather than guessing.
    Found when the tightened URL rule exposed Kevin Lincoln's row: two wiki links plus
    'kevinlincolnforcongress.com/meet-kevin', which the old loose split had counted as a
    corroborating source. It is the subject describing himself."""
    host = re.sub(r"^https?://(www\.)?", "", url).split("/")[0].lower()
    sur = re.sub(r"[^a-z]", "", name.split()[-1].lower())
    return len(sur) > 3 and sur in host


def classify(name, url):
    """TERTIARY | GOV | OWN_GOV | SELF | NAMED — what kind of source a citation is."""
    host = re.sub(r"^https?://(www\.)?", "", url).split("/")[0].lower()
    if any(t in host for t in TERTIARY):
        return "TERTIARY"
    own = self_host(name, url)
    if GOVISH.search(url):
        return "OWN_GOV" if own else "GOV"   # a court/DOJ .gov vs the member's own office
    return "SELF" if own else "NAMED"


def coding():
    for v in vars(phase2).values():
        if isinstance(v, dict) and v and all(isinstance(x, dict) for x in v.values()):
            if any("tier" in x for x in v.values()):
                return v
    raise SystemExit("tier gate: could not locate the master coding table")


def check_row(name, rec):
    """The claim a row's tier makes, and whether its citations can carry it.

    Returns a failure reason string, or None if the row is sound. The rule is by CLAIM
    TYPE: a biographical fact may rest on a government record (including the member's own
    official House bio); an ADVERSE finding may not — it needs a court record or a named
    independent outlet, never the subject's own page or a wiki."""
    tier = (rec.get("tier") or "").strip()

    # ADVERSE findings are enforced FIRST and at EVERY tier, by the SOURCE, not the tier
    # label. Two red-teamers walked past the old tier-keyed logic: one coded a Conviction on
    # a Tier-3 (wiki-only) row, which the gate never examined because it only checked Tier
    # 1/2; another coded it at Tier 2 on a PR-firm host the classifier waved through. A
    # finding against a named person needs a court/enforcement record or an accepted outlet
    # wherever it sits on the ladder — a wiki, a campaign site, or the subject's own bio
    # never carries one.
    if is_adverse(rec):
        if not citations(rec):
            return "carries an adverse finding (%r) with NO CITATION AT ALL" % (
                (rec.get("legal") or "").strip())
        if not adverse_ok(name, rec):
            srcs = ", ".join(sorted({_host(u) for u in citations(rec)}))
            return ("carries an adverse finding (%r) not supported by a court/enforcement "
                    "record or an accepted independent outlet — its only sources are: %s. "
                    "An adverse finding needs a court record or a named outlet on the "
                    "accepted list, at any tier." % ((rec.get("legal") or "").strip(), srcs))

    if not tier.startswith(NEEDS_PRIMARY):
        return None                       # Tier 3 or lower claims no primary source
    cats = [classify(name, u) for u in citations(rec)]
    if not cats:
        return "claims %s with NO CITATION AT ALL" % tier[:1]

    if tier.startswith(GOV_BIO_TIER):
        if is_adverse(rec):
            return ("carries an adverse finding on a Government-record (own House bio) "
                    "tier — a subject's own bio can never establish a finding against them")
        if "GOV" not in cats and "OWN_GOV" not in cats:
            return "claims a Government-record tier but cites no .gov source"
        return None

    if tier.startswith("1 - "):           # Court/govt record — typically an adverse matter
        # A court/government-record tier is supported ONLY by a court/government record. The
        # old test accepted a named outlet here too, which made tiers 1 and 2 the same tier
        # with two labels — the final pre-publication reviewer found two adverse rows labelled
        # "court record" whose citations were CNN, the Post and Wisconsin Watch. Those rows
        # now carry the outlet tier their citations support; a row that wants this label
        # cites the record (a .gov that is not the subject's own office, PACER, a court site).
        if "GOV" in cats:
            return None
        return ("claims a court/government record but cites no court or government record — "
                "only " + ", ".join(sorted(set(cats))).lower()
                + ". An outlet's report of a ruling supports the outlet tier, not this one.")

    if tier.startswith("2 - "):           # Major outlet
        if "NAMED" in cats:
            return None
        return "claims a major outlet but cites only " + ", ".join(sorted(set(cats))).lower()
    return None


def main():
    rows = coding()
    bad = [(n, (r.get("tier") or "").strip(), why)
           for n, r in rows.items() if (why := check_row(n, r))]
    if bad:
        print("TIER GATE FAILED — a tier claim its citations do not support:\n")
        for n, t, why in sorted(bad):
            print(f"  {n:28s} {why}")
            print(f"  {'':28s} (tier: {t})")
        print(f"\n{len(bad)} rows. Cite the underlying source, or lower the tier to what")
        print("the citation actually supports. A ladder nobody enforces is not a ladder.")
        raise SystemExit(1)
    stale = TIER_UNVERIFIED - set(rows)
    if stale:
        raise SystemExit(f"tier gate: TIER_UNVERIFIED names rows that no longer exist: {stale}")
    n1 = sum(1 for r in rows.values() if (r.get("tier") or "").startswith(NEEDS_PRIMARY))
    ng = sum(1 for r in rows.values() if (r.get("tier") or "").startswith(GOV_BIO_TIER))
    print(f"    tier gate: {n1} Tier 1/2 claims, each supported by its citation "
          f"({ng} on an official House-bio government record, non-adverse); "
          f"{len(TIER_UNVERIFIED)} rows outstanding")


if __name__ == "__main__":
    main()