import json, csv, re, statistics as st, sys
sys.path.insert(0, '/home/claude/cvai/scripts')

MASTER = '/home/claude/cvai/style_corpus_all.csv'
FIELDS = ['candidate','register','subtype','date','source_url','text','capture_date','word_count']

# ---- merge any new round corpora (idempotent on candidate+text prefix) ----
NEW = ['/tmp/corpus_me02.json', '/tmp/corpus_ia01.json', '/tmp/corpus_va01.json']
rows = list(csv.DictReader(open(MASTER)))
seen = {(r['candidate'], r['text'][:60]) for r in rows}
added = 0
with open(MASTER, 'a', newline='') as f:
    w = csv.DictWriter(f, fieldnames=FIELDS)
    for p in NEW:
        try: samples = json.load(open(p))['samples']
        except Exception: continue
        for s in samples:
            if (s['candidate'], s['text'][:60]) in seen: continue
            seen.add((s['candidate'], s['text'][:60]))
            s['capture_date'] = '2026-08-22'
            s['word_count'] = len(s['text'].split())
            w.writerow(s); added += 1
print(f'merged {added} new samples')
rows = list(csv.DictReader(open(MASTER)))
print('total samples:', len(rows))

ATTACK = {'corrupt','corruption','radical','extreme','extremist','extremists','lie','lies','lying','liar',
          'disgrace','disgraceful','failed','failure','weak','crooked','monster','invasion','swamp','elites',
          'elite','weaponization','weaponizing','hoax','attacking','attacks','rigged','vile','grotesque',
          'disaster','disastrous','destroying','cowards','self-serving','fraud','gutted','gut','shafted'}
POLICY = {'bill','bills','act','vote','voted','voting','fund','funding','funds','veterans','border','water',
          'healthcare','health','jobs','tribal','district','housing','agriculture','education','energy',
          'drought','wildfire','appropriations','committee','legislation','hearing','medicaid','medicare',
          'snap','taxes','tax','airport','research','infrastructure','tariffs','security'}

selfnames = {
 'Crane':['Rep. Crane','Rep. Eli Crane','Congressman Crane'],
 'Nez':['Jonathan Nez','President Nez',' Nez '],
 'Kaptur':['Congresswoman Kaptur','Congresswoman Marcy Kaptur','Marcy Kaptur',' Kaptur '],
 'Merrin':['Derek Merrin','Representative Merrin',' Merrin '],
 'Van Orden':['Derrick Van Orden','Van Orden'],
 'Cooke':['Rebecca Cooke',' Cooke '],
 'Lawler':['Mike Lawler','Rep. Lawler',' Lawler '],
 'Conley':['Cait Conley',' Conley '],
 'Titus':['Congresswoman Dina Titus','Congresswoman Titus','Dina Titus','Rep. Dina Titus',' Titus '],
 'Buck':['Carrie Buck','Senator Carrie Buck',' Carrie ',' Buck '],
 'Costa':['Congressman Jim Costa','Jim Costa','Rep. Jim Costa','Congressman Costa',' Costa '],
 'Kirkland':['Kyle Kirkland',' Kirkland ',' Kyle '],
 'Perry':['Congressman Scott Perry','Scott Perry','Congressman Perry',' Perry '],
 'Stelson':['Janelle Stelson',' Janelle ',' Stelson '],
 'LePage':['Paul LePage','Governor LePage',' LePage ',' Paul '],
 'Dunlap':['Matt Dunlap',' Dunlap '],
 'Miller-Meeks':['Congresswoman Mariannette Miller-Meeks','Mariannette Miller-Meeks','Miller-Meeks'],
 'Bohannan':['Christina Bohannan',' Bohannan ',' Christina '],
 'Wittman':['Congressman Rob Wittman','Congressman Wittman','Rob Wittman',' Wittman '],
 'Taylor':['Shannon Taylor',' Taylor '],
}

def feats(text, self_names):
    words = re.findall(r"[\w'@#-]+", text)
    n = len(words)
    if n == 0: return None
    sents = [s for s in re.split(r'[.!?]+', text) if s.strip()]
    slens = [len(re.findall(r"[A-Za-z']+", s)) for s in sents]
    lower = [w.lower().strip("'-") for w in words if re.match(r"[A-Za-z]", w)]
    fps = sum(1 for w in lower if w in ("i","i'm","i've","i'll","me","my","mine","myself"))
    fpp = sum(1 for w in lower if w in ("we","we're","we've","our","ours","us"))
    third = sum(text.count(nm) for nm in self_names)
    excl = text.count('!')
    atk = sum(1 for w in lower if w in ATTACK)
    pol = sum(1 for w in lower if w in POLICY)
    syll = sum(max(1, len(re.findall(r'[aeiouy]+', w.lower()))) for w in lower)
    fk = 0.39*(n/max(1,len(sents))) + 11.8*(syll/max(1,len(lower))) - 15.59 if lower else 0
    return dict(n=n, msl=(st.mean(slens) if slens else 0), fps=100*fps/n, fpp=100*fpp/n,
                third=100*third/n, excl=100*excl/n, atk=100*atk/n, pol=100*pol/n, fk=fk)

# v0.3: four cells — release (written body), quote (WRITTEN attributed), spoken (interview), post
cells = {}
for r in rows:
    if r['register'] == 'official':
        sub = {'quote':'quote', 'spoken':'spoken'}.get(r['subtype'], 'release')
    else:
        sub = 'post'
    cells.setdefault((r['candidate'], sub), []).append(r)

def agg(rs, cand):
    pooled = '\n'.join(r['text'] for r in rs)
    f = feats(pooled, selfnames[cand]); f['k'] = len(rs)
    return f

CANDS = ['Crane','Nez','Kaptur','Merrin','Van Orden','Cooke','Lawler','Conley','Titus','Buck',
         'Costa','Kirkland','Perry','Stelson','LePage','Dunlap','Miller-Meeks','Bohannan','Wittman','Taylor']
AGE_TEN = {'Crane':(46,3.8),'Nez':(51,0),'Kaptur':(80,43.8),'Merrin':(40,0),'Van Orden':(56,3.8),
 'Cooke':(38,0),'Lawler':(40,3.8),'Conley':(41,0),'Titus':(76,15.8),'Buck':(55,0),
 'Costa':(74,21.8),'Kirkland':(64,0),'Perry':(64,13.8),'Stelson':(66,0),'LePage':(78,0),
 'Dunlap':(61,0),'Miller-Meeks':(71,5.8),'Bohannan':(55,0),'Wittman':(67,18.9),'Taylor':(58,0)}

res = {}
print(f"\n{'cell':24s} k  words  msl   FK    I/100 we/100 3rd   excl  atk   pol")
for cand in CANDS:
    for reg in ['release','quote','spoken','post']:
        rs = cells.get((cand, reg))
        if not rs: continue
        f = agg(rs, cand); res[(cand, reg)] = f
        print(f"{cand+' '+reg:24s} {f['k']:2d} {f['n']:5d}  {f['msl']:4.1f}  {f['fk']:4.1f}  {f['fps']:5.2f} {f['fpp']:5.2f}  {f['third']:4.2f}  {f['excl']:4.2f}  {f['atk']:4.2f}  {f['pol']:4.2f}")

# ---- protocol minimums: WRITTEN official (release+quote) >=8 samples/600w; social >=8/300w ----
MIN_K, MIN_W_OFF, MIN_W_SOC = 8, 600, 300
status = {}
print("\n=== Protocol minimums (v0.3: written-official excludes spoken) ===")
for cand in CANDS:
    off = [r for r in rows if r['candidate']==cand and r['register']=='official' and r['subtype']!='spoken']
    spo = [r for r in rows if r['candidate']==cand and r['subtype']=='spoken']
    soc = [r for r in rows if r['candidate']==cand and r['register']=='social']
    ow, sw = sum(int(r['word_count']) for r in off), sum(int(r['word_count']) for r in soc)
    status[(cand,'official')] = 'OK' if (len(off)>=MIN_K and ow>=MIN_W_OFF) else 'INSUFFICIENT'
    status[(cand,'social')]   = 'OK' if (len(soc)>=MIN_K and sw>=MIN_W_SOC) else 'INSUFFICIENT'
    print(f"  {cand:13s} written-off k={len(off):2d}/{ow:5d}w {status[(cand,'official')]:12s} | social k={len(soc):2d}/{sw:5d}w {status[(cand,'social')]:12s} | spoken k={len(spo)}")

def ok(c): return status[(c,'official')]=='OK' and status[(c,'social')]=='OK'

print("\n=== 1. Three-voice architecture (third-person-self /100w: release | written quote | post) ===")
for cand in CANDS:
    p = []
    for reg in ['release','quote','post']:
        f = res.get((cand,reg))
        p.append(f"{reg}={f['third']:.2f}" if f else f"{reg}=n/a")
    print(f"  {cand:13s} " + '  '.join(p))

print("\n=== 2. Register gap: WRITTEN official quote -> social (FK grade) ===")
gaps = {}
for cand in CANDS:
    a, b = res.get((cand,'quote')), res.get((cand,'post'))
    if a and b and ok(cand):
        gaps[cand] = a['fk']-b['fk']
        age, ten = AGE_TEN[cand]
        print(f"  {cand:13s} age={str(age):>4s} ten={ten:5.1f}  quote {a['fk']:5.1f} -> social {b['fk']:5.1f}  gap {a['fk']-b['fk']:+5.1f}")
    elif a and b:
        print(f"  {cand:13s} [insufficient cell - withheld]")

print("\n=== 3. Quote-to-post I-density distance (same-writer instrument, written quotes only) ===")
for cand in CANDS:
    a, b = res.get((cand,'quote')), res.get((cand,'post'))
    if a and b and ok(cand):
        print(f"  {cand:13s} quote I={a['fps']:5.2f}  post I={b['fps']:5.2f}  distance {abs(a['fps']-b['fps']):.2f}")

print("\n=== 4. SPOKEN vs WRITTEN check (does spoken read simpler? the v0.3 contamination test) ===")
for cand in CANDS:
    q, s = res.get((cand,'quote')), res.get((cand,'spoken'))
    if q and s:
        print(f"  {cand:13s} written-quote FK={q['fk']:5.1f}  spoken FK={s['fk']:5.1f}  diff {q['fk']-s['fk']:+5.1f}")

print("\n=== 5. Age / tenure vs register gap ===")
try:
    from scipy import stats as sps
    pts = [(AGE_TEN[c][0], AGE_TEN[c][1], g, c) for c,g in gaps.items() if AGE_TEN[c][0] is not None]
    ages = [p[0] for p in pts]; tens = [p[1] for p in pts]; gs = [p[2] for p in pts]
    ra,pa = sps.pearsonr(ages,gs); rt,pt = sps.pearsonr(tens,gs)
    rsa,psa = sps.spearmanr(ages,gs)
    print(f"  n={len(pts)}")
    print(f"  AGE    vs gap: Pearson r={ra:+.2f} (p={pa:.3f})  Spearman {rsa:+.2f} (p={psa:.3f})")
    print(f"  TENURE vs gap: Pearson r={rt:+.2f} (p={pt:.3f})")
    # partial: age controlling for tenure, and tenure controlling for age
    import numpy as np
    def partial(x,y,z):
        x,y,z = np.array(x,float),np.array(y,float),np.array(z,float)
        rxy=np.corrcoef(x,y)[0,1]; rxz=np.corrcoef(x,z)[0,1]; ryz=np.corrcoef(y,z)[0,1]
        return (rxy-rxz*ryz)/np.sqrt((1-rxz**2)*(1-ryz**2))
    print(f"  AGE vs gap, controlling TENURE:  r={partial(ages,gs,tens):+.2f}")
    print(f"  TENURE vs gap, controlling AGE:  r={partial(tens,gs,ages):+.2f}")
    print("\n  zero-tenure candidates only (isolates AGE):")
    zt = [(a,g,c) for a,t,g,c in pts if t==0]
    for a,g,c in sorted(zt): print(f"    {c:13s} age={a:3d}  gap={g:+5.1f}")
    if len(zt)>2:
        rz,pz = sps.pearsonr([z[0] for z in zt],[z[1] for z in zt])
        print(f"    AGE vs gap among zero-tenure (n={len(zt)}): r={rz:+.2f} (p={pz:.3f})")
except Exception as e:
    print('  stats unavailable:', e)

print("\n=== 6. Median social post length (corpus fact, always reportable) ===")
for cand in CANDS:
    soc = [int(r['word_count']) for r in rows if r['candidate']==cand and r['register']=='social']
    if soc: print(f"  {cand:13s} median {st.median(soc):5.1f}  n={len(soc)}")