#!/usr/bin/env python3 """Frozen 308-assessment analysis. No AI calls; never changes an assessment. Run with numpy/scipy installed. All interval resampling uses seed 20260904. Public scores, moves, section weights and final ledgers are cross-checked. """ from __future__ import annotations import csv import hashlib import importlib.util import json import math import subprocess from collections import Counter, defaultdict from pathlib import Path import numpy as np from scipy.stats import rankdata, spearmanr ROOT = Path(__file__).resolve().parents[3] OUT = Path(__file__).resolve().parent SEED, DRAWS = 20260904, 20000 DIMS = dict(logicalCoherence=.25, evidenceWarrant=.20, responsiveness=.20, relevanceBurden=.15, precisionClarity=.10, calibrationCharity=.10) LABELS = ['Logical coherence', 'Evidence and warrant', 'Responsiveness', 'Relevance and burden', 'Precision and clarity', 'Confidence and fairness'] SOURCE_COMMIT = 'e65bc3c0b6' # Retain the 169 explicitly reviewed September 1 classifications, not an # open-ended rule that PRO means theism. Every added debate has a decision. ADDITIONS = { 227: ('pro', 'Religion, culture & meaning', 'Religious purpose versus secular meaning'), 228: ('con', 'Scripture, revelation & doctrine', 'Religious reliability defense versus skeptical history'), 229: ('', '', 'A historical Jesus does not imply a divine Jesus'), 230: ('', '', 'Idealism versus naturalism does not itself establish a God contrast'), 231: ('', '', 'A nonphysical soul does not itself imply God; consistent with exclusions 190/201'), 232: ('pro', 'General theism & naturalism', 'Finite theism versus naturalism'), 233: ('pro', 'General theism & naturalism', 'Christianity as a rational worldview versus naturalistic atheism'), 234: ('', '', 'Two nonstandard mind-first views; no clean religious-versus-skeptical contrast'), 235: ('', '', 'Soul existence alone is religion-adjacent, not a theism contrast'), 236: ('', '', 'Moral realism versus anti-realism is not theism versus atheism'), 237: ('pro', 'Resurrection & historical evidence', 'Resurrection defense versus skeptical historical method'), 238: ('pro', 'General theism & naturalism', 'Christian rationality versus skeptical challenge'), 239: ('pro', 'Religion, culture & meaning', 'Religious narrative necessity versus secular alternatives'), 240: ('', '', 'Both sides defend cosmic purpose; alternatives include a limited designer'), 241: ('pro', 'General theism & naturalism', 'Mind-like spiritual God versus evidential skepticism'), 242: ('', '', 'Two religious readings of Jesus; consistent with Borg/Crossan exclusions'), 243: ('pro', 'General theism & naturalism', 'Christian truth versus evidential challenge'), 244: ('pro', 'Cosmology, science & design', 'Necessary divine mind versus insufficient identification'), 245: ('pro', 'Mind, reason & logic', 'Explicit nontraditional theism versus skeptical objections'), 246: ('pro', 'Scripture, revelation & doctrine', 'Classical-theist doctrine versus agnostic criticism; broad-set only'), 247: ('con', 'Morality & moral foundations', 'God-independent goodness versus foundation in Christ'), 248: ('pro', 'Scripture, revelation & doctrine', 'Classical-theist explanation versus agnostic criticism; broad-set only'), 249: ('', '', 'Stance-independent morality is not equivalent to theism'), 250: ('pro', 'Mind, reason & logic', 'Theistic versus naturalistic explanation of logic'), 251: ('con', 'Evil, suffering & hiddenness', 'Evil-God challenge versus divine-goodness defense'), 252: ('pro', 'Evil, suffering & hiddenness', 'Christian permission-of-evil defense versus suffering challenge'), 253: ('pro', 'Evil, suffering & hiddenness', 'Christian plausibility versus hiddenness and hell challenge'), 254: ('', '', 'Multi-speaker team assessment; excluded from comparable position analysis'), 255: ('pro', 'Resurrection & historical evidence', 'Christian resurrection case versus skeptical historical account'), 256: ('con', 'Religion, culture & meaning', 'Defense of Christianity against an atheist social-harm challenge; broad set only'), 257: ('', '', 'Science versus philosophy is not a religious-versus-skeptical contrast'), 258: ('pro', 'Scripture, revelation & doctrine', 'Biblical reliability defense versus skeptical history; broad set only'), 259: ('', '', 'Idealism versus illusionism does not itself establish a God contrast'), 260: ('pro', 'General theism & naturalism', 'Jewish theism versus atheism'), 261: ('pro', 'General theism & naturalism', 'Personal theism versus atheism'), 262: ('', '', 'Two naturalistic accounts of morality'), 263: ('pro', 'General theism & naturalism', 'Broad good-God case versus skeptical objections'), 264: ('', '', 'Spiritual nondualism and consciousness claims do not provide a sufficiently explicit God contrast'), 265: ('pro', 'Cosmology, science & design', 'Theistic fine-tuning versus naturalistic and probabilistic objections'), 266: ('con', 'Mind, reason & logic', 'Theistic justification versus a skeptical prior-possibility requirement'), 267: ('pro', 'General theism & naturalism', 'Christian theism versus atheism across truth, goodness and beauty'), 268: ('pro', 'Religion, culture & meaning', 'Religious partnership with science versus naturalistic criticism; broad set only'), 269: ('pro', 'Cosmology, science & design', 'First-cause/agency defense versus cosmological skepticism; motion concerns cosmic origins, not a complete religious doctrine'), 270: ('pro', 'General theism & naturalism', 'Christian-God defense versus atheism'), 271: ('pro', 'General theism & naturalism', 'Christian-God defense versus skeptical naturalism'), 272: ('pro', 'General theism & naturalism', 'Theistic cumulative case versus atheism'), 273: ('pro', 'General theism & naturalism', 'God-existence case versus skeptical naturalism'), 274: ('pro', 'General theism & naturalism', 'Biblical-God and transcendental defense versus atheism'), 275: ('pro', 'General theism & naturalism', 'God-existence case versus skeptical naturalism'), 276: ('pro', 'Scripture, revelation & doctrine', 'Reasonable divine revelation versus atheist challenge'), 277: ('con', 'Morality & moral foundations', 'Christian moral foundations versus secular independence'), 278: ('pro', 'General theism & naturalism', 'Christianity as reasonable versus skeptical naturalism'), 279: ('pro', 'General theism & naturalism', 'God-existence defense versus atheism'), 280: ('pro', 'General theism & naturalism', 'God-existence defense versus atheism'), 281: ('pro', 'Resurrection & historical evidence', 'Resurrection defense versus evidential skepticism'), 282: ('pro', 'General theism & naturalism', 'Christian-God defense versus atheism'), 283: ('pro', 'General theism & naturalism', 'God-existence defense versus atheism'), 284: ('pro', 'General theism & naturalism', 'God-existence defense versus skeptical naturalism'), 285: ('pro', 'General theism & naturalism', 'Biblical-God defense versus atheism'), 286: ('pro', 'Scripture, revelation & doctrine', 'Jesus prophecy argument versus evidential skepticism'), 287: ('pro', 'General theism & naturalism', 'God-existence case versus atheism'), 288: ('con', 'Resurrection & historical evidence', 'Credibility of Jesus miracles versus skeptical challenge'), 289: ('pro', 'Morality & moral foundations', 'Divine moral grounding versus non-theistic moral realism'), 290: ('con', 'Evil, suffering & hiddenness', 'Theism under evidential suffering challenge'), 291: ('pro', 'General theism & naturalism', 'God-existence defense versus atheism'), 292: ('pro', 'Morality & moral foundations', 'God-grounded morality and purpose versus secular alternatives'), 293: ('pro', 'Resurrection & historical evidence', 'Martyrdom as support for resurrection versus historical skepticism'), 294: ('', '', 'Two Christian interpretations of conquest; not a religious-versus-skeptical contrast'), 295: ('pro', 'Resurrection & historical evidence', 'Apostolic sincerity defense versus skeptical scope objection; broad historical set only, not proof of resurrection'), 296: ('con', 'Religion, culture & meaning', 'Christian social contribution versus secular humanism; broad set only'), 297: ('pro', 'Resurrection & historical evidence', 'Resurrection case versus skeptical historical alternatives'), 298: ('pro', 'General theism & naturalism', 'God-existence defense versus atheism'), 299: ('con', 'Evil, suffering & hiddenness', 'Loving-God defense versus hiddenness challenge'), 300: ('', '', 'Free will versus determinism does not itself establish a God contrast'), 301: ('', '', 'Christian resurrection versus an expressly Quranic survival/ascension alternative; interfaith contrast'), 302: ('pro', 'Scripture, revelation & doctrine', 'Christian worship-worthiness versus skeptical moral criticism; broad set only'), 303: ('pro', 'General theism & naturalism', 'God-existence defense versus atheism'), 304: ('pro', 'Evil, suffering & hiddenness', 'Christian explanation of suffering versus skeptical challenge'), 305: ('pro', 'Resurrection & historical evidence', 'Resurrection defense versus skeptical historical account'), 306: ('pro', 'Cosmology, science & design', 'Intelligent-origin explanation versus naturalistic method; creator/design-only, not full personal theism'), 307: ('con', 'Scripture, revelation & doctrine', 'Biblical textual reliability defense versus skeptical history; broad set only'), 308: ('con', 'Evil, suffering & hiddenness', 'Theism under religious-disagreement and hiddenness challenge'), } TOPICS = { 'Religion, culture & meaning': [3,6,9,15,16,21,46,62,89,101,106,124,145,171,172,207,214,216,217,218], 'Scripture, revelation & doctrine': [7,25,39,42,50,53,63,66,90,93,102,120,142,157,176,177,178,215,220], 'Mind, reason & logic': [5,12,49,67,68,70,72,76,80,91,99,100,104,108,114,143,147,197], 'Evil, suffering & hiddenness': [1,11,27,48,74,98,103,113,116,118,119,122,123,128,209,210,211], 'Morality & moral foundations': [10,13,20,24,47,54,57,58,61,97,127,148,151,155,156,160,162,164,198,204], 'Cosmology, science & design': [17,22,33,43,51,55,59,65,75,83,85,92,107,117,121,126,139,144,168,169,191,206,223], 'General theism & naturalism': [2,8,14,26,28,29,30,36,64,77,79,81,86,88,94,109,112,140,141,152,161,163,165,166,167,170,175,194,195,196,200,202,203,205,219,222], 'Resurrection & historical evidence': [31,37,52,60,69,78,87,130,136,137,138,150,158,179,181,212], } NARROW_EXCLUDE = {7,13,20,24,25,42,47,54,57,61,90,97,160,176,177,178,228,246,248,258,295,302,307} def dump(name, obj): (OUT/name).write_text(json.dumps(obj, indent=2, ensure_ascii=False, allow_nan=False)+'\n') def csvout(name, rows): with (OUT/name).open('w', newline='', encoding='utf-8') as f: w=csv.DictWriter(f, fieldnames=list(rows[0]),lineterminator='\n'); w.writeheader(); w.writerows(rows) def mean(x): return float(np.mean(x)) def rnd(x): return math.floor(x+.5) def interval(x): return [float(v) for v in np.quantile(x,[.025,.975])] def move_score(dimensions): # Python 3.12's compensated sum changes some half-point edge cases. # Match the published JavaScript left-to-right floating-point reducer. total=0.0 for k,w in DIMS.items(): total+=dimensions[k]*w return rnd(total) def boots(x, offset=0): x=np.asarray(x,dtype=float); rng=np.random.default_rng(SEED+offset) return np.concatenate([rng.choice(x,(1000,len(x)),replace=True).mean(axis=1) for _ in range(DRAWS//1000)]) def stats(x): x=np.asarray(x,dtype=float) return dict(n=len(x), mean=mean(x), median=float(np.median(x)), sd=float(x.std(ddof=1)) if len(x)>1 else 0, ci=interval(boots(x)), positive=int(sum(x>0)), zero=int(sum(x==0)), negative=int(sum(x<0))) def precision(f): if f['propositionRecoverability']=='failed': return 35 if f['termStability']=='materially-unstable' or f['scopeStability']=='materially-unstable' or f['qualificationExplicitness']=='materially-misleading': return 60 if f['propositionRecoverability']=='partial' or f['termStability']=='partly-unstable' or f['scopeStability']=='partly-unstable' or f['qualificationExplicitness']=='missing': return 75 return 85 if f['qualificationExplicitness']=='implicit' else 95 def calibration(f): if f['warrantFit'] in ['radically-overstated','materially-overstated','slightly-overstated']: return dict(zip(['radically-overstated','materially-overstated','slightly-overstated'],[35,60,75]))[f['warrantFit']] return 95 if f['qualificationStatus'] in ['explicit','not-needed'] and f['uncertaintyAcknowledged'] in ['yes','not-needed'] else 85 def public(): code=""" import {publishedDebates} from './src/data/debates.js'; import {avatarsForSpeakerText} from './src/data/interlocutors.js'; console.log(JSON.stringify(publishedDebates.map(d=>({...d, canonical:Object.fromEntries(['pro','con'].map(s=>[s,avatarsForSpeakerText(d.sides[s].speaker).map(x=>x.name)])) })))); """ return json.loads(subprocess.check_output(['node','--input-type=module','-e',code],cwd=ROOT,text=True)) def main(): public_rows=public(); assert len(public_rows)==308 assert len({p['id'] for p in public_rows})==308 source_groups=defaultdict(list) for p in public_rows: source_groups[p['youtubeUrl']].append(int(p['number'])) repeated_sources=[{'url':url,'assessments':ns} for url,ns in source_groups.items() if len(ns)>1] assert len(source_groups)==307 and sorted(repeated_sources[0]['assessments'])==[13,125], repeated_sources old={int(x['number']):x for x in csv.DictReader((ROOT/'docs/analysis/non-theist-vs-theist-2026-09-01/taxonomy.csv').open())} topic_by_n={n:t for t,ns in TOPICS.items() for n in ns} assert len(topic_by_n)==169 hash_paths={ROOT/'src/data'/name for name in ['debates.js','interlocutors.js','topic-assignments.js','topics.js','ai-contributions.js']} hash_paths.add(ROOT/'docs/analysis/non-theist-vs-theist-2026-09-01/taxonomy.csv') rows=[]; moves=[]; casebook=[]; checks=Counter(); raw_sources={} for p in public_rows: n=int(p['number']); path=ROOT/'docs/assessment-ledgers'/f"{p['id']}.json" detail=ROOT/'src/data/debate-details'/f"{p['id']}.js" if detail.exists(): hash_paths.add(detail) cards=[] for sec in p.get('sections',[]): for exc in sec.get('exchanges',[]): for side in ['pro','con']: raw=exc.get(side); seq=raw if isinstance(raw,list) else [raw] for card in seq: if not isinstance(card,dict): continue cards.append(dict(number=n,side=side,move_id=card.get('ledgerMoveId'),score=card['score'], words=card.get('words',''),critique=card.get('critique',''),time=card.get('time',''), fallacies=[t['label'] for t in card.get('tags',[]) if t['type']=='fallacy'], biases=[t['label'] for t in card.get('tags',[]) if t['type']=='bias'])) r=dict(number=n,id=p['id'],title=p['title'],motion=p['motion'],youtube=p['youtubeUrl'],date=p.get('date'), model=p.get('assessmentModel'),rubric=p.get('assessmentRubric'), pro=p['score']['pro'],con=p['score']['con'],con_gap=p['score']['con']-p['score']['pro'], speakers={s:p['canonical'][s][0] if len(p['canonical'][s])==1 else p['sides'][s]['speaker'] for s in ['pro','con']}, ranking_eligible=p.get('interlocutorRankingEligible') is not False and all(len(p['canonical'][s])==1 for s in ['pro','con']), cohort='unlocked',theist_side='',topic='',narrow=False,reason='Multi-speaker or non-comparable scoring format', public_moves=len(cards),public_fallacies={s:sum(bool(c['fallacies']) for c in cards if c['side']==s) for s in ['pro','con']}, public_anytags={s:sum(bool(c['fallacies'] or c['biases']) for c in cards if c['side']==s) for s in ['pro','con']}) if n in old: r['reason']=old[n]['reason'] if old[n]['included']=='True': r.update(theist_side=old[n]['theist_side'],topic=topic_by_n[n]) elif n in ADDITIONS: r['theist_side'],r['topic'],r['reason']=ADDITIONS[n] elif n<=226 and not r['ranking_eligible']: r['reason']='Historical multi-speaker assessment; excluded from comparable position analysis' else: raise ValueError(f'No reviewed inclusion decision for debate {n}') if r['theist_side']: r['non_side']='con' if r['theist_side']=='pro' else 'pro' r['gap']=r[r['non_side']]-r[r['theist_side']] r['narrow']=r['topic']!='Religion, culture & meaning' and n not in NARROW_EXCLUDE if path.exists() and r['ranking_eligible']: hash_paths.add(path); a=json.loads(path.read_text()); calc=a['calculated'] if 'scoringJudgment' in a: r['cohort']='earlier'; src=a['scoringJudgment']['moves'] else: r['cohort']='later'; final=ROOT/a['evidenceLocks']['finalLedger']['path']; hash_paths.add(final) assert hashlib.sha256(final.read_bytes()).hexdigest()==a['evidenceLocks']['finalLedger']['sha256'] checks['final_ledger_hashes']+=1; src=json.loads(final.read_text())['moves'] raw_sources[n]=src by_id={} for m in src: if r['cohort']=='earlier': d={k:m['ratings'][k]['value'] for k in list(DIMS)[:4]} d['precisionClarity']=precision(m['precisionFindings']) d['calibrationCharity']=rnd((calibration(m['calibrationFindings'])+m['ratings']['representationalCharity']['value'])/2) f=m['precisionFindings']; c=m['calibrationFindings'] flags=dict(low_warrant=d['evidenceWarrant']<70, overclaim=c['warrantFit'] in ['materially-overstated','radically-overstated'], compression=f['termStability']!='stable' or f['scopeStability']!='stable' or f['qualificationExplicitness'] in ['missing','materially-misleading']) flags['slogan_risk']=all(flags.values()) else: d={k:m['finalDimensions'][k]['value'] for k in DIMS}; flags={} item=dict(number=n,move_id=m['moveId'],side=m['side'],speaker=r['speakers'][m['side']], cohort=r['cohort'],kind=m.get('moveKind','unknown'),importance=int(m['importance']),dimensions=d, score=move_score(d),flags=flags, theist=(m['side']==r['theist_side']) if r['theist_side'] else None) by_id[m['moveId']]=item assert len(by_id)==len(src) seen=set(); r['dimensions']={}; r['adjustments']={} for s in ['pro','con']: d={k:0. for k in DIMS}; tot=0. assert sum(x['weightPercent'] for x in calc['sections'])==100 for sec in calc['sections']: its=sec['sides'][s]['moves']; den=sum(x['importance'] for x in its) for it in its: m=by_id[it['moveId']]; assert (m['side'],m['score'],m['importance'])==(s,it['score'],it['importance']), (n,it['moveId'],m['score'],it['score']) seen.add(it['moveId']); checks['move_scores']+=1 sc=rnd(sum(x['importance']*x['score'] for x in its)/den) assert sc==sec['sides'][s]['score']; checks['section_scores']+=1 tot+=sc*sec['weightPercent']/100 for k in d: d[k]+=sum(by_id[x['moveId']]['dimensions'][k]*x['importance'] for x in its)/den*sec['weightPercent']/100 adj=calc['overall'][s].get('burdenCompletionAdjustment',0) assert rnd(tot+adj)==calc['overall'][s]['score']==r[s], (n,s,tot,adj,r[s]) checks['overall_scores']+=1; r['dimensions'][s]=d; r['adjustments'][s]=adj assert seen==set(by_id) assert len(cards)==len(src), (n,len(cards),len(src)) assert {c['move_id'] for c in cards}==set(by_id) for c in cards: assert c['score']==by_id[c['move_id']]['score'],(n,c['move_id']) checks['public_move_scores']+=1 by_id[c['move_id']]['fallacies']=c['fallacies']; by_id[c['move_id']]['biases']=c['biases'] casebook.append({**c,'debate_id':r['id'],'youtube':r['youtube'],'speaker':r['speakers'][c['side']], 'dimensions':by_id[c['move_id']]['dimensions']}) moves.extend(by_id.values()) if r['theist_side']: t,nt=r['theist_side'],r['non_side'] r['dimension_gaps']={k:r['dimensions'][nt][k]-r['dimensions'][t][k] for k in DIMS} r['contributions']={k:r['dimension_gaps'][k]*w for k,w in DIMS.items()} r['adjustment_gap']=r['adjustments'][nt]-r['adjustments'][t] r['rounding_gap']=r['gap']-sum(r['contributions'].values())-r['adjustment_gap'] else: casebook.extend({**c,'debate_id':r['id'],'youtube':r['youtube'],'speaker':r['speakers'][c['side']]} for c in cards) rows.append(r) assert len(rows)==308 locked=[r for r in rows if r['cohort']!='unlocked']; religious=[r for r in locked if r['theist_side']] assert len(locked)==291 and len(religious)==234, (len(locked),len(religious)) assert len([r for r in religious if r['number']<=226])==169 by_n={r['number']:r for r in rows}; m_by_n=defaultdict(list) for m in moves: m_by_n[m['number']].append(m) result=dict(edition='October 9, 2026',source_commit=SOURCE_COMMIT,seed=SEED,draws=DRAWS, repeated_sources=repeated_sources, counts=dict(published=len(rows),unique_video_urls=len(source_groups),locked=len(locked),cohorts=dict(Counter(r['cohort'] for r in rows)), locked_moves=len(moves),public_moves=sum(r['public_moves'] for r in rows),religious=len(religious), religious_moves=sum(len(m_by_n[r['number']]) for r in religious),models=dict(Counter(r['model'] for r in rows))),checks=dict(checks)) # Paper 1: equal-debate contrasts, weighted decomposition, exclusions. p1={'gap':stats([r['gap'] for r in religious]), 'means':{s:mean([r[r[s+'_side']] for r in religious]) for s in ['theist','non']}, 'dimensions':{k:stats([r['dimension_gaps'][k] for r in religious]) for k in DIMS}, 'contributions':{k:mean([r['contributions'][k] for r in religious]) for k in DIMS}, 'adjustment':mean([r['adjustment_gap'] for r in religious]),'rounding':mean([r['rounding_gap'] for r in religious])} p1['sensitivity']={name:stats([r['gap'] for r in subset]) for name,subset in { 'original169':[r for r in religious if r['number']<=226], 'added_since_baseline':[r for r in religious if r['number']>226], 'september_snapshot':[r for r in religious if r['number']<=253], 'new_since_september':[r for r in religious if r['number']>253], 'without_new_origin_cases':[r for r in religious if r['number'] not in [269,306]], 'narrow':[r for r in religious if r['narrow']], 'earlier':[r for r in religious if r['cohort']=='earlier'], 'later':[r for r in religious if r['cohort']=='later'], 'theist_pro':[r for r in religious if r['theist_side']=='pro'], 'theist_con':[r for r in religious if r['theist_side']=='con'], 'without_four_frequent_skeptics':[r for r in religious if r['speakers'][r['non_side']] not in ['Matt Dillahunty',"Alex O’Connor", "Alex O'Connor",'Christopher Hitchens','Graham Oppy']], }.items()} p1['speaker_balanced']={} for role in ['theist','non']: grouped=defaultdict(list) for r in religious: grouped[r['speakers'][r[role+'_side']]].append(r['gap']) p1['speaker_balanced'][role]=stats([mean(v) for v in grouped.values()]) p1['evidence_thresholds']={} for threshold in [60,70,80]: group={s:[] for s in ['theist','non']} for r in religious: for s in group: ms=[m for m in m_by_n[r['number']] if m['side']==r[s+'_side']] group[s].append(mean([m['dimensions']['evidenceWarrant']0 else 'con'; hi='con' if lo=='pro' else 'pro' loss=dict(number=r['number'],cohort=r['cohort'],margin=abs(r['con_gap']),lower=lo, lower_no_fallacy=r['public_fallacies'][lo]==0,neither_fallacy=sum(r['public_fallacies'].values())==0, lower_no_tag=r['public_anytags'][lo]==0,higher_has_fallacy=r['public_fallacies'][hi]>0) if 'dimensions' in r: loss['dimension_gaps']={k:r['dimensions'][hi][k]-r['dimensions'][lo][k] for k in DIMS} loss['dimensions_behind']=sum(v>1e-9 for v in loss['dimension_gaps'].values()) losses.append(loss) p5={'decisive':len(losses),'ties':len(rows)-len(losses),'losses_without_fallacy':sum(x['lower_no_fallacy'] for x in losses), 'neither_fallacy':sum(x['neither_fallacy'] for x in losses),'losses_without_any_tag':sum(x['lower_no_tag'] for x in losses), 'winner_with_fallacy':sum(x['higher_has_fallacy'] for x in losses),'cohorts':{}} for c in ['earlier','later','unlocked']: ll=[x for x in losses if x['cohort']==c]; p5['cohorts'][c]=dict(n=len(ll),no_fallacy=sum(x['lower_no_fallacy'] for x in ll),neither=sum(x['neither_fallacy'] for x in ll)) lf=[x for x in losses if x['lower_no_fallacy'] and 'dimension_gaps' in x] p5['untagged_locked']={'n':len(lf),'behind_distribution':dict(Counter(x['dimensions_behind'] for x in lf)), 'dimension_gaps':{k:stats([x['dimension_gaps'][k] for x in lf]) for k in DIMS}, 'five_or_six':sum(x['dimensions_behind']>=5 for x in lf)} p5['tag_inventory']=dict(Counter(t for c in casebook for t in c['fallacies'])) p5['tagged_moves']=sum(bool(c['fallacies']) for c in casebook) p5['large_untagged_losses']=sorted([x for x in losses if x['lower_no_fallacy']],key=lambda x:-x['margin'])[:15] result['p5']=p5 # Paper 6: compare process families; these were not randomized generations. p6={'cohorts':{},'bridge':[]} for c in ['earlier','later']: rr=[r for r in locked if r['cohort']==c]; mm=[m for m in moves if m['cohort']==c] matrix=np.array([[m['dimensions'][k] for k in DIMS] for m in mm]); corr=np.corrcoef(matrix,rowvar=False) p6['cohorts'][c]={'debates':len(rr),'moves':len(mm),'midpoints':stats([(r['pro']+r['con'])/2 for r in rr]), 'absolute_margin':stats([abs(r['con_gap']) for r in rr]), 'dimensions':{k:mean([(r['dimensions']['pro'][k]+r['dimensions']['con'][k])/2 for r in rr]) for k in DIMS}, 'corr':corr.tolist(),'first_component_share':float(np.linalg.eigvalsh(corr)[-1]/6), 'tag_pct':100*mean([bool(m['fallacies'] or m['biases']) for m in mm]), 'fallacy_pct':100*mean([bool(m['fallacies']) for m in mm]), 'tag_pct_equal_debate':mean([100*sum(r['public_anytags'].values())/r['public_moves'] for r in rr])} for speaker,aa in grouped.items(): e=[a['score'] for a in aa if a['cohort']=='earlier'];l=[a['score'] for a in aa if a['cohort']=='later'] if e and l:p6['bridge'].append(dict(speaker=speaker,earlier_n=len(e),later_n=len(l),earlier=mean(e),later=mean(l),gap=mean(l)-mean(e))) p6['bridge_summary']=stats([b['gap'] for b in p6['bridge']]) es=[(r['pro']+r['con'])/2 for r in locked if r['cohort']=='earlier'];ls=[(r['pro']+r['con'])/2 for r in locked if r['cohort']=='later'] p6['level_difference']={'mean':mean(ls)-mean(es),'ci':interval(boots(ls,301)-boots(es,302))} p6['sequence']=[dict(number=r['number'],cohort=r['cohort'],midpoint=(r['pro']+r['con'])/2,margin=abs(r['con_gap'])) for r in locked] result['p6']=p6 # Paper 7: finite leaderboard, observed-repeatability and two rank models. eligible={s:aa for s,aa in grouped.items() if len(aa)>=3} names=sorted(eligible,key=lambda s:(-mean([a['score'] for a in eligible[s]]),-len(eligible[s]),s)) gs=[np.array([a['score'] for a in eligible[s]],float) for s in names];N=sum(len(g) for g in gs);G=len(gs) grand=mean(np.concatenate(gs)); between=sum(len(g)*(mean(g)-grand)**2 for g in gs)/(G-1) within=sum(sum((g-mean(g))**2) for g in gs)/(N-G) k0=(N-sum(len(g)**2 for g in gs)/N)/(G-1);bv=max(0,(between-within)/k0) rng=np.random.default_rng(SEED+400) empirical=np.column_stack([rng.choice(g,(DRAWS,len(g)),replace=True).mean(axis=1) for g in gs]) # Average ranks avoid alphabetical tie-break artefacts in uncertainty. eranks=np.stack([rankdata(-r,method='average') for r in empirical]) postvar=np.array([1/(1/bv+len(g)/within) for g in gs]) postmean=np.array([v*(grand/bv+len(g)*mean(g)/within) for v,g in zip(postvar,gs)]) latent=rng.normal(postmean,np.sqrt(postvar),(DRAWS,G));lranks=np.stack([rankdata(-r) for r in latent]) ranking=[] for i,(s,g) in enumerate(zip(names,gs)): loo=[] for j in range(len(g)): av=np.array([mean(h) for h in gs]);av[i]=mean(np.delete(g,j));loo.append(float(rankdata(-av,method='average')[i])) ranking.append(dict(speaker=s,rank=i+1,n=len(g),mean=mean(g),sd=float(g.std(ddof=1)), earlier_n=sum(a['cohort']=='earlier' for a in eligible[s]),later_n=sum(a['cohort']=='later' for a in eligible[s]), empirical_mean_ci=interval(empirical[:,i]),empirical_rank_ci=interval(eranks[:,i]), model_mean=float(postmean[i]),model_rank_ci=interval(lranks[:,i]),model_top10=float(mean(lranks[:,i]<=10)), leave_one_out=[min(loo),max(loo)])) split_names=[s for s,aa in eligible.items() if len(aa)>=6];splitcorr=[] for _ in range(3000): a=[];b=[] for s in split_names: g=rng.permutation([x['score'] for x in eligible[s]]);cut=len(g)//2;a.append(mean(g[:cut]));b.append(mean(g[cut:])) splitcorr.append(float(spearmanr(a,b).statistic)) chrono_a=[];chrono_b=[] for s in split_names: aa=sorted(eligible[s],key=lambda a:a['number']);cut=len(aa)//2;chrono_a.append(mean([a['score'] for a in aa[:cut]]));chrono_b.append(mean([a['score'] for a in aa[cut:]])) # Shared positive random weights retain the fixed ranked field and keep # opponents paired. This Bayesian-bootstrap-style sensitivity is different # from resampling independent appearances and introduces no new performances. rn=[r['number'] for r in locked if r['ranking_eligible']]; index={n:i for i,n in enumerate(rn)} counts=np.zeros((len(rn),G));totals=counts.copy() for j,s in enumerate(names): for a in eligible[s]:counts[index[a['number']],j]+=1;totals[index[a['number']],j]+=a['score'] weights=rng.exponential(1,size=(DRAWS,len(rn))) dn=weights@counts;dt=weights@totals cr=np.stack([rankdata(-r,method='average') for r in (dt/dn)]) for j,r in enumerate(ranking):r['paired_resample_rank_ci']=interval(cr[:,j]) # Generation-centering is a sensitivity exercise, not a calibrated ranking. cm={c:mean([a['score'] for a in appearances if a['cohort']==c]) for c in ['earlier','later']} centered=[mean([a['score']-cm[a['cohort']] for a in eligible[s]]) for s in names] newrank=rankdata(-np.array(centered),method='average') for i,r in enumerate(ranking):r['generation_centered_rank']=float(newrank[i]) p7=dict(appearances=len(appearances),eligible_appearances=N,all_speakers=len(grouped),ranked_speakers=G, appearance_counts=dict(Counter(len(aa) for aa in grouped.values())),ranking=ranking, repeatability=dict(between_variance=bv,within_variance=within,single=bv/(bv+within), means={str(k):bv/(bv+within/k) for k in [3,5,10]},within_sd=math.sqrt(within)), split_half=dict(n=len(split_names),median=float(np.median(splitcorr)),interval=interval(splitcorr),draws=3000, sequence_split_spearman=float(spearmanr(chrono_a,chrono_b).statistic)), median_adjacent_gap=float(np.median(np.diff([-mean(g) for g in gs]))), median_empirical_rank_width=float(np.median([r['empirical_rank_ci'][1]-r['empirical_rank_ci'][0] for r in ranking])), median_model_rank_width=float(np.median([r['model_rank_ci'][1]-r['model_rank_ci'][0] for r in ranking])), median_paired_rank_width=float(np.median([r['paired_resample_rank_ci'][1]-r['paired_resample_rank_ci'][0] for r in ranking])), paired_weight_draws=DRAWS, zero_variation_speakers=[r['speaker'] for r in ranking if r['sd']==0], centered_rank_spearman=float(spearmanr(range(1,G+1),newrank).statistic), top4_appearance_share=sum(sorted([len(aa) for aa in grouped.values()],reverse=True)[:4])/len(appearances)) result['p7']=p7 # Publication helpers keep every repeated number tied to the same calculation. previous=json.loads((ROOT/'docs/analysis/astra-corpus-papers-2026-09-04/debates.json').read_text()) changed=[a['number'] for a in previous if (a['pro'],a['con'],a['cohort']) != (by_n[a['number']]['pro'],by_n[a['number']]['con'],by_n[a['number']]['cohort'])] assert not changed, ('Review historical score changes before publishing',changed) assert len({r['youtube'] for r in locked})==len(locked), 'Comparable set repeats a video' result['derived']=dict(new_assessments=len(rows)-len(previous),new_comparisons=p1['sensitivity']['new_since_september']['n'], score_gap_reduction=6.342245989304812-p1['gap']['mean'], top_three_contribution=sum(p1['contributions'][k] for k in list(DIMS)[:3]), top_three_share=100*sum(p1['contributions'][k] for k in list(DIMS)[:3])/p1['gap']['mean'], loss_no_fallacy_pct=100*p5['losses_without_fallacy']/p5['decisive'], later_no_fallacy_pct=100*p5['cohorts']['later']['no_fallacy']/p5['cohorts']['later']['n'], earlier_no_fallacy_pct=100*p5['cohorts']['earlier']['no_fallacy']/p5['cohorts']['earlier']['n'], untagged_five_six_pct=100*p5['untagged_locked']['five_or_six']/p5['untagged_locked']['n'], absolute_process_difference=-p6['level_difference']['mean'], history_changed=changed,public_label_instances=sum(p5['tag_inventory'].values())) # Recalculate the historical direct study from its published debate-level counts. # No new AI judgments; never infer slogan use from score thresholds. direct_path=ROOT/'docs/analysis/direct-slogan-study-2026-09-04/light-debates.json' direct_rows=json.loads(direct_path.read_text());assert len(direct_rows)==187 direct={} saved=json.loads((direct_path.parent/'light-results.json').read_text()) assert saved['status']=='complete' and not saved['issues'] for metric in ['unsupported','protected_slogan']: levels={s:np.array([10000*r[s+'_'+metric]/r[s+'_words'] for r in direct_rows]) for s in ['theist','non_theist']} gap=levels['theist']-levels['non_theist'] direct[metric]={'n':187,'counts':{s:sum(r[s+'_'+metric] for r in direct_rows) for s in levels}, 'rates':{s:mean(v) for s,v in levels.items()},'gap':mean(gap),'ci':interval(boots(gap)), 'neither':sum(r['theist_'+metric]==r['non_theist_'+metric]==0 for r in direct_rows)} old=saved['statistics']['all'][metric] assert abs(direct[metric]['gap']-old['mean_gap'])<1e-10 for s in levels: assert direct[metric]['counts'][s]==old[s+'_count'] assert abs(direct[metric]['rates'][s]-old[s+'_equal_debate_rate'])<1e-10 assert np.allclose(direct[metric]['ci'],old['gap_ci']) result['historical_direct']=dict(review_date='September 5, 2026',new_transcripts_reviewed=0,results=direct) hash_paths.update([direct_path,direct_path.parent/'light-results.json']) dump('results.json',result);dump('debates.json',rows);dump('moves.json',moves);dump('casebook.json',casebook);dump('losses.json',losses) csvout('classification.csv',[{k:r.get(k,'') for k in ['number','id','title','motion','cohort','theist_side','topic','narrow','reason','gap']} for r in rows]) csvout('ranking.csv',ranking) hashes=[{'path':str(p.relative_to(ROOT)),'sha256':hashlib.sha256(p.read_bytes()).hexdigest()} for p in sorted(hash_paths)] dump('source-manifest.json',dict(edition='2026-10-09',source_commit=SOURCE_COMMIT,files=hashes,checks=dict(checks), exclusions=f'{len(rows)-len(locked)} multi-speaker/non-comparable assessments omitted from locked dimension, position and ranking comparisons.')) print(json.dumps({k:v for k,v in result.items() if k not in ['p2','p7']},indent=2)[:4000]) print('P7:',json.dumps({k:v for k,v in p7.items() if k not in ['ranking','appearance_counts']},indent=2)) if __name__=='__main__': main()