"""Recompute an archived data-quality report, offline, using Python stdlib. This is not an AGS calculation, model benchmark or proof of causal mode effects. Response bodies are deliberately omitted: response-derived flags cannot be independently re-extracted from the public file. Prompt-derived checks and all published numerators/denominators can be recomputed from it. """ import argparse import collections import hashlib import json from pathlib import Path import re if not __debug__: raise SystemExit('Validation requires ordinary Python; optimization flags -O/-OO are unsupported') MARKETING_PHRASE = 'Auditez la visibilité de vos clients sur ChatGPT, Claude, Gemini & Perplexity et' BRAND = re.compile(r'\bai[\s-]*labs[\s-]*(?:audit|solutions)\b|(? 1] out = dict(stored_rows=len(rows), distinct_prompt_ids=len(identities), localized_questions=len(questions), model_configurations=len(configurations), base_model_families=len({r['model_family'] for r in rows}), total=aggregate(rows), by_language=[], by_configuration=[], by_prompt_context=[], paired_configurations=[]) for language in languages: subset = [row for row in rows if questions[row['question_id']]['language'] == language] out['by_language'].append(dict(language=language, **aggregate(subset))) for supplied in (False, True): selected = [row for row in subset if questions[row['question_id']]['brand_in_stored_prompt'] == supplied] out['by_prompt_context'].append(dict(language=language, brand_in_stored_prompt=supplied, localized_questions=sum(q['language']==language and q['brand_in_stored_prompt']==supplied for q in questions.values()), **aggregate(selected))) for model in configurations: subset = [row for row in rows if row['model_id'] == model] out['by_configuration'].append(dict(model_id=model, model_family=subset[0]['model_family'], mode_marker=subset[0]['mode_marker'], **aggregate(subset), by_language=[dict(language=language, **aggregate([r for r in subset if r['response_language']==language])) for language in languages])) indexed = {(row['question_id'],row['model_id']):row for row in rows} for model in configurations: if model.endswith(':online') or model+':online' not in configurations: continue for language in languages: ids = sorted(q['id'] for q in questions.values() if q['language']==language) outcomes = collections.Counter() usable_pairs = 0 identical_hashes = 0 for qid in ids: left, right = indexed[(qid,model)], indexed[(qid,model+':online')] if not (left['usable_text_row'] and right['usable_text_row']): continue usable_pairs += 1 identical_hashes += left['response_sha256']==right['response_sha256'] outcomes['both_mention' if left['lexical_product_mention'] and right['lexical_product_mention'] else 'no_online_suffix_only' if left['lexical_product_mention'] else 'online_suffix_only' if right['lexical_product_mention'] else 'neither_mention'] += 1 out['paired_configurations'].append(dict(model_family=model, language=language, no_online_suffix_model_id=model, online_suffix_model_id=model+':online', candidate_questions=len(ids), usable_pairs=usable_pairs, usable_pair_ratio=ratio(usable_pairs,len(ids)), identical_response_hashes=identical_hashes, lexical_presence_outcomes={key:outcomes[key] for key in ( 'both_mention','no_online_suffix_only','online_suffix_only','neither_mention')}, interpretation='Same stored question and language, different stored configuration; effective web search/full provider request not verified. Not a causal comparison or quality ranking.')) out['prompt_quality'] = dict(cross_language_brand_context_differs=ratio(len(different_context),len(identities)), affected_prompt_ids=different_context, with_marketing_phrase=ratio(sum(MARKETING_PHRASE.casefold() in q['text'].casefold() for q in questions.values()),len(questions)), by_language=[dict(language=language, with_marketing_phrase=ratio(sum(q['language']==language and MARKETING_PHRASE.casefold() in q['text'].casefold() for q in questions.values()), sum(q['language']==language for q in questions.values())), brand_in_stored_prompt=sum(q['language']==language and q['brand_in_stored_prompt'] for q in questions.values()), no_brand_in_stored_prompt=sum(q['language']==language and not q['brand_in_stored_prompt'] for q in questions.values())) for language in languages], spontaneous_title_but_brand_supplied=[q['id'] for q in questions.values() if 'spontan' in q['title'].casefold() and q['brand_in_stored_prompt']], stored_brand_flag_disagreements=sum(q['stored_is_branded']!=q['brand_in_stored_prompt'] for q in questions.values())) out['first_response_created_at_raw'] = min(row['created_at_raw'] for row in rows) out['last_response_created_at_raw'] = max(row['created_at_raw'] for row in rows) return out def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('dataset', type=Path) parser.add_argument('--check', action='store_true', help='Verify the published summary, print no private data') args = parser.parse_args() data = json.loads(args.dataset.read_text()) result = recompute(data) if args.check: assert result == data['summary'], 'summary_drift' print(json.dumps(dict(status='PASS', stored_rows=result['stored_rows'], localized_questions=result['localized_questions'], usable_rows=result['total']['usable_rows']))) else: print(json.dumps(result,ensure_ascii=False,indent=2)) if __name__ == '__main__': main()