import json, re, difflib, os
BASE = os.path.dirname(os.path.abspath(__file__))
prod = {r['id']: r for r in json.load(open(os.path.join(BASE,'_needsrev_half.json'),encoding='utf-8'))}
dev  = {r['id']: r for r in json.load(open(os.path.join(BASE,'_dev_versions.json'),encoding='utf-8'))}

def skel(s):
    if not s: return ''
    s = re.sub(r'\$\$.*?\$\$', ' M ', s, flags=re.S)   # display math
    s = re.sub(r'\\\(.*?\\\)', ' M ', s, flags=re.S)   # inline math
    s = re.sub(r'<[^>]+>', ' ', s)                      # html tags
    s = re.sub(r'[^a-z0-9]+', ' ', s.lower())           # punctuation
    return ' '.join(s.split())

updates=[]; identical=[]; no_dev=[]; not_resemble=[]
for id, p in prod.items():
    d = dev.get(id)
    if not d or not d.get('exists'):
        no_dev.append(id); continue
    pq, dq = p.get('question') or '', d.get('dev_question') or ''
    if dq == pq:
        identical.append(id); continue
    ratio = difflib.SequenceMatcher(None, skel(pq), skel(dq)).ratio()
    if ratio >= 0.85:
        updates.append({'id': id, 'question': dq, 'ratio': round(ratio,3)})
    else:
        not_resemble.append({'id': id, 'ratio': round(ratio,3)})

json.dump([{'id':u['id'],'question':u['question']} for u in updates],
          open(os.path.join(BASE,'_recover_updates.json'),'w',encoding='utf-8'), ensure_ascii=False)
print(f"RECOVER(resemble & differ)={len(updates)}  identical(genuine1/2 or dev-also-corrupt)={len(identical)}  no_dev={len(no_dev)}  not_resemble={len(not_resemble)}")
print("no_dev ids:", no_dev)
print("not_resemble:", not_resemble)
print("--- sample recoveries ---")
for u in updates[:6]:
    print(f"#{u['id']} r={u['ratio']}")
    print("  PROD:", re.sub(r'\s+',' ',(prod[u['id']]['question'] or ''))[:115])
    print("  DEV :", re.sub(r'\s+',' ',u['question'])[:115])