import pandas as pd,numpy as np
from scipy import optimize,stats
R=pd.read_csv('ni_study.csv')
def reml(y,v):
    y=np.asarray(y);v=np.asarray(v);k=len(y)
    def nll(t2):
        W=1/(v+t2);mu=sum(W*y)/sum(W);return 0.5*(sum(np.log(v+t2))+np.log(sum(W))+sum(W*(y-mu)**2))
    t2=optimize.minimize_scalar(nll,bounds=(0,20),method='bounded').x
    if nll(0)<=nll(t2): t2=0.0
    W=1/(v+t2);mu=sum(W*y)/sum(W);return mu,np.sqrt(1/sum(W)),t2
inv=lambda z:1/(1+np.exp(-z))
order=['Chinese characters','Pictures','Tones','Color','Food']
P={}
for c in order:
    x=R[R.cls==c];mu,se,t2=reml(x.logit,x.v_logit)
    P[c]=dict(k=len(x),NI=inv(mu),lo=inv(mu-1.96*se),hi=inv(mu+1.96*se),tau2=t2)
    print(f"{c:20s} k={len(x)} NI={inv(mu):.3f} [{inv(mu-1.96*se):.3f}, {inv(mu+1.96*se):.3f}] tau2_logit={t2:.3f}")
lo,hi=sorted([P['Color']['NI'],P['Food']['NI']])
print(f"\ncolor-food interval: [{lo:.3f}, {hi:.3f}]")
inside=[c for c in ['Chinese characters','Pictures','Tones'] if lo<P[c]['NI']<hi]
print("null classes inside:",inside,"->", "CONSISTENT" if not inside else "CONTRADICTS")
for c in ['Chinese characters','Pictures','Tones']:
    side='below' if P[c]['NI']<lo else ('above' if P[c]['NI']>hi else 'inside')
    ov=not (P[c]['hi']<lo or P[c]['lo']>hi)
    print(f"  {c}: {side}; its CI overlaps the color-food interval: {ov}")
pd.DataFrame(P).T.to_csv('ni_class.csv')