import json,re,urllib.parse,base64,collections,statistics
D=json.load(open('work/scratch/saml-all.json'))
rows=[]
for x in D:
 if x['method']!='POST': continue
 req=next((b['message'] for b in x['blobs'] if b['kind']=='request'), '')
 resp=next((b['message'] for b in x['blobs'] if b['kind']=='response'), '')
 body=req.split('\r\n\r\n',1)[-1]
 q=urllib.parse.parse_qs(body,keep_blank_values=True)
 ctx=q.get('context',[''])[0]
 sr=q.get('SAMLResponse',[''])[0]
 xml=''
 try: xml=base64.b64decode(sr+'===').decode('utf-8','replace')
 except: pass
 feat=[]
 for k,p in [('doctype','<!DOCTYPE'),('xinclude','xi:include'),('retrieval','RetrievalMethod'),('keyinfo','KeyInfo'),('signature','Signature'),('encrypted','EncryptedAssertion'),('cmd','id%20|/bin/|whoami|Runtime'),('oob','9b.au'),('assertion','Assertion'),('response','Response')]:
  if re.search(p,xml,re.I): feat.append(k)
 status=x['response_status']
 rbody=resp.split('\r\n\r\n',1)[-1]
 code=''
 m=re.search(r'"code":"?([^",}]+)',rbody)
 if m: code=m.group(1)
 rows.append((x['exchange_id'],status,x['duration_ms'],ctx,','.join(feat),len(xml),code,rbody[:180].replace('\n',' ')))
print('POSTs',len(rows))
print('statuses',collections.Counter(r[1] for r in rows))
print('contexts',collections.Counter(r[3] for r in rows))
print('codes',collections.Counter(r[6] for r in rows))
print('\nNon500/unique:')
for r in rows:
 if r[1]!=500 or r[6]: print('\t'.join(map(str,r)))
print('\nGrouped feature/status/duration:')
g=collections.defaultdict(list)
for r in rows: g[(r[3],r[4],r[1],r[6])].append(r[2])
for k,v in sorted(g.items(), key=lambda x:(x[0][2],x[0][0],x[0][1])):
 print(k,'n',len(v),'median',statistics.median(v),'min',min(v),'max',max(v))