import json,re,base64,urllib.parse,collections
from lxml import etree
x=json.load(open('work/scratch/saml-all.json'))
issuers=collections.Counter(); roots=collections.Counter(); agents=collections.Counter(); attrs=collections.Counter(); names=[]
for row in x:
 agents[row.get('agent')]+=1
 req='\n'.join(b.get('message','') for b in row.get('blobs',[]) if b.get('kind')=='request')
 body=req.split('\r\n\r\n',1)[-1] if '\r\n\r\n' in req else req.split('\n\n',1)[-1]
 m=re.search(r'(?:^|&)SAMLResponse=([^&]*)',body)
 if not m: continue
 raw=urllib.parse.unquote_plus(m.group(1))
 xml=None
 for cand in [raw]:
  try:
   bb=base64.b64decode(cand+'===' ,validate=False)
   if b'<' in bb: xml=bb
  except: pass
  if xml is None and '<' in cand: xml=cand.encode()
 if not xml: continue
 try:
  root=etree.fromstring(xml)
 except Exception: continue
 roots[root.tag.split('}')[-1]]+=1
 for e in root.xpath('//*[local-name()="Issuer"]'):
  if e.text: issuers[e.text.strip()]+=1
 for e in root.xpath('//*[local-name()="Attribute"]'):
  attrs[e.get('Name','')]+=1
 txt=xml.decode(errors='ignore')
 if 'bugbounty' not in txt.lower() and 'dwbzn' not in txt.lower():
  names.append((row['exchange_id'],row.get('agent'),row.get('response_status'),len(xml),txt[:500].replace('\n',' ')))
print('agents',agents)
print('roots',roots)
print('issuers'); [print(n,v) for n,v in issuers.most_common()]
print('attrs'); [print(n,v) for n,v in attrs.most_common(80)]
print('nonbb',len(names)); [print(*z,sep='\t') for z in names[:100]]