import requests,re,concurrent.futures,json
from bs4 import BeautifulSoup
s=open('work/scratch/liferay-critical/airlock-docs-sitemap.xml').read()
urls=[u for u in re.findall(r'<loc>(.*?)</loc>',s) if '/iam/latest/' in u]
terms=['release notes','8.6.1','8.6.0','8.5.','8.4.','angular 19','jersey 3.1.10','loginapp','third-party libraries','software bill of materials']
S=requests.Session(); S.headers['User-Agent']='Mozilla/5.0'
def one(u):
 try:
  r=S.get(u,timeout=20)
  if r.status_code!=200:return None
  t=BeautifulSoup(r.text,'html.parser').get_text(' ',strip=True)
  lo=t.lower()
  hits=[x for x in terms if x in lo]
  title=(BeautifulSoup(r.text,'html.parser').title.get_text(' ',strip=True) if BeautifulSoup(r.text,'html.parser').title else '')
  if hits or 'release' in title.lower() or 'version' in title.lower():
   snippets=[]
   for x in hits:
    i=lo.find(x); snippets.append(t[max(0,i-250):i+500])
   return {'url':u,'title':title,'hits':hits,'snippets':snippets}
 except Exception as e: return None
with concurrent.futures.ThreadPoolExecutor(max_workers=16) as ex:
 rows=[x for x in ex.map(one,urls) if x]
open('work/scratch/liferay-critical/airlock-docs-matches.json','w').write(json.dumps(rows,ensure_ascii=False,indent=2))
print('urls',len(urls),'matches',len(rows))
for x in rows:
 print('\n',x['title'],x['url'],x['hits'])
 for s in x['snippets'][:3]:print(' ',s[:700])