import re,requests,concurrent.futures,bs4,json,time
from urllib.parse import urljoin
s=requests.Session(); s.headers['User-Agent']='Mozilla/5.0 research'
xml=open('work/scratch/swissdec/sitemap.xml').read(); urls=re.findall(r'<loc>(.*?)</loc>',xml)
def fetch(u):
 try:
  r=s.get(u,timeout=20)
  soup=bs4.BeautifulSoup(r.text,'html.parser')
  title=soup.title.get_text(' ',strip=True) if soup.title else ''
  out=[]
  for a in soup.find_all('a',href=True):
   h=urljoin(u,a['href'])
   if '/document/share/' in h or '/web/content/' in h or '/fileadmin/' in h:
    ctx=' '.join((a.get_text(' ',strip=True), a.parent.get_text(' ',strip=True) if a.parent else ''))
    out.append({'source':u,'title':title,'url':h,'text':ctx[:1000]})
  return out
 except Exception as e: return [{'source':u,'error':repr(e)}]
all=[]
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as ex:
 for x in ex.map(fetch,urls): all.extend(x)
open('work/scratch/swissdec/document-links.json','w').write(json.dumps(all,indent=2,ensure_ascii=False))
print('pages',len(urls),'links',len(all),'unique',len({x.get('url') for x in all if x.get('url')}))
for x in all:
 if x.get('url'): print(x['source'],'\t',x['url'],'\t',x.get('text','')[:240].replace('\n',' '))