import requests,feedparser,json,time
subs=['AskWomenOver30','WomenOver40','workingmoms','careerchange','findapath','Menopause','divorce','solotravel','digitalnomad','expats','LifeAfterSchool','careerwomen','womenintech']
terms=['therapy','career coach','life coach','somatic','nervous system','retreat','reiki','spirituality','tarot','human design','coaching program','breathwork','meditation','burnout therapy','EMDR','career transition','career change therapy','quit job therapy','divorce therapy','midlife coach']
out=[]
H={'User-Agent':'Mozilla/5.0 (compatible; academic-research/0.1)'}
for sub in subs:
 for q in terms:
  u=f'https://www.reddit.com/r/{sub}/search.rss?q='+requests.utils.quote(q)+'&restrict_sr=on&sort=new&t=year'
  r=requests.get(u,headers=H)
  if r.status_code==200:
   d=feedparser.parse(r.text)
   for e in d.entries:
    out.append({'sub':sub,'q':q,'title':e.get('title'),'url':e.get('link'),'author':e.get('author'),'date':e.get('updated'),'content':e.get('content',[{}])[0].get('value','')})
  time.sleep(.1)
# unique URL
z={x['url']:x for x in out}
json.dump(list(z.values()),open('/home/spazio/rss.json','w'),indent=2)
print(len(z))