add semaphores to scrapers
(maybe) fix hanging on watchfooty
misc. edits
This commit is contained in:
doms9 2025-12-23 03:17:48 -05:00
parent 6e9729bf8c
commit 00000d920a
20 changed files with 103 additions and 73 deletions

View file

@ -77,7 +77,14 @@ async def scrape() -> None:
log.info(f'Scraping from "{BASE_URL}"')
urls.update(await get_events())
events = await network.safe_process(
get_events,
url_num=1,
semaphore=network.HTTP_S,
log=log,
)
urls.update(events or {})
CACHE_FILE.write(urls)