e
This commit is contained in:
parent
00000d90bc
commit
00000d9c6a
3 changed files with 25 additions and 16 deletions
|
|
@ -38,7 +38,11 @@ CERT_FILE = Path(__file__).parent / "utils" / "cached-ca.pem"
|
|||
CACHE_FILE = Path(__file__).parent / "caches" / "livetvsx.json"
|
||||
|
||||
|
||||
async def write_to_cert(client: httpx.AsyncClient, url: str, cert: Path) -> None:
|
||||
async def write_to_cert(
|
||||
client: httpx.AsyncClient,
|
||||
url: str,
|
||||
cert: Path,
|
||||
) -> None:
|
||||
try:
|
||||
r = await client.get(url)
|
||||
r.raise_for_status()
|
||||
|
|
@ -263,17 +267,21 @@ async def get_events(
|
|||
|
||||
|
||||
async def main(client: httpx.AsyncClient) -> None:
|
||||
log.info(f'Scraping from "{BASE_URL}"')
|
||||
|
||||
cert = await get_cert(client)
|
||||
|
||||
cached_urls = load_cache(CACHE_FILE, exp=14400)
|
||||
cached_count = len(cached_urls)
|
||||
urls.update(cached_urls)
|
||||
|
||||
log.info(f"Collected {cached_count} event(s) from cache")
|
||||
|
||||
events = await get_events(BASE_URL, cert, set(cached_urls.keys()))
|
||||
log.info(f'Scraping from "{BASE_URL}"')
|
||||
|
||||
cert = await get_cert(client)
|
||||
|
||||
events = await get_events(
|
||||
BASE_URL,
|
||||
cert,
|
||||
set(cached_urls.keys()),
|
||||
)
|
||||
|
||||
log.info(f"Processing {len(events)} new URL(s)")
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue