e
This commit is contained in:
parent
a180e18935
commit
00000d9050
5 changed files with 295 additions and 189 deletions
|
|
@ -15,11 +15,6 @@ API_FILE = Cache(Path(__file__).parent / "caches" / "ppv_api.json", exp=28_800)
|
|||
|
||||
CACHE_FILE = Cache(Path(__file__).parent / "caches" / "ppv.json", exp=10_800)
|
||||
|
||||
# MIRRORS = [
|
||||
# "https://ppv.to",
|
||||
# "https://ppvs.su",
|
||||
# ]
|
||||
|
||||
BASE_URL = "https://ppv.to"
|
||||
|
||||
|
||||
|
|
@ -41,15 +36,13 @@ async def refresh_api_cache(
|
|||
|
||||
async def get_events(
|
||||
client: httpx.AsyncClient,
|
||||
base_url: str,
|
||||
cached_keys: set[str],
|
||||
) -> list[dict[str, str]]:
|
||||
|
||||
if not (api_data := API_FILE.load(per_entry=False)):
|
||||
api_data = await refresh_api_cache(
|
||||
client,
|
||||
urljoin(
|
||||
base_url,
|
||||
BASE_URL,
|
||||
"api/streams",
|
||||
),
|
||||
)
|
||||
|
|
@ -107,18 +100,9 @@ async def scrape(client: httpx.AsyncClient) -> None:
|
|||
|
||||
log.info(f"Loaded {cached_count} event(s) from cache")
|
||||
|
||||
# if not (base_url := await network.get_base(MIRRORS)):
|
||||
# log.warning("No working PPV mirrors")
|
||||
# CACHE_FILE.write(cached_urls)
|
||||
# return
|
||||
|
||||
log.info(f'Scraping from "{BASE_URL}"')
|
||||
|
||||
events = await get_events(
|
||||
client,
|
||||
BASE_URL,
|
||||
set(cached_urls.keys()),
|
||||
)
|
||||
events = await get_events(client, set(cached_urls.keys()))
|
||||
|
||||
log.info(f"Processing {len(events)} new URL(s)")
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue