From 00000d99acb0e764d6799cd74757ab5b560f5261 Mon Sep 17 00:00:00 2001 From: doms9 <96013514+doms9@users.noreply.github.com> Date: Tue, 9 Sep 2025 13:34:16 -0400 Subject: [PATCH] e --- M3U8/scrapers/livetvsx.py | 2 +- M3U8/scrapers/ppv.py | 26 +++++++++---------- M3U8/scrapers/utils/config.py | 47 +++++++++++++++++++---------------- 3 files changed, 39 insertions(+), 36 deletions(-) diff --git a/M3U8/scrapers/livetvsx.py b/M3U8/scrapers/livetvsx.py index c05f8e2..7afc54e 100644 --- a/M3U8/scrapers/livetvsx.py +++ b/M3U8/scrapers/livetvsx.py @@ -311,7 +311,7 @@ async def main(client: httpx.AsyncClient) -> None: urls[key] = cached_urls[key] = entry - if (new_count := len(cached_urls) - cached_count) > 0: + if new_count := len(cached_urls) - cached_count: CACHE_FILE.write_text(json.dumps(cached_urls, indent=2), encoding="utf-8") log.info(f"Collected and cached {new_count} new event(s)") diff --git a/M3U8/scrapers/ppv.py b/M3U8/scrapers/ppv.py index 676b909..66be2e8 100644 --- a/M3U8/scrapers/ppv.py +++ b/M3U8/scrapers/ppv.py @@ -38,7 +38,9 @@ MIRRORS = [ ] -async def refresh_api_cache(client: httpx.AsyncClient, url: str) -> dict: +async def refresh_api_cache( + client: httpx.AsyncClient, url: str +) -> dict[str, dict[str, str]]: log.info("Refreshing API cache") try: @@ -51,17 +53,6 @@ async def refresh_api_cache(client: httpx.AsyncClient, url: str) -> dict: return r.json() -def load_api_cache() -> dict[str, dict[str, str | str]]: - try: - data: dict = json.loads(API_FILE.read_text(encoding="utf-8")) - - age: float = now.timestamp() - data.get("timestamp", 0) - - return data if age < 86400 else {} # 24 hours - except (FileNotFoundError, json.JSONDecodeError): - return {} - - async def process_event(url: str, url_num: int) -> str | None: async with async_playwright() as p: browser = await p.firefox.launch(headless=True) @@ -126,7 +117,14 @@ async def get_events( base_url = re.match(r"(https?://.+?)/", api_url)[1] - if not (api_data := load_api_cache()): + if not ( + api_data := load_cache( + API_FILE, + exp=86400, + nearest_hr=True, + per_entry=False, + ) + ): api_data = await refresh_api_cache(client, api_url) API_FILE.write_text(json.dumps(api_data, indent=2), encoding="utf-8") @@ -211,7 +209,7 @@ async def main(client: httpx.AsyncClient) -> None: urls[key] = cached_urls[key] = entry - if (new_count := len(cached_urls) - cached_count) > 0: + if new_count := len(cached_urls) - cached_count: CACHE_FILE.write_text(json.dumps(cached_urls, indent=2), encoding="utf-8") log.info(f"Collected and cached {new_count} new event(s)") diff --git a/M3U8/scrapers/utils/config.py b/M3U8/scrapers/utils/config.py index 0595020..a3eaaae 100644 --- a/M3U8/scrapers/utils/config.py +++ b/M3U8/scrapers/utils/config.py @@ -73,35 +73,40 @@ def near_hr(dt: datetime) -> float: return dt.replace(minute=0, second=0, microsecond=0).timestamp() +def is_fresh( + entry: dict, + nearest_hr: bool, + exp: int, +) -> bool: + ts = entry.get("timestamp", 31496400) + + if nearest_hr: + ts = near_hr(datetime.fromtimestamp(ts)) + + return now.timestamp() - ts < exp + + def load_cache( file: Path, - exp: int | float = None, + exp: int | float, nearest_hr: bool = False, + per_entry: bool = True, ) -> dict[str, dict[str, str | float]]: try: - data: dict[str, dict[str, str | float]] = json.loads( - file.read_text(encoding="utf-8") - ) - - return ( - { - k: v - for k, v in data.items() - if now.timestamp() - - near_hr(datetime.fromtimestamp(v.get("timestamp", 31496400))) - < exp - } - if nearest_hr - else { - k: v - for k, v in data.items() - if now.timestamp() - v.get("timestamp", 31496400) < exp - } - ) - + data: dict = json.loads(file.read_text(encoding="utf-8")) except (FileNotFoundError, json.JSONDecodeError): return {} + if per_entry: + return {k: v for k, v in data.items() if is_fresh(v, nearest_hr, exp)} + + ts = data.get("timestamp", 31496400) + + if nearest_hr: + ts = near_hr(datetime.fromtimestamp(ts)) + + return data if now.timestamp() - ts < exp else {} + async def safe_process_event( fn: Callable,