From 00000d903ed76a07071183bb0a16fe784e15d66d Mon Sep 17 00:00:00 2001 From: doms9 <96013514+doms9@users.noreply.github.com> Date: Mon, 29 Dec 2025 09:57:40 -0500 Subject: [PATCH] e fix scraper crashing if api url(s) do not work misc. edits --- M3U8/scrapers/cdnlivetv.py | 8 +++++--- M3U8/scrapers/embedhd.py | 2 +- M3U8/scrapers/ppv.py | 6 +++--- M3U8/scrapers/streamcenter.py | 4 ++-- M3U8/scrapers/streamsgate.py | 2 +- M3U8/scrapers/strmd.py | 2 +- M3U8/scrapers/totalsportek.py | 7 ++----- M3U8/scrapers/watchfooty.py | 2 +- M3U8/scrapers/webcast.py | 9 ++------- 9 files changed, 18 insertions(+), 24 deletions(-) diff --git a/M3U8/scrapers/cdnlivetv.py b/M3U8/scrapers/cdnlivetv.py index 3907609..ef8677f 100644 --- a/M3U8/scrapers/cdnlivetv.py +++ b/M3U8/scrapers/cdnlivetv.py @@ -18,12 +18,14 @@ API_URL = "https://api.cdn-live.tv/api/v1/events/sports" async def get_events(cached_keys: list[str]) -> list[dict[str, str]]: + now = Time.clean(Time.now()) + events = [] if not (api_data := API_FILE.load(per_entry=False)): log.info("Refreshing API cache") - api_data = {} + api_data = {"timestamp": now.timestamp()} if r := await network.request( API_URL, @@ -32,13 +34,13 @@ async def get_events(cached_keys: list[str]) -> list[dict[str, str]]: ): api_data: dict = r.json() + api_data["timestamp"] = now.timestamp() + API_FILE.write(api_data) if not (data := api_data.get("cdn-live-tv")): return events - now = Time.clean(Time.now()) - start_dt = now.delta(minutes=-30) end_dt = now.delta(minutes=30) diff --git a/M3U8/scrapers/embedhd.py b/M3U8/scrapers/embedhd.py index c959f38..12d7615 100644 --- a/M3U8/scrapers/embedhd.py +++ b/M3U8/scrapers/embedhd.py @@ -27,7 +27,7 @@ async def get_events(cached_keys: list[str]) -> list[dict[str, str]]: if not (api_data := API_CACHE.load(per_entry=False)): log.info("Refreshing API cache") - api_data = {} + api_data = {"timestamp": now.timestamp()} if r := await network.request(BASE_URL, log=log): api_data: dict = r.json() diff --git a/M3U8/scrapers/ppv.py b/M3U8/scrapers/ppv.py index ef5f18b..4449116 100644 --- a/M3U8/scrapers/ppv.py +++ b/M3U8/scrapers/ppv.py @@ -28,19 +28,19 @@ BASE_MIRRORS = [ async def get_events(api_url: str, cached_keys: list[str]) -> list[dict[str, str]]: - events = [] + now = Time.clean(Time.now()) if not (api_data := API_FILE.load(per_entry=False)): log.info("Refreshing API cache") - api_data = {} + api_data = {"timestamp": now.timestamp()} if r := await network.request(api_url, log=log): api_data: dict = r.json() API_FILE.write(api_data) - now = Time.clean(Time.now()) + events = [] start_dt = now.delta(minutes=-30) end_dt = now.delta(minutes=30) diff --git a/M3U8/scrapers/streamcenter.py b/M3U8/scrapers/streamcenter.py index bc1de7c..0276b50 100644 --- a/M3U8/scrapers/streamcenter.py +++ b/M3U8/scrapers/streamcenter.py @@ -14,7 +14,7 @@ CACHE_FILE = Cache(f"{TAG.lower()}.json", exp=10_800) API_FILE = Cache(f"{TAG.lower()}-api.json", exp=28_800) -BASE_URL = "https://backendstreamcenter.youshop.pro:488/api/Parties" +BASE_URL = "https://backend.streamcenter.live/api/Parties" CATEGORIES = { 4: "Basketball", @@ -37,7 +37,7 @@ async def get_events(cached_keys: list[str]) -> list[dict[str, str]]: if not (api_data := API_FILE.load(per_entry=False, index=-1)): log.info("Refreshing API cache") - api_data = [] + api_data = [{"timestamp": now.timestamp()}] if r := await network.request( BASE_URL, diff --git a/M3U8/scrapers/streamsgate.py b/M3U8/scrapers/streamsgate.py index a056d4c..5709604 100644 --- a/M3U8/scrapers/streamsgate.py +++ b/M3U8/scrapers/streamsgate.py @@ -59,7 +59,7 @@ async def refresh_api_cache(now_ts: float) -> list[dict[str, Any]]: results = await asyncio.gather(*tasks) if not (data := [*chain.from_iterable(r.json() for r in results if r)]): - return [] + return [{"timestamp": now_ts}] for ev in data: ev["ts"] = ev.pop("timestamp") diff --git a/M3U8/scrapers/strmd.py b/M3U8/scrapers/strmd.py index 30b63ae..69daaf4 100644 --- a/M3U8/scrapers/strmd.py +++ b/M3U8/scrapers/strmd.py @@ -39,7 +39,7 @@ async def get_events(url: str, cached_keys: list[str]) -> list[dict[str, str]]: if not (api_data := API_FILE.load(per_entry=False, index=-1)): log.info("Refreshing API cache") - api_data = [] + api_data = [{"timestamp": now.timestamp()}] if r := await network.request( urljoin(url, "api/matches/all-today"), diff --git a/M3U8/scrapers/totalsportek.py b/M3U8/scrapers/totalsportek.py index 9a86bf6..8ec5c48 100644 --- a/M3U8/scrapers/totalsportek.py +++ b/M3U8/scrapers/totalsportek.py @@ -82,13 +82,10 @@ async def get_events(cached_keys: list[str]) -> list[dict[str, str]]: if not (time_node := node.css_first(".col-3")): continue - if not time_node.text(strip=True) == "MatchStarted": + if time_node.text(strip=True) != "MatchStarted": continue - if not (href := node.attributes.get("href")): - continue - - if href.startswith("http"): + if not (href := node.attributes.get("href")) or href.startswith("http"): continue sport = fix_league(sport) diff --git a/M3U8/scrapers/watchfooty.py b/M3U8/scrapers/watchfooty.py index 3180668..612b4af 100644 --- a/M3U8/scrapers/watchfooty.py +++ b/M3U8/scrapers/watchfooty.py @@ -60,7 +60,7 @@ async def refresh_api_cache(now: Time) -> list[dict[str, Any]]: results = await asyncio.gather(*tasks) if not (data := [*chain.from_iterable(r.json() for r in results if r)]): - return [] + return [{"timestamp": now.timestamp()}] for ev in data: ev["ts"] = ev.pop("timestamp") diff --git a/M3U8/scrapers/webcast.py b/M3U8/scrapers/webcast.py index 92e06f5..439bbf5 100644 --- a/M3U8/scrapers/webcast.py +++ b/M3U8/scrapers/webcast.py @@ -1,4 +1,3 @@ -import asyncio from functools import partial from playwright.async_api import async_playwright @@ -33,10 +32,6 @@ async def refresh_html_cache() -> dict[str, dict[str, str | float]]: soup = HTMLParser(html_data.content) - title = soup.css_first("title").text(strip=True) - - sport = "NFL" if "NFL" in title else "NHL" - date_text = now.strftime("%B %d, %Y") if date_row := soup.css_first("tr.mdatetitle"): @@ -66,10 +61,10 @@ async def refresh_html_cache() -> dict[str, dict[str, str | float]]: event = fix_event(event_name) - key = f"[{sport}] {event} ({TAG})" + key = f"[NHL] {event} ({TAG})" events[key] = { - "sport": sport, + "sport": "NHL", "event": event, "link": href, "event_ts": event_dt.timestamp(),