e

misc. edits

M3U8/scrapers/istreameast.py

@@ -1,10 +1,9 @@
 import base64
 import re
 
-import httpx
 from selectolax.parser import HTMLParser
 
-from .utils import Cache, Time, get_logger, leagues
+from .utils import Cache, Time, get_logger, leagues, network
 
 log = get_logger(__name__)
 
@@ -17,31 +16,14 @@ CACHE_FILE = Cache(f"{TAG.lower()}.json", exp=3_600)
 BASE_URL = "https://istreameast.app"
 
 
-async def get_html_data(client: httpx.AsyncClient, url: str) -> str:
-    try:
-        r = await client.get(url)
-        r.raise_for_status()
-    except Exception as e:
-        log.error(f'Failed to fetch "{url}": {e}')
-
-        return b""
-
-    return r.text
-
-
-async def process_event(
-    client: httpx.AsyncClient,
-    url: str,
-    url_num: int,
-) -> str | None:
-
+async def process_event(url: str, url_num: int) -> str | None:
     pattern = re.compile(r"source:\s*window\.atob\(\s*'([^']+)'\s*\)", re.IGNORECASE)
 
-    if not (event_data := await get_html_data(client, url)):
-        log.warning(f"URL {url_num}) Failed to load event url.")
+    if not (event_data := await network.request(url, log=log)):
+        log.info(f"URL {url_num}) Failed to load url.")
         return
 
-    soup = HTMLParser(event_data)
+    soup = HTMLParser(event_data.content)
 
     if not (iframe := soup.css_first("iframe#wp_player")):
         log.warning(f"URL {url_num}) No iframe element found.")
@@ -51,11 +33,11 @@ async def process_event(
         log.warning(f"URL {url_num}) No iframe source found.")
         return
 
-    if not (iframe_src_data := await get_html_data(client, iframe_src)):
-        log.warning(f"URL {url_num}) Failed to load iframe source.")
+    if not (iframe_src_data := await network.request(iframe_src, log=log)):
+        log.info(f"URL {url_num}) Failed to load iframe source.")
         return
 
-    if not (match := pattern.search(iframe_src_data)):
+    if not (match := pattern.search(iframe_src_data.text)):
         log.warning(f"URL {url_num}) No Clappr source found.")
         return
 
@@ -63,16 +45,15 @@ async def process_event(
     return base64.b64decode(match[1]).decode("utf-8")
 
 
-async def get_events(
-    client: httpx.AsyncClient, cached_keys: set[str]
-) -> list[dict[str, str]]:
+async def get_events(cached_keys: list[str]) -> list[dict[str, str]]:
+    events = []
+
+    if not (html_data := await network.request(BASE_URL, log=log)):
+        return events
+
     pattern = re.compile(r"^(?:LIVE|\d+\s+(minutes?)\b)", re.IGNORECASE)
 
-    html_data = await get_html_data(client, BASE_URL)
-
-    soup = HTMLParser(html_data)
-
-    events = []
+    soup = HTMLParser(html_data.content)
 
     for link in soup.css("li.f1-podium--item > a.f1-podium--link"):
         li_item = link.parent
@@ -90,6 +71,9 @@ async def get_events(
         if inner_span := driver_elem.css_first("span.d-md-inline"):
             event_name = inner_span.text(strip=True)
 
+        if f"[{sport}] {event_name} ({TAG})" in cached_keys:
+            continue
+
         if not (href := link.attributes.get("href")):
             continue
 
@@ -101,11 +85,6 @@ async def get_events(
         if not pattern.search(time_text):
             continue
 
-        key = f"[{sport}] {event_name} ({TAG})"
-
-        if cached_keys & {key}:
-            continue
-
         events.append(
             {
                 "sport": sport,
@@ -117,7 +96,7 @@ async def get_events(
     return events
 
 
-async def scrape(client: httpx.AsyncClient) -> None:
+async def scrape() -> None:
     cached_urls = CACHE_FILE.load()
     cached_count = len(cached_urls)
     urls.update(cached_urls)
@@ -126,7 +105,7 @@ async def scrape(client: httpx.AsyncClient) -> None:
 
     log.info(f'Scraping from "{BASE_URL}"')
 
-    events = await get_events(client, set(cached_urls.keys()))
+    events = await get_events(cached_urls.keys())
 
     log.info(f"Processing {len(events)} new URL(s)")
 
@@ -134,11 +113,7 @@ async def scrape(client: httpx.AsyncClient) -> None:
         now = Time.clean(Time.now()).timestamp()
 
         for i, ev in enumerate(events, start=1):
-            if url := await process_event(
-                client,
-                ev["link"],
-                i,
-            ):
+            if url := await process_event(ev["link"], i):
                 sport, event, link = (
                     ev["sport"],
                     ev["event"],