e

M3U8/scrapers/streamed.py

@@ -1,6 +1,5 @@
 import asyncio
 import re
-from datetime import datetime, timedelta
 from functools import partial
 from pathlib import Path
 
@@ -8,40 +7,22 @@ import httpx
 from playwright.async_api import async_playwright
 from selectolax.parser import HTMLParser
 
-from .utils import (
-    TZ,
-    capture_req,
-    get_logger,
-    leagues,
-    load_cache,
-    new_browser,
-    now,
-    safe_process_event,
-    write_cache,
-)
+from .utils import Cache, Time, get_logger, leagues, network
 
 log = get_logger(__name__)
 
 urls: dict[str, dict[str, str | float]] = {}
 
-CACHE_FILE = Path(__file__).parent / "caches" / "streamed.json"
+CACHE_FILE = Cache(Path(__file__).parent / "caches" / "streamed.json", exp=10_800)
 
-HTML_CACHE = Path(__file__).parent / "caches" / "streamed_php.json"
+HTML_CACHE = Cache(Path(__file__).parent / "caches" / "streamed_php.json", exp=86_400)
 
-base_url = "https://streamed.site/webmaster.php"
-
-
-def get_date(s: str) -> datetime:
-    try:
-        return datetime.strptime(s, "%Y-%m-%d %H:%M %Z").astimezone(TZ)
-    except ValueError:
-        s = s.replace("ET", "").strip()
-        return datetime.strptime(s, "%Y-%m-%d %H:%M").astimezone(TZ)
+BASE_URL = "https://streamed.site/webmaster.php"
 
 
 async def process_event(url: str, url_num: int) -> str | None:
     async with async_playwright() as p:
-        browser, context = await new_browser(p, browser="brave")
+        browser, context = await network.browser(p, browser="brave")
 
         page = await context.new_page()
 
@@ -49,7 +30,7 @@ async def process_event(url: str, url_num: int) -> str | None:
 
         got_one = asyncio.Event()
 
-        handler = partial(capture_req, captured=captured, got_one=got_one)
+        handler = partial(network.capture_req, captured=captured, got_one=got_one)
 
         page.on("request", handler)
 
@@ -102,12 +83,13 @@ async def refresh_html_cache(client: httpx.AsyncClient, url: str) -> dict[str, s
 
     soup = HTMLParser(r.text)
     events = {}
+    now = Time.now().to_tz("EST")
 
     for row in soup.css("div.wrap div.row"):
         if not (date := row.css_first("div.date")):
             continue
 
-        event_dt = get_date(date.text(strip=True))
+        event_dt = Time.from_str(date.text(strip=True)).to_tz("EST")
 
         if event_dt.date() != now.date():
             continue
@@ -134,8 +116,8 @@ async def refresh_html_cache(client: httpx.AsyncClient, url: str) -> dict[str, s
             "sport": sport,
             "event": event,
             "link": f"https://streamed.site/set.php?{m[1]}",
-            "event_ts": event_dt.timestamp(),
             "timestamp": now.timestamp(),
+            "event_ts": event_dt.timestamp(),
         }
 
     return events
@@ -147,13 +129,15 @@ async def get_events(
     cached_keys: set[str],
 ) -> list[dict[str, str]]:
 
-    if not (events := load_cache(HTML_CACHE, exp=86_400, nearest_hr=True)):
+    if not (events := HTML_CACHE.load(nearest_hr=True)):
         events = await refresh_html_cache(client, url)
-        write_cache(HTML_CACHE, events)
+        HTML_CACHE.write(events)
 
     live = []
-    start_ts = (now - timedelta(minutes=30)).timestamp()
-    end_ts = (now + timedelta(minutes=30)).timestamp()
+    now = Time.now().to_tz("EST")
+
+    start_ts = now.delta(minutes=-30).to_tz("EST").timestamp()
+    end_ts = now.delta(minutes=30).to_tz("EST").timestamp()
 
     for k, v in events.items():
         if cached_keys & {k}:
@@ -168,31 +152,31 @@ async def get_events(
 
 
 async def scrape(client: httpx.AsyncClient) -> None:
-    cached_urls = load_cache(CACHE_FILE, exp=10_800)
+    cached_urls = CACHE_FILE.load()
     cached_count = len(cached_urls)
     urls.update(cached_urls)
 
-    log.info(f"Collected {cached_count} event(s) from cache")
+    log.info(f"Loaded {cached_count} event(s) from cache")
 
-    log.info(f'Scraping from "{base_url}"')
+    log.info(f'Scraping from "{BASE_URL}"')
 
     events = await get_events(
         client,
-        base_url,
+        BASE_URL,
         set(cached_urls.keys()),
     )
 
     log.info(f"Processing {len(events)} new URL(s)")
 
     for i, ev in enumerate(events, start=1):
-        url = await safe_process_event(
+        url = await network.safe_process(
             lambda: process_event(ev["link"], url_num=i),
             url_num=i,
             log=log,
         )
 
         if url:
-            sport, event = ev["sport"], ev["event"]
+            sport, event, ts = ev["sport"], ev["event"], ev["timestamp"]
 
             tvg_id, logo = leagues.info(sport)
 
@@ -202,8 +186,8 @@ async def scrape(client: httpx.AsyncClient) -> None:
                 "url": url,
                 "logo": logo,
                 "base": "https://streamed.site/",
-                "timestamp": now.timestamp(),
-                "id": tvg_id,
+                "timestamp": ts,
+                "id": tvg_id or "Live.Event.us",
             }
 
             urls[key] = cached_urls[key] = entry
@@ -213,4 +197,4 @@ async def scrape(client: httpx.AsyncClient) -> None:
     else:
         log.info("No new events found")
 
-    write_cache(CACHE_FILE, cached_urls)
+    CACHE_FILE.write(cached_urls)