e

M3U8/fetch.py

@@ -9,6 +9,7 @@ from scrapers import (
     pixel,
     ppv,
     roxie,
+    shark,
     streambtw,
     streameast,
     streamfree,
@@ -49,6 +50,7 @@ async def main() -> None:
         asyncio.create_task(pixel.scrape()),
         asyncio.create_task(ppv.scrape(network.client)),
         asyncio.create_task(roxie.scrape(network.client)),
+        asyncio.create_task(shark.scrape(network.client)),
         asyncio.create_task(streambtw.scrape(network.client)),
         asyncio.create_task(streameast.scrape(network.client)),
         asyncio.create_task(streamfree.scrape(network.client)),
@@ -66,6 +68,7 @@ async def main() -> None:
         | pixel.urls
         | ppv.urls
         | roxie.urls
+        | shark.urls
         | streambtw.urls
         | streameast.urls
         | strmd.urls

M3U8/scrapers/pixel.py

@@ -11,15 +11,10 @@ urls: dict[str, dict[str, str | float]] = {}
 
 CACHE_FILE = Cache("pixel.json", exp=86_400)
 
-API_FILE = Cache("pixel-api.json", exp=86_400)
-
 BASE_URL = "https://pixelsport.tv/backend/livetv/events"
 
 
-async def refresh_api_cache(
+async def get_api_data(url: str) -> dict[str, list[dict, str, str]]:
-    url: str,
-    ts: float,
-) -> dict[str, list[dict, str, str]]:
     log.info("Refreshing API cache")
 
     async with async_playwright() as p:
@@ -44,23 +39,13 @@ async def refresh_api_cache(
         finally:
             await browser.close()
 
-    data = json.loads(raw_json)
+    return json.loads(raw_json)
-
-    data["timestamp"] = ts
-
-    return data
 
 
-async def get_events(cached_keys: set[str]) -> dict[str, str | float]:
+async def get_events() -> dict[str, dict[str, str | float]]:
     now = Time.clean(Time.now())
 
-    if not (api_data := API_FILE.load(per_entry=False)):
+    api_data = await get_api_data(BASE_URL)
-        api_data = await refresh_api_cache(
-            BASE_URL,
-            now.timestamp(),
-        )
-
-        API_FILE.write(api_data)
 
     events = {}
 
@@ -88,9 +73,6 @@ async def get_events(cached_keys: set[str]) -> dict[str, str | float]:
                 if pattern.search(stream_link):
                     key = f"[{sport}] {event_name} {z} (PIXL)"
 
-                    if cached_keys & {key}:
-                        continue
-
                     tvg_id, logo = leagues.get_tvg_info(sport, event_name)
 
                     events[key] = {
@@ -105,23 +87,17 @@ async def get_events(cached_keys: set[str]) -> dict[str, str | float]:
 
 
 async def scrape() -> None:
-    cached_urls = CACHE_FILE.load()
+    if cached := CACHE_FILE.load():
-    cached_count = len(cached_urls)
+        urls.update(cached)
-    urls.update(cached_urls)
+        log.info(f"Loaded {len(urls)} event(s) from cache")
-
+        return
-    log.info(f"Loaded {cached_count} event(s) from cache")
 
     log.info(f'Scraping from "{BASE_URL}"')
 
-    events = await get_events(set(cached_urls.keys()))
+    events = await get_events()
 
-    if events:
+    urls.update(events)
-        for d in (urls, cached_urls):
-            d |= events
 
-    if new_count := len(cached_urls) - cached_count:
+    CACHE_FILE.write(urls)
-        log.info(f"Collected and cached {new_count} new event(s)")
-    else:
-        log.info("No new events found")
 
-    CACHE_FILE.write(cached_urls)
+    log.info(f"Collected and cached {len(urls)} new event(s)")

M3U8/scrapers/roxie.py

@@ -58,7 +58,7 @@ async def refresh_html_cache(
     url: str,
     sport: str,
     now_ts: float,
-) -> dict[str, str | float]:
+) -> dict[str, dict[str, str | float]]:
 
     try:
         r = await client.get(url)
@@ -73,9 +73,7 @@ async def refresh_html_cache(
     events = {}
 
     for row in soup.css("table#eventsTable tbody tr"):
-        a_tag = row.css_first("td a")
+        if not (a_tag := row.css_first("td a")):
-
-        if not a_tag:
             continue
 
         event = a_tag.text(strip=True)

M3U8/scrapers/shark.py

@@ -0,0 +1,183 @@
+import re
+from functools import partial
+
+import httpx
+from selectolax.parser import HTMLParser
+
+from .utils import Cache, Time, get_logger, leagues, network
+
+log = get_logger(__name__)
+
+urls: dict[str, dict[str, str | float]] = {}
+
+CACHE_FILE = Cache("shark.json", exp=10_800)
+
+HTML_CACHE = Cache("shark-html.json", exp=28_800)
+
+BASE_URL = "https://sharkstreams.net"
+
+
+async def process_event(
+    client: httpx.AsyncClient,
+    url: str,
+    url_num: int,
+) -> str | None:
+
+    try:
+        r = await client.get(url)
+        r.raise_for_status()
+    except Exception as e:
+        log.error(f'URL {url_num}) Failed to fetch "{url}": {e}')
+        return
+
+    data: dict[str, list[str]] = r.json()
+
+    if not data.get("urls"):
+        log.info(f"URL {url_num}) No M3U8 found")
+
+        return
+
+    log.info(f"URL {url_num}) Captured M3U8")
+
+    return data["urls"][0]
+
+
+async def refresh_html_cache(
+    client: httpx.AsyncClient,
+    url: str,
+    now_ts: float,
+) -> dict[str, dict[str, str | float]]:
+
+    try:
+        r = await client.get(url)
+        r.raise_for_status()
+    except Exception as e:
+        log.error(f'Failed to fetch "{url}": {e}')
+
+        return {}
+
+    soup = HTMLParser(r.text)
+
+    events = {}
+
+    for row in soup.css(".row"):
+        date_node = row.css_first(".ch-date")
+        sport_node = row.css_first(".ch-category")
+        name_node = row.css_first(".ch-name")
+
+        if not (date_node and sport_node and name_node):
+            continue
+
+        event_dt = Time.from_str(date_node.text(strip=True), timezone="EST")
+        sport = sport_node.text(strip=True)
+        event_name = name_node.text(strip=True)
+
+        embed_btn = row.css_first("a.hd-link.secondary")
+
+        if not embed_btn or not (onclick := embed_btn.attributes.get("onclick")):
+            continue
+
+        pattern = re.compile(r"openEmbed\('([^']+)'\)", re.IGNORECASE)
+
+        if not (match := pattern.search(onclick)):
+            continue
+
+        link = match[1].replace("player.php", "get-stream.php")
+
+        key = f"[{sport}] {event_name} (SHARK)"
+
+        events[key] = {
+            "sport": sport,
+            "event": event_name,
+            "link": link,
+            "event_ts": event_dt.timestamp(),
+            "timestamp": now_ts,
+        }
+
+    return events
+
+
+async def get_events(
+    client: httpx.AsyncClient,
+    cached_keys: set[str],
+) -> list[dict[str, str]]:
+
+    now = Time.clean(Time.now())
+
+    if not (events := HTML_CACHE.load()):
+        events = await refresh_html_cache(
+            client,
+            BASE_URL,
+            now.timestamp(),
+        )
+
+        HTML_CACHE.write(events)
+
+    live = []
+
+    start_ts = now.delta(hours=-1).timestamp()
+    end_ts = now.delta(minutes=10).timestamp()
+
+    for k, v in events.items():
+        if cached_keys & {k}:
+            continue
+
+        if not start_ts <= v["event_ts"] <= end_ts:
+            continue
+
+        live.append({**v})
+
+    return live
+
+
+async def scrape(client: httpx.AsyncClient) -> None:
+    cached_urls = CACHE_FILE.load()
+    cached_count = len(cached_urls)
+    urls.update(cached_urls)
+
+    log.info(f"Loaded {cached_count} event(s) from cache")
+
+    log.info(f'Scraping from "{BASE_URL}"')
+
+    events = await get_events(client, set(cached_urls.keys()))
+
+    log.info(f"Processing {len(events)} new URL(s)")
+
+    if events:
+        for i, ev in enumerate(events, start=1):
+            handler = partial(
+                process_event,
+                client=client,
+                url=ev["link"],
+                url_num=i,
+            )
+
+            url = await network.safe_process(
+                handler,
+                url_num=i,
+                log=log,
+            )
+
+            if url:
+                sport, event, ts = ev["sport"], ev["event"], ev["event_ts"]
+
+                tvg_id, logo = leagues.get_tvg_info(sport, event)
+
+                key = f"[{sport}] {event} (SHARK)"
+
+                entry = {
+                    "url": url,
+                    "logo": logo,
+                    "base": BASE_URL,
+                    "timestamp": ts,
+                    "id": tvg_id or "Live.Event.us",
+                }
+
+                urls[key] = cached_urls[key] = entry
+
+    if new_count := len(cached_urls) - cached_count:
+        log.info(f"Collected and cached {new_count} new event(s)")
+    else:
+        log.info("No new events found")
+
+    CACHE_FILE.write(cached_urls)

M3U8/scrapers/tvpass.py

@@ -13,7 +13,7 @@ CACHE_FILE = Cache("tvpass.json", exp=86_400)
 BASE_URL = "https://tvpass.org/playlist/m3u"
 
 
-async def fetch_m3u8(client: httpx.AsyncClient) -> list[str]:
+async def get_data(client: httpx.AsyncClient) -> list[str]:
     try:
         r = await client.get(BASE_URL)
         r.raise_for_status()
@@ -25,19 +25,12 @@ async def fetch_m3u8(client: httpx.AsyncClient) -> list[str]:
     return r.text.splitlines()
 
 
-async def scrape(client: httpx.AsyncClient) -> None:
+async def get_events(client: httpx.AsyncClient) -> dict[str, dict[str, str | float]]:
-    if cached := CACHE_FILE.load():
-        urls.update(cached)
-        log.info(f"Loaded {len(urls)} event(s) from cache")
-        return
-
-    log.info(f'Scraping from "{BASE_URL}"')
-
     now = Time.now().timestamp()
 
-    if not (data := await fetch_m3u8(client)):
+    events = {}
-        log.warning("No M3U8 data received")
+
-        return
+    data = await get_data(client)
 
     for i, line in enumerate(data):
         if line.startswith("#EXTINF"):
@@ -59,7 +52,7 @@ async def scrape(client: httpx.AsyncClient) -> None:
 
                     tvg_id, logo = leagues.info(sport)
 
-                    entry = {
+                    events[key] = {
                         "url": f"http://origin.thetvapp.to/hls/{channel}/mono.m3u8",
                         "logo": logo,
                         "id": tvg_id or "Live.Event.us",
@@ -67,8 +60,21 @@ async def scrape(client: httpx.AsyncClient) -> None:
                         "timestamp": now,
                     }
 
-                    urls[key] = entry
+    return events
+
+
+async def scrape(client: httpx.AsyncClient) -> None:
+    if cached := CACHE_FILE.load():
+        urls.update(cached)
+        log.info(f"Loaded {len(urls)} event(s) from cache")
+        return
+
+    log.info(f'Scraping from "{BASE_URL}"')
+
+    events = await get_events(client)
+
+    urls.update(events)
 
     CACHE_FILE.write(urls)
 
-    log.info(f"Cached {len(urls)} event(s)")
+    log.info(f"Collected and cached {len(urls)} new event(s)")

M3U8/scrapers/volo.py

@@ -115,7 +115,7 @@ async def refresh_html_cache(
     client: httpx.AsyncClient,
     url: str,
     sport: str,
-) -> dict[str, str | float]:
+) -> dict[str, dict[str, str | float]]:
 
     try:
         r = await client.get(url)