e

replace sport9 with sportzone

M3U8/fetch.py

@@ -16,7 +16,7 @@ from scrapers import (
     ppv,
     roxie,
     shark,
-    sport9,
+    sportzone,
     streambtw,
     streamcenter,
     streamhub,
@@ -70,7 +70,7 @@ async def main() -> None:
                 asyncio.create_task(pixel.scrape(hdl_brwsr)),
                 asyncio.create_task(ppv.scrape(xtrnl_brwsr)),
                 asyncio.create_task(roxie.scrape(hdl_brwsr)),
-                # asyncio.create_task(sport9.scrape(xtrnl_brwsr)),
+                asyncio.create_task(sportzone.scrape(xtrnl_brwsr)),
                 asyncio.create_task(streamcenter.scrape(hdl_brwsr)),
                 # asyncio.create_task(streamhub.scrape(xtrnl_brwsr)),
                 asyncio.create_task(streamsgate.scrape(xtrnl_brwsr)),
@@ -117,7 +117,7 @@ async def main() -> None:
         | ppv.urls
         | roxie.urls
         | shark.urls
-        | sport9.urls
+        | sportzone.urls
         | streambtw.urls
         | streamcenter.urls
         | streamhub.urls

M3U8/scrapers/sport9.py

@@ -1,148 +0,0 @@
-import asyncio
-from functools import partial
-from urllib.parse import urljoin
-
-from playwright.async_api import Browser
-from selectolax.parser import HTMLParser
-
-from .utils import Cache, Time, get_logger, leagues, network
-
-log = get_logger(__name__)
-
-urls: dict[str, dict[str, str | float]] = {}
-
-TAG = "SPORT9"
-
-CACHE_FILE = Cache(TAG, exp=5_400)
-
-BASE_URL = "https://sport9.ru"
-
-
-async def get_events(cached_keys: list[str]) -> list[dict[str, str]]:
-    now = Time.now()
-
-    tasks = [
-        network.request(
-            BASE_URL,
-            log=log,
-            params={"date": d.date()},
-        )
-        for d in [
-            now.delta(days=-1),
-            now,
-            now.delta(days=1),
-        ]
-    ]
-
-    results = await asyncio.gather(*tasks)
-
-    events = []
-
-    if not (soups := [HTMLParser(html.content) for html in results if html]):
-        return events
-
-    for soup in soups:
-        for card in soup.css("a.match-card"):
-            live_badge = card.css_first(".live-badge")
-
-            if not live_badge or live_badge.text(strip=True).lower() != "live":
-                continue
-
-            if not (sport_node := card.css_first(".tournament-name")):
-                continue
-
-            sport = sport_node.text(strip=True)
-
-            team_1_node = card.css_first(".team1 .team-name")
-
-            team_2_node = card.css_first(".team2 .team-name")
-
-            if team_1_node and team_2_node:
-                event = event = (
-                    f"{team_1_node.text(strip=True)} vs {team_2_node.text(strip=True)}"
-                )
-
-            elif team_1_node:
-                event = team_1_node.text(strip=True)
-
-            elif team_2_node:
-                event = team_2_node.text(strip=True)
-
-            else:
-                continue
-
-            if f"[{sport}] {event} ({TAG})" in cached_keys:
-                continue
-
-            if not (href := card.attributes.get("href")):
-                continue
-
-            events.append(
-                {
-                    "sport": sport,
-                    "event": event,
-                    "link": urljoin(BASE_URL, href),
-                }
-            )
-
-    return events
-
-
-async def scrape(browser: Browser) -> None:
-    cached_urls = CACHE_FILE.load()
-
-    cached_count = len(cached_urls)
-
-    urls.update(cached_urls)
-
-    log.info(f"Loaded {cached_count} event(s) from cache")
-
-    log.info(f'Scraping from "{BASE_URL}"')
-
-    if events := await get_events(cached_urls.keys()):
-        log.info(f"Processing {len(events)} new URL(s)")
-
-        now = Time.clean(Time.now())
-
-        async with network.event_context(browser, stealth=False) as context:
-            for i, ev in enumerate(events, start=1):
-                async with network.event_page(context) as page:
-                    handler = partial(
-                        network.process_event,
-                        url=(link := ev["link"]),
-                        url_num=i,
-                        page=page,
-                        log=log,
-                    )
-
-                    url = await network.safe_process(
-                        handler,
-                        url_num=i,
-                        semaphore=network.PW_S,
-                        log=log,
-                    )
-
-                    if url:
-                        sport, event = ev["sport"], ev["event"]
-
-                        key = f"[{sport}] {event} ({TAG})"
-
-                        tvg_id, logo = leagues.get_tvg_info(sport, event)
-
-                        entry = {
-                            "url": url,
-                            "logo": logo,
-                            "base": "https://vividmosaica.com/",
-                            "timestamp": now.timestamp(),
-                            "id": tvg_id or "Live.Event.us",
-                            "link": link,
-                        }
-
-                        urls[key] = cached_urls[key] = entry
-
-        log.info(f"Collected and cached {len(cached_urls) - cached_count} new event(s)")
-
-    else:
-        log.info("No new events found")
-
-    CACHE_FILE.write(cached_urls)

M3U8/scrapers/sportzone.py

@@ -0,0 +1,154 @@
+from functools import partial
+from typing import Any
+
+from playwright.async_api import Browser
+
+from .utils import Cache, Time, get_logger, leagues, network
+
+log = get_logger(__name__)
+
+urls: dict[str, dict[str, str | float]] = {}
+
+TAG = "SPRTZONE"
+
+CACHE_FILE = Cache(TAG, exp=10_800)
+
+API_FILE = Cache(f"{TAG}-api", exp=19_800)
+
+API_URL = "https://sportzone.su/data.json"
+
+
+async def refresh_api_cache(now_ts: float) -> list[dict[str, Any]]:
+    api_data = [{"timestamp": now_ts}]
+
+    if r := await network.request(API_URL, log=log):
+        api_data: list[dict] = r.json().get("matches", [])
+
+        if api_data:
+            for event in api_data:
+                event["ts"] = event.pop("timestamp")
+
+        api_data[-1]["timestamp"] = now_ts
+
+    return api_data
+
+
+async def get_events(cached_keys: list[str]) -> list[dict[str, str]]:
+    now = Time.clean(Time.now())
+
+    if not (api_data := API_FILE.load(per_entry=False, index=-1)):
+        log.info("Refreshing API cache")
+
+        api_data = await refresh_api_cache(now.timestamp())
+
+        API_FILE.write(api_data)
+
+    events = []
+
+    start_dt = now.delta(hours=-2.5)
+    end_dt = now.delta(minutes=30)
+
+    for stream_group in api_data:
+        sport = stream_group.get("league")
+
+        team_1, team_2 = stream_group.get("team1"), stream_group.get("team2")
+
+        if not (sport and team_1 and team_2):
+            continue
+
+        event_name = f"{team_1} vs {team_2}"
+
+        if f"[{sport}] {event_name} ({TAG})" in cached_keys:
+            continue
+
+        if not (event_ts := stream_group.get("ts")):
+            continue
+
+        event_dt = Time.from_ts(int(f"{event_ts}"[:-3]))
+
+        if not start_dt <= event_dt <= end_dt:
+            continue
+
+        if not (event_channels := stream_group.get("channels")):
+            continue
+
+        if not (event_links := event_channels[0].get("links")):
+            continue
+
+        event_url: str = event_links[0]
+
+        events.append(
+            {
+                "sport": sport,
+                "event": event_name,
+                "link": event_url,
+            }
+        )
+
+    return events
+
+
+async def scrape(browser: Browser) -> None:
+    cached_urls = CACHE_FILE.load()
+
+    valid_urls = {k: v for k, v in cached_urls.items() if v["url"]}
+
+    valid_count = cached_count = len(valid_urls)
+
+    urls.update(valid_urls)
+
+    log.info(f"Loaded {cached_count} event(s) from cache")
+
+    log.info('Scraping from "https://sportzone.su"')
+
+    if events := await get_events(cached_urls.keys()):
+        log.info(f"Processing {len(events)} new URL(s)")
+
+        now = Time.clean(Time.now())
+
+        async with network.event_context(browser, stealth=False) as context:
+            for i, ev in enumerate(events, start=1):
+                async with network.event_page(context) as page:
+                    handler = partial(
+                        network.process_event,
+                        url=(link := ev["link"]),
+                        url_num=i,
+                        page=page,
+                        log=log,
+                    )
+
+                    url = await network.safe_process(
+                        handler,
+                        url_num=i,
+                        semaphore=network.PW_S,
+                        log=log,
+                    )
+
+                    sport, event = ev["sport"], ev["event"]
+
+                    key = f"[{sport}] {event} ({TAG})"
+
+                    tvg_id, logo = leagues.get_tvg_info(sport, event)
+
+                    entry = {
+                        "url": url,
+                        "logo": logo,
+                        "base": "https://vividmosaica.com/",
+                        "timestamp": now.timestamp(),
+                        "id": tvg_id or "Live.Event.us",
+                        "link": link,
+                    }
+
+                    cached_urls[key] = entry
+
+                    if url:
+                        valid_count += 1
+
+                        urls[key] = entry
+
+        log.info(f"Collected and cached {valid_count - cached_count} new event(s)")
+
+    else:
+        log.info("No new events found")
+
+    CACHE_FILE.write(cached_urls)