health log

- edit scraping for mainportal.py

M3U8/scrapers/mainportal.py

@@ -1,4 +1,6 @@
 import asyncio
+import json
+import re
 from functools import partial
 from urllib.parse import urljoin
 
@@ -53,35 +55,30 @@ async def process_event(
     return m3u8_url
 
 
-async def get_api_data() -> dict[str, dict[str, list[dict]]]:
-    tasks = [
-        (
-            sport,
-            network.request(
-                urljoin(url, "api/v2/stateshot"),
-                log=log,
-            ),
-        )
-        for sport, url in API_URLS.items()
-    ]
-
-    results = await asyncio.gather(*(task for _, task in tasks))
-
-    return {sport: r.json() for (sport, _), r in zip(tasks, results) if r}
-
-
 async def get_events(cached_keys: list[str]) -> list[dict[str, str]]:
-    now = Time.clean(Time.now())
+    tasks = [network.request(url, log=log) for url in BASE_URLS.values()]
 
-    api_data = await get_api_data()
+    results = await asyncio.gather(*tasks)
 
     events = []
 
+    if not (html_data := [(html.text, html.url) for html in results if html]):
+        return events
+
+    now = Time.clean(Time.now())
+
+    stateshot_ptrn = re.compile(r"var\s+stateshot\s+=\s+(.*);", re.I)
+
     start_dt = now.delta(hours=-1)
     end_dt = now.delta(minutes=1)
 
-    for sport in api_data:
+    for content, url in html_data:
-        data = api_data[sport]
+        sport = next((k for k, v in BASE_URLS.items() if v == url), "Live Event")
+
+        if not (match := stateshot_ptrn.search(content)):
+            continue
+
+        data: dict = json.loads(f"{match[1]}")
 
         teams = data.get("teams", {})
 

M3U8/scrapers/watchfooty.py

@@ -1,277 +1,277 @@
-import asyncio
+import asyncio
-import json
+import json
-from functools import partial
+from functools import partial
-from typing import Any
+from typing import Any
-from urllib.parse import urlencode, urljoin
+from urllib.parse import urlencode, urljoin
-
+
-from playwright.async_api import Browser, Page
+from playwright.async_api import Browser, Page
-
+
-from .utils import Cache, Time, get_logger, leagues, network
+from .utils import Cache, Time, get_logger, leagues, network
-
+
-log = get_logger(__name__)
+log = get_logger(__name__)
-
+
-urls: dict[str, dict[str, str | float]] = {}
+urls: dict[str, dict[str, str | float]] = {}
-
+
-TAG = "WATCHFTY"
+TAG = "WATCHFTY"
-
+
-CACHE_FILE = Cache(TAG, exp=10_800)
+CACHE_FILE = Cache(TAG, exp=10_800)
-
+
-BASE_DOMAIN = "watchfooty.st"
+BASE_DOMAIN = "watchfooty.ru"
-
+
-API_URL, BASE_URL = f"https://api.{BASE_DOMAIN}", f"https://www.{BASE_DOMAIN}"
+API_URL, BASE_URL = f"https://api.{BASE_DOMAIN}", f"https://www.{BASE_DOMAIN}"
-
+
-
+
-def build_wfty_url(live: bool, event_id: str = None) -> str:
+def build_wfty_url(live: bool, event_id: str = None) -> str:
-    url = urljoin(
+    url = urljoin(
-        API_URL,
+        API_URL,
-        (
+        (
-            "_internal/trpc/sports.getSportsLiveMatchesCount,sports.getPopularMatches,sports.getPopularLiveMatches"
+            "_internal/trpc/sports.getSportsLiveMatchesCount,sports.getPopularMatches,sports.getPopularLiveMatches"
-            if live
+            if live
-            else "_internal/trpc/sports.getSportsLiveMatchesCount,sports.getMatchById"
+            else "_internal/trpc/sports.getSportsLiveMatchesCount,sports.getMatchById"
-        ),
+        ),
-    )
+    )
-
+
-    input_data = {
+    input_data = {
-        "0": {
+        "0": {
-            "json": {
+            "json": {
-                "start": (now := Time.now()).isoformat(),
+                "start": (now := Time.now()).isoformat(),
-                "end": now.delta(days=1).isoformat(),
+                "end": now.delta(days=1).isoformat(),
-            }
+            }
-        },
+        },
-    } | (
+    } | (
-        {
+        {
-            "1": {
+            "1": {
-                "json": None,
+                "json": None,
-                "meta": {"values": ["undefined"]},
+                "meta": {"values": ["undefined"]},
-            },
+            },
-            "2": {
+            "2": {
-                "json": None,
+                "json": None,
-                "meta": {"values": ["undefined"]},
+                "meta": {"values": ["undefined"]},
-            },
+            },
-        }
+        }
-        if live
+        if live
-        else {
+        else {
-            "1": {
+            "1": {
-                "json": {
+                "json": {
-                    "id": event_id,
+                    "id": event_id,
-                    "withoutAdditionalInfo": True,
+                    "withoutAdditionalInfo": True,
-                    "withoutLinks": False,
+                    "withoutLinks": False,
-                }
+                }
-            }
+            }
-        }
+        }
-    )
+    )
-
+
-    params = {
+    params = {
-        "batch": "1",
+        "batch": "1",
-        "input": json.dumps(input_data, separators=(",", ":")),
+        "input": json.dumps(input_data, separators=(",", ":")),
-    }
+    }
-
+
-    return f"{url}?{urlencode(params)}"
+    return f"{url}?{urlencode(params)}"
-
+
-
+
-async def pre_process(url: str, url_num: int) -> str | None:
+async def pre_process(url: str, url_num: int) -> str | None:
-    if not (event_data := await network.request(url, log=log)):
+    if not (event_data := await network.request(url, log=log)):
-        return
+        return
-
+
-    api_data: dict = (
+    api_data: dict = (
-        (event_data.json() or [{}])[-1].get("result", {}).get("data", {}).get("json")
+        (event_data.json() or [{}])[-1].get("result", {}).get("data", {}).get("json")
-    )
+    )
-
+
-    if not api_data:
+    if not api_data:
-        log.warning(f"URL {url_num}) No API data found.")
+        log.warning(f"URL {url_num}) No API data found.")
-        return
+        return
-
+
-    if not (links := api_data.get("fixtureData", {}).get("links")):
+    if not (links := api_data.get("fixtureData", {}).get("links")):
-        log.warning(f"URL {url_num}) No stream links found.")
+        log.warning(f"URL {url_num}) No stream links found.")
-        return
+        return
-
+
-    quality_links: list[dict] = sorted(
+    quality_links: list[dict] = sorted(
-        [link for link in links if (wld := link.get("wld")) and "e" not in wld],
+        [link for link in links if (wld := link.get("wld")) and "e" not in wld],
-        key=lambda x: x.get("viewerCount") or -1,
+        key=lambda x: x.get("viewerCount") or -1,
-        reverse=True,
+        reverse=True,
-    )
+    )
-
+
-    if not quality_links:
+    if not quality_links:
-        log.warning(f"URL {url_num}) No valid quality link found.")
+        log.warning(f"URL {url_num}) No valid quality link found.")
-        return
+        return
-
+
-    link_data = quality_links[0]
+    link_data = quality_links[0]
-
+
-    embed_path = (
+    embed_path = (
-        link_data["gi"],
+        link_data["gi"],
-        link_data["t"],
+        link_data["t"],
-        link_data["wld"]["cn"],
+        link_data["wld"]["cn"],
-        link_data["wld"]["sn"],
+        link_data["wld"]["sn"],
-    )
+    )
-
+
-    return f"https://sportsembed.su/embed/{'/'.join(embed_path)}?player=clappr&autoplay=true"
+    return f"https://sportsembed.su/embed/{'/'.join(embed_path)}?player=clappr&autoplay=true"
-
+
-
+
-async def process_event(
+async def process_event(
-    url: str,
+    url: str,
-    url_num: int,
+    url_num: int,
-    page: Page,
+    page: Page,
-) -> str | None:
+) -> str | None:
-
+
-    nones = None, None
+    nones = None, None
-
+
-    captured: list[str] = []
+    captured: list[str] = []
-
+
-    got_one = asyncio.Event()
+    got_one = asyncio.Event()
-
+
-    handler = partial(
+    handler = partial(
-        network.capture_req,
+        network.capture_req,
-        captured=captured,
+        captured=captured,
-        got_one=got_one,
+        got_one=got_one,
-    )
+    )
-
+
-    page.on("request", handler)
+    page.on("request", handler)
-
+
-    if not (iframe_url := await pre_process(url, url_num)):
+    if not (iframe_url := await pre_process(url, url_num)):
-        return nones
+        return nones
-
+
-    try:
+    try:
-        resp = await page.goto(
+        resp = await page.goto(
-            iframe_url,
+            iframe_url,
-            wait_until="domcontentloaded",
+            wait_until="domcontentloaded",
-            timeout=6_000,
+            timeout=6_000,
-        )
+        )
-
+
-        if not resp or resp.status != 200:
+        if not resp or resp.status != 200:
-            log.warning(
+            log.warning(
-                f"URL {url_num}) Status Code: {resp.status if resp else 'None'}"
+                f"URL {url_num}) Status Code: {resp.status if resp else 'None'}"
-            )
+            )
-            return nones
+            return nones
-
+
-        wait_task = asyncio.create_task(got_one.wait())
+        wait_task = asyncio.create_task(got_one.wait())
-
+
-        try:
+        try:
-            await asyncio.wait_for(wait_task, timeout=6)
+            await asyncio.wait_for(wait_task, timeout=6)
-        except asyncio.TimeoutError:
+        except asyncio.TimeoutError:
-            log.warning(f"URL {url_num}) Timed out waiting for M3U8.")
+            log.warning(f"URL {url_num}) Timed out waiting for M3U8.")
-            return nones
+            return nones
-
+
-        finally:
+        finally:
-            if not wait_task.done():
+            if not wait_task.done():
-                wait_task.cancel()
+                wait_task.cancel()
-
+
-                try:
+                try:
-                    await wait_task
+                    await wait_task
-                except asyncio.CancelledError:
+                except asyncio.CancelledError:
-                    pass
+                    pass
-
+
-        if captured:
+        if captured:
-            log.info(f"URL {url_num}) Captured M3U8")
+            log.info(f"URL {url_num}) Captured M3U8")
-            return captured[0], iframe_url
+            return captured[0], iframe_url
-
+
-    except Exception as e:
+    except Exception as e:
-        log.warning(f"URL {url_num}) {e}")
+        log.warning(f"URL {url_num}) {e}")
-        return nones
+        return nones
-
+
-    finally:
+    finally:
-        page.remove_listener("request", handler)
+        page.remove_listener("request", handler)
-
+
-
+
-async def get_events(cached_keys: list[str]) -> list[dict[str, str]]:
+async def get_events(cached_keys: list[str]) -> list[dict[str, str]]:
-    events = []
+    events = []
-
+
-    live_url = build_wfty_url(live=True)
+    live_url = build_wfty_url(live=True)
-
+
-    if not (live_data := await network.request(live_url, log=log)):
+    if not (live_data := await network.request(live_url, log=log)):
-        return events
+        return events
-
+
-    api_data: list[dict[str, Any]] = (
+    api_data: list[dict[str, Any]] = (
-        (live_data.json() or [{}])[-1].get("result", {}).get("data", {}).get("json")
+        (live_data.json() or [{}])[-1].get("result", {}).get("data", {}).get("json")
-    )
+    )
-
+
-    if not api_data:
+    if not api_data:
-        return events
+        return events
-
+
-    for link in api_data:
+    for link in api_data:
-        if not link.get("viewerCount"):
+        if not link.get("viewerCount"):
-            continue
+            continue
-
+
-        event_id: str = link["id"]
+        event_id: str = link["id"]
-
+
-        event_league: str = link["league"]
+        event_league: str = link["league"]
-
+
-        event_name: str = link["title"]
+        event_name: str = link["title"]
-
+
-        if f"[{event_league}] {event_name} ({TAG})" in cached_keys:
+        if f"[{event_league}] {event_name} ({TAG})" in cached_keys:
-            continue
+            continue
-
+
-        events.append(
+        events.append(
-            {
+            {
-                "sport": event_league,
+                "sport": event_league,
-                "event": event_name,
+                "event": event_name,
-                "link": build_wfty_url(live=False, event_id=event_id),
+                "link": build_wfty_url(live=False, event_id=event_id),
-            }
+            }
-        )
+        )
-
+
-    return events
+    return events
-
+
-
+
-async def scrape(browser: Browser) -> None:
+async def scrape(browser: Browser) -> None:
-    cached_urls = CACHE_FILE.load()
+    cached_urls = CACHE_FILE.load()
-
+
-    valid_urls = {k: v for k, v in cached_urls.items() if v["url"]}
+    valid_urls = {k: v for k, v in cached_urls.items() if v["url"]}
-
+
-    valid_count = cached_count = len(valid_urls)
+    valid_count = cached_count = len(valid_urls)
-
+
-    urls.update(valid_urls)
+    urls.update(valid_urls)
-
+
-    log.info(f"Loaded {cached_count} event(s) from cache")
+    log.info(f"Loaded {cached_count} event(s) from cache")
-
+
-    log.info(f'Scraping from "{BASE_URL}"')
+    log.info(f'Scraping from "{BASE_URL}"')
-
+
-    if events := await get_events(cached_urls.keys()):
+    if events := await get_events(cached_urls.keys()):
-        log.info(f"Processing {len(events)} new URL(s)")
+        log.info(f"Processing {len(events)} new URL(s)")
-
+
-        now = Time.clean(Time.now())
+        now = Time.clean(Time.now())
-
+
-        async with network.event_context(browser, stealth=False) as context:
+        async with network.event_context(browser, stealth=False) as context:
-            for i, ev in enumerate(events, start=1):
+            for i, ev in enumerate(events, start=1):
-                async with network.event_page(context) as page:
+                async with network.event_page(context) as page:
-                    handler = partial(
+                    handler = partial(
-                        process_event,
+                        process_event,
-                        url=(link := ev["link"]),
+                        url=(link := ev["link"]),
-                        url_num=i,
+                        url_num=i,
-                        page=page,
+                        page=page,
-                    )
+                    )
-
+
-                    url, iframe = await network.safe_process(
+                    url, iframe = await network.safe_process(
-                        handler,
+                        handler,
-                        url_num=i,
+                        url_num=i,
-                        semaphore=network.PW_S,
+                        semaphore=network.PW_S,
-                        log=log,
+                        log=log,
-                        timeout=20,
+                        timeout=20,
-                    )
+                    )
-
+
-                    sport, event = ev["sport"], ev["event"]
+                    sport, event = ev["sport"], ev["event"]
-
+
-                    key = f"[{sport}] {event} ({TAG})"
+                    key = f"[{sport}] {event} ({TAG})"
-
+
-                    tvg_id, logo = leagues.get_tvg_info(sport, event)
+                    tvg_id, logo = leagues.get_tvg_info(sport, event)
-
+
-                    entry = {
+                    entry = {
-                        "url": url,
+                        "url": url,
-                        "logo": logo,
+                        "logo": logo,
-                        "base": iframe,
+                        "base": iframe,
-                        "timestamp": now.timestamp(),
+                        "timestamp": now.timestamp(),
-                        "id": tvg_id or "Live.Event.us",
+                        "id": tvg_id or "Live.Event.us",
-                        "link": link,
+                        "link": link,
-                    }
+                    }
-
+
-                    cached_urls[key] = entry
+                    cached_urls[key] = entry
-
+
-                    if url:
+                    if url:
-                        valid_count += 1
+                        valid_count += 1
-
+
-                        urls[key] = entry
+                        urls[key] = entry
-
+
-        log.info(f"Collected and cached {valid_count - cached_count} new event(s)")
+        log.info(f"Collected and cached {valid_count - cached_count} new event(s)")
-
+
-    else:
+    else:
-        log.info("No new events found")
+        log.info("No new events found")
-
+
-    CACHE_FILE.write(cached_urls)
+    CACHE_FILE.write(cached_urls)

readme.md

@@ -1,7 +1,13 @@
-## Base Log @ 2026-05-15 16:20 UTC
+## Base Log @ 2026-05-16 21:10 UTC
 
-### ✅ Working Streams: 160<br>❌ Dead Streams: 0
+### ✅ Working Streams: 156<br>❌ Dead Streams: 4
 
+| Channel | Error (Code) | Link |
+| ------- | ------------ | ---- |
+| ABC | HTTP Error (404) | `http://stream.cammonitorplus.net/1809/index.m3u8` |
+| Comedy TV | HTTP Error (403) | `http://realsport.scalecdn.co:8080/live/supersonicstreams/cUewZolveU8Jh18jk34BDJ/77411.ts` |
+| Fox | HTTP Error (404) | `http://stream.cammonitorplus.net/1752/index.m3u8` |
+| NBC | HTTP Error (404) | `http://stream.cammonitorplus.net/1812/index.m3u8` |
 ---
 #### Base Channels URL
 ```