From 00000d978872bafb6492acd9f4b5b15e093a353a Mon Sep 17 00:00:00 2001 From: doms9 <96013514+doms9@users.noreply.github.com> Date: Sun, 5 Apr 2026 17:26:17 -0400 Subject: [PATCH] e - edit scraping for streamhub.py - edit scraping for streamsgate.py - misc edits. --- M3U8/scrapers/streamhub.py | 134 +++++++++++++++++++++++----------- M3U8/scrapers/streamsgate.py | 113 +++++++++++++++++----------- M3U8/scrapers/totalsportek.py | 26 ++----- 3 files changed, 171 insertions(+), 102 deletions(-) diff --git a/M3U8/scrapers/streamhub.py b/M3U8/scrapers/streamhub.py index 21e23402..055686a8 100644 --- a/M3U8/scrapers/streamhub.py +++ b/M3U8/scrapers/streamhub.py @@ -1,8 +1,8 @@ import asyncio +import re from functools import partial -from urllib.parse import urljoin +from urllib.parse import urljoin, urlparse -from playwright.async_api import Browser from selectolax.parser import HTMLParser from .utils import Cache, Time, get_logger, leagues, network @@ -34,6 +34,61 @@ SPORT_ENDPOINTS = [ ] +async def process_event(url: str, url_num: int) -> tuple[str | None, str | None]: + if not (event_data := await network.request(url, log=log)): + log.warning(f"URL {url_num}) Failed to load url.") + return + + soup_1 = HTMLParser(event_data.content) + + ifr_1 = soup_1.css_first("iframe#playerIframe") + + if not ifr_1 or not (src := ifr_1.attributes.get("src")): + log.warning(f"URL {url_num}) No iframe element found.") + return + + parsed = urlparse(src) + + ifr_1_src = urljoin( + BASE_URL, + f"embed1/{parsed.path.split('/')[-1].split('_')[0]}.php", + ) + + if not ( + ifr_1_src_data := await network.request( + ifr_1_src, + headers={"Referer": url}, + log=log, + ) + ): + log.warning(f"URL {url_num}) Failed to load iframe source. (IFR1)") + return + + soup_2 = HTMLParser(ifr_1_src_data.content) + + ifr_2 = soup_2.css_first("center iframe") + + if not ifr_2 or not (ifr_2_src := ifr_2.attributes.get("src")): + log.warning(f"URL {url_num}) Unable to locate iframe. (IFR2)") + return + + ifr_2_src = f"https:{ifr_2_src}" if ifr_2_src.startswith("//") else ifr_2_src + + if not (ifr_2_src_data := await network.request(ifr_2_src, log=log)): + log.warning(f"URL {url_num}) Failed to load iframe source.") + return + + valid_m3u8 = re.compile(r"src:\s+(\'|\")([^\']+)(\'|\")", re.I) + + if not (match := valid_m3u8.search(ifr_2_src_data.text)): + log.warning(f"URL {url_num}) No source found.") + return + + log.info(f"URL {url_num}) Captured M3U8") + + return match[2] + + async def refresh_html_cache( date: str, sport_id: str, @@ -115,8 +170,8 @@ async def get_events(cached_keys: list[str]) -> list[dict[str, str]]: live = [] - start_ts = now.delta(hours=-1).timestamp() - end_ts = now.delta(minutes=1).timestamp() + start_ts = now.delta(minutes=-30).timestamp() + end_ts = now.delta(minutes=30).timestamp() for k, v in events.items(): if k in cached_keys: @@ -130,7 +185,7 @@ async def get_events(cached_keys: list[str]) -> list[dict[str, str]]: return live -async def scrape(browser: Browser) -> None: +async def scrape() -> None: cached_urls = CACHE_FILE.load() valid_urls = {k: v for k, v in cached_urls.items() if v["url"]} @@ -146,52 +201,47 @@ async def scrape(browser: Browser) -> None: if events := await get_events(cached_urls.keys()): log.info(f"Processing {len(events)} new URL(s)") - async with network.event_context(browser, stealth=False) as context: - for i, ev in enumerate(events, start=1): - async with network.event_page(context) as page: - handler = partial( - network.process_event, - url=(link := ev["link"]), - url_num=i, - page=page, - timeout=5, - log=log, - ) + for i, ev in enumerate(events, start=1): - url = await network.safe_process( - handler, - url_num=i, - semaphore=network.PW_S, - log=log, - ) + handler = partial( + process_event, + url=(link := ev["link"]), + url_num=i, + ) - sport, event, ts = ( - ev["sport"], - ev["event"], - ev["event_ts"], - ) + url = await network.safe_process( + handler, + url_num=i, + semaphore=network.PW_S, + log=log, + ) - key = f"[{sport}] {event} ({TAG})" + sport, event, ts = ( + ev["sport"], + ev["event"], + ev["event_ts"], + ) - tvg_id, logo = leagues.get_tvg_info(sport, event) + key = f"[{sport}] {event} ({TAG})" - entry = { - "url": url, - "logo": logo, - "base": "https://storytrench.net/", - "timestamp": ts, - "id": tvg_id or "Live.Event.us", - "link": link, - } + tvg_id, logo = leagues.get_tvg_info(sport, event) - cached_urls[key] = entry + entry = { + "url": url, + "logo": logo, + "base": "https://hardsmart.click", + "timestamp": ts, + "id": tvg_id or "Live.Event.us", + "link": link, + "UA": "curl/8.19.0", + } - if url: - valid_count += 1 + cached_urls[key] = entry - entry["url"] = url.split("?")[0] + if url: + valid_count += 1 - urls[key] = entry + urls[key] = entry log.info(f"Collected and cached {valid_count - cached_count} new event(s)") diff --git a/M3U8/scrapers/streamsgate.py b/M3U8/scrapers/streamsgate.py index 3b50548f..717d786a 100644 --- a/M3U8/scrapers/streamsgate.py +++ b/M3U8/scrapers/streamsgate.py @@ -1,10 +1,11 @@ import asyncio +import re from functools import partial from itertools import chain from typing import Any from urllib.parse import urljoin -from playwright.async_api import Browser +from selectolax.parser import HTMLParser from .utils import Cache, Time, get_logger, leagues, network @@ -46,6 +47,42 @@ def get_event(t1: str, t2: str) -> str: return f"{t1.strip()} vs {t2.strip()}" +async def process_event(url: str, url_num: int) -> tuple[str | None, str | None]: + if not (event_data := await network.request(url, log=log)): + log.warning(f"URL {url_num}) Failed to load url.") + return + + soup_1 = HTMLParser(event_data.content) + + ifr = soup_1.css_first("iframe") + + if not ifr or not (src := ifr.attributes.get("src")): + log.warning(f"URL {url_num}) No iframe element found.") + return + + ifr_src = f"https:{src}" if src.startswith("//") else src + + if not ( + ifr_src_data := await network.request( + ifr_src, + headers={"Referer": url}, + log=log, + ) + ): + log.warning(f"URL {url_num}) Failed to load iframe source. (IFR1)") + return + + valid_m3u8 = re.compile(r"file:\s+(\'|\")([^\"]*)(\'|\")", re.I) + + if not (match := valid_m3u8.search(ifr_src_data.text)): + log.warning(f"URL {url_num}) No source found.") + return + + log.info(f"URL {url_num}) Captured M3U8") + + return match[2] + + async def refresh_api_cache(now_ts: float) -> list[dict[str, Any]]: tasks = [network.request(url, log=log) for url in SPORT_URLS] @@ -74,8 +111,8 @@ async def get_events(cached_keys: list[str]) -> list[dict[str, str]]: events = [] - start_dt = now.delta(hours=-1) - end_dt = now.delta(minutes=5) + start_dt = now.delta(minutes=-30) + end_dt = now.delta(minutes=30) for stream_group in api_data: date = stream_group.get("time") @@ -118,7 +155,7 @@ async def get_events(cached_keys: list[str]) -> list[dict[str, str]]: return events -async def scrape(browser: Browser) -> None: +async def scrape() -> None: cached_urls = CACHE_FILE.load() valid_urls = {k: v for k, v in cached_urls.items() if v["url"]} @@ -134,51 +171,45 @@ async def scrape(browser: Browser) -> None: if events := await get_events(cached_urls.keys()): log.info(f"Processing {len(events)} new URL(s)") - async with network.event_context(browser, stealth=False) as context: - for i, ev in enumerate(events, start=1): - async with network.event_page(context) as page: - handler = partial( - network.process_event, - url=(link := ev["link"]), - url_num=i, - page=page, - log=log, - ) + for i, ev in enumerate(events, start=1): + handler = partial( + process_event, + url=(link := ev["link"]), + url_num=i, + ) - url = await network.safe_process( - handler, - url_num=i, - semaphore=network.PW_S, - log=log, - ) + url = await network.safe_process( + handler, + url_num=i, + semaphore=network.PW_S, + log=log, + ) - sport, event, ts = ( - ev["sport"], - ev["event"], - ev["timestamp"], - ) + sport, event, ts = ( + ev["sport"], + ev["event"], + ev["timestamp"], + ) - key = f"[{sport}] {event} ({TAG})" + key = f"[{sport}] {event} ({TAG})" - tvg_id, logo = leagues.get_tvg_info(sport, event) + tvg_id, logo = leagues.get_tvg_info(sport, event) - entry = { - "url": url, - "logo": logo, - "base": "https://instreams.click/", - "timestamp": ts, - "id": tvg_id or "Live.Event.us", - "link": link, - } + entry = { + "url": url, + "logo": logo, + "base": "https://streamfree.click", + "timestamp": ts, + "id": tvg_id or "Live.Event.us", + "link": link, + } - cached_urls[key] = entry + cached_urls[key] = entry - if url: - valid_count += 1 + if url: + valid_count += 1 - entry["url"] = url.split("&e")[0] - - urls[key] = entry + urls[key] = entry log.info(f"Collected and cached {valid_count - cached_count} new event(s)") diff --git a/M3U8/scrapers/totalsportek.py b/M3U8/scrapers/totalsportek.py index 47da793d..0782a509 100644 --- a/M3U8/scrapers/totalsportek.py +++ b/M3U8/scrapers/totalsportek.py @@ -27,36 +27,26 @@ def fix_txt(s: str) -> str: async def process_event(url: str, url_num: int) -> str | None: if not (event_data := await network.request(url, log=log)): log.warning(f"URL {url_num}) Failed to load url.") - return soup_1 = HTMLParser(event_data.content) - if not (iframe_1 := soup_1.css_first("iframe")): + iframe_1 = soup_1.css_first("iframe") + + if not iframe_1 or not (iframe_1_src := iframe_1.attributes.get("src")): log.warning(f"URL {url_num}) No iframe element found. (IFR1)") - - return - - if not (iframe_1_src := iframe_1.attributes.get("src")): - log.warning(f"URL {url_num}) No iframe source found. (IFR1)") - return if not (iframe_1_src_data := await network.request(iframe_1_src, log=log)): log.warning(f"URL {url_num}) Failed to load iframe source. (IFR1)") - return soup_2 = HTMLParser(iframe_1_src_data.content) - if not (iframe_2 := soup_2.css_first("iframe")): + iframe_2 = soup_2.css_first("iframe") + + if not iframe_2 or not (iframe_2_src := iframe_2.attributes.get("src")): log.warning(f"URL {url_num}) No iframe element found. (IFR2)") - - return - - if not (iframe_2_src := iframe_2.attributes.get("src")): - log.warning(f"URL {url_num}) No iframe source found. (IFR2)") - return if not ( @@ -67,14 +57,12 @@ async def process_event(url: str, url_num: int) -> str | None: ) ): log.warning(f"URL {url_num}) Failed to load iframe source. (IFR2)") - return valid_m3u8 = re.compile(r'currentStreamUrl\s+=\s+"([^"]*)"', re.I) if not (match := valid_m3u8.search(iframe_2_src_data.text)): - log.warning(f"URL {url_num}) No Clappr source found. (IFR2)") - + log.warning(f"URL {url_num}) No Clappr source found.") return log.info(f"URL {url_num}) Captured M3U8")