From 00000d964b17e824f452d7ecc3f898c4d0db6601 Mon Sep 17 00:00:00 2001 From: doms9 <96013514+doms9@users.noreply.github.com> Date: Thu, 12 Feb 2026 18:15:01 -0500 Subject: [PATCH] e misc edits --- M3U8/scrapers/fawa.py | 2 +- M3U8/scrapers/istreameast.py | 4 +-- M3U8/scrapers/livetvsx.py | 2 +- M3U8/scrapers/pawa.py | 2 +- M3U8/scrapers/roxie.py | 56 +++++++++++++++++++--------------- M3U8/scrapers/shark.py | 4 +-- M3U8/scrapers/sport9.py | 2 +- M3U8/scrapers/streambtw.py | 4 +-- M3U8/scrapers/streamhub.py | 2 +- M3U8/scrapers/utils/webwork.py | 2 -- M3U8/scrapers/webcast.py | 2 +- M3U8/scrapers/xstreameast.py | 2 +- 12 files changed, 44 insertions(+), 40 deletions(-) diff --git a/M3U8/scrapers/fawa.py b/M3U8/scrapers/fawa.py index 74497f48..6cf3d01f 100644 --- a/M3U8/scrapers/fawa.py +++ b/M3U8/scrapers/fawa.py @@ -25,7 +25,7 @@ async def process_event(url: str, url_num: int) -> str | None: valid_m3u8 = re.compile( r'var\s+(\w+)\s*=\s*\[["\']?(https?:\/\/[^"\'\s>]+\.m3u8(?:\?[^"\'\s>]*)?)["\']\]?', - re.IGNORECASE, + re.I, ) if not (match := valid_m3u8.search(html_data.text)): diff --git a/M3U8/scrapers/istreameast.py b/M3U8/scrapers/istreameast.py index b3cfcaa7..787ce5fd 100644 --- a/M3U8/scrapers/istreameast.py +++ b/M3U8/scrapers/istreameast.py @@ -40,7 +40,7 @@ async def process_event(url: str, url_num: int) -> str | None: return - pattern = re.compile(r"source:\s*window\.atob\(\s*'([^']+)'\s*\)", re.IGNORECASE) + pattern = re.compile(r"source:\s*window\.atob\(\s*'([^']+)'\s*\)", re.I) if not (match := pattern.search(iframe_src_data.text)): log.warning(f"URL {url_num}) No Clappr source found.") @@ -58,7 +58,7 @@ async def get_events(cached_keys: list[str]) -> list[dict[str, str]]: if not (html_data := await network.request(BASE_URL, log=log)): return events - pattern = re.compile(r"^(?:LIVE|(?:[1-9]|[12]\d|30)\s+minutes?\b)", re.IGNORECASE) + pattern = re.compile(r"^(?:LIVE|(?:[1-9]|[12]\d|30)\s+minutes?\b)", re.I) soup = HTMLParser(html_data.content) diff --git a/M3U8/scrapers/livetvsx.py b/M3U8/scrapers/livetvsx.py index bf417bd3..198acfad 100644 --- a/M3U8/scrapers/livetvsx.py +++ b/M3U8/scrapers/livetvsx.py @@ -184,7 +184,7 @@ async def get_events(cached_keys: list[str]) -> list[dict[str, str]]: if not start_ts <= v["event_ts"] <= end_ts: continue - live.append({**v}) + live.append(v) return live diff --git a/M3U8/scrapers/pawa.py b/M3U8/scrapers/pawa.py index 8c2bad21..d9ec2501 100644 --- a/M3U8/scrapers/pawa.py +++ b/M3U8/scrapers/pawa.py @@ -41,7 +41,7 @@ async def process_event(url: str, url_num: int) -> str | None: return - pattern = re.compile(r"source:\s*window\.atob\(\s*'([^']+)'\s*\)", re.IGNORECASE) + pattern = re.compile(r"source:\s*window\.atob\(\s*'([^']+)'\s*\)", re.I) if not (match := pattern.search(iframe_src_data.text)): log.warning(f"URL {url_num}) No Clappr source found.") diff --git a/M3U8/scrapers/roxie.py b/M3U8/scrapers/roxie.py index 0808f56a..bf3409be 100644 --- a/M3U8/scrapers/roxie.py +++ b/M3U8/scrapers/roxie.py @@ -34,7 +34,7 @@ async def process_event(url: str, url_num: int) -> str | None: if not (html_data := await network.request(url, log=log)): return - valid_m3u8 = re.compile(r"'clappr',\s+'([^\"]*)'", re.IGNORECASE) + valid_m3u8 = re.compile(r"'clappr',\s+'([^\"]*)'", re.I) if not (match := valid_m3u8.search(html_data.text)): log.info(f"URL {url_num}) No M3U8 found") @@ -125,7 +125,7 @@ async def get_events(cached_keys: list[str]) -> list[dict[str, str]]: if not start_ts <= v["event_ts"] <= end_ts: continue - live.append({**v}) + live.append(v) return live @@ -133,9 +133,11 @@ async def get_events(cached_keys: list[str]) -> list[dict[str, str]]: async def scrape() -> None: cached_urls = CACHE_FILE.load() - cached_count = len(cached_urls) + valid_urls = {k: v for k, v in cached_urls.items() if v["url"]} - urls.update(cached_urls) + valid_count = cached_count = len(valid_urls) + + urls.update(valid_urls) log.info(f"Loaded {cached_count} event(s) from cache") @@ -160,30 +162,34 @@ async def scrape() -> None: log=log, ) + sport, event, ts, link = ( + ev["sport"], + ev["event"], + ev["event_ts"], + ev["link"], + ) + + tvg_id, logo = leagues.get_tvg_info(sport, event) + + key = f"[{sport}] {event} ({TAG})" + + entry = { + "url": url, + "logo": logo, + "base": BASE_URL, + "timestamp": ts, + "id": tvg_id or "Live.Event.us", + "link": link, + } + + cached_urls[key] = entry + if url: - sport, event, ts, link = ( - ev["sport"], - ev["event"], - ev["event_ts"], - ev["link"], - ) + valid_count += 1 - tvg_id, logo = leagues.get_tvg_info(sport, event) + urls[key] = entry - key = f"[{sport}] {event} ({TAG})" - - entry = { - "url": url, - "logo": logo, - "base": BASE_URL, - "timestamp": ts, - "id": tvg_id or "Live.Event.us", - "link": link, - } - - urls[key] = cached_urls[key] = entry - - if new_count := len(cached_urls) - cached_count: + if new_count := valid_count - cached_count: log.info(f"Collected and cached {new_count} new event(s)") else: diff --git a/M3U8/scrapers/shark.py b/M3U8/scrapers/shark.py index f2a50a5d..19034cac 100644 --- a/M3U8/scrapers/shark.py +++ b/M3U8/scrapers/shark.py @@ -44,7 +44,7 @@ async def refresh_html_cache(now_ts: float) -> dict[str, dict[str, str | float]] if not (html_data := await network.request(BASE_URL, log=log)): return events - pattern = re.compile(r"openEmbed\('([^']+)'\)", re.IGNORECASE) + pattern = re.compile(r"openEmbed\('([^']+)'\)", re.I) soup = HTMLParser(html_data.content) @@ -106,7 +106,7 @@ async def get_events(cached_keys: list[str]) -> list[dict[str, str]]: if not start_ts <= v["event_ts"] <= end_ts: continue - live.append({**v}) + live.append(v) return live diff --git a/M3U8/scrapers/sport9.py b/M3U8/scrapers/sport9.py index a25554ad..139993af 100644 --- a/M3U8/scrapers/sport9.py +++ b/M3U8/scrapers/sport9.py @@ -45,7 +45,7 @@ async def get_events(cached_keys: list[str]) -> list[dict[str, str]]: for card in soup.css("a.match-card"): live_badge = card.css_first(".live-badge") - if not live_badge or live_badge.text(strip=True) != "Live": + if not live_badge or live_badge.text(strip=True).lower() != "live": continue if not (sport_node := card.css_first(".tournament-name")): diff --git a/M3U8/scrapers/streambtw.py b/M3U8/scrapers/streambtw.py index eefb8a48..4c7eaf0b 100644 --- a/M3U8/scrapers/streambtw.py +++ b/M3U8/scrapers/streambtw.py @@ -19,7 +19,7 @@ BASE_URL = "https://hiteasport.info" def fix_league(s: str) -> str: - pattern = re.compile(r"^\w*-\w*", re.IGNORECASE) + pattern = re.compile(r"^\w*-\w*", re.I) return " ".join(s.split("-")) if pattern.search(s) else s @@ -28,7 +28,7 @@ async def process_event(url: str, url_num: int) -> str | None: if not (html_data := await network.request(url, log=log)): return - valid_m3u8 = re.compile(r'var\s+(\w+)\s*=\s*"([^"]*)"', re.IGNORECASE) + valid_m3u8 = re.compile(r'var\s+(\w+)\s*=\s*"([^"]*)"', re.I) if not (match := valid_m3u8.search(html_data.text)): log.info(f"URL {url_num}) No M3U8 found") diff --git a/M3U8/scrapers/streamhub.py b/M3U8/scrapers/streamhub.py index c2c87244..ebf7c728 100644 --- a/M3U8/scrapers/streamhub.py +++ b/M3U8/scrapers/streamhub.py @@ -130,7 +130,7 @@ async def get_events(url: str, cached_keys: list[str]) -> list[dict[str, str]]: if not start_ts <= v["event_ts"] <= end_ts: continue - live.append({**v}) + live.append(v) return live diff --git a/M3U8/scrapers/utils/webwork.py b/M3U8/scrapers/utils/webwork.py index 248a948f..83607594 100644 --- a/M3U8/scrapers/utils/webwork.py +++ b/M3U8/scrapers/utils/webwork.py @@ -28,8 +28,6 @@ class Network: PW_S = asyncio.Semaphore(3) - proxy_base = "https://stream.nvrmind.xyz" - def __init__(self) -> None: self.client = httpx.AsyncClient( timeout=httpx.Timeout(5.0), diff --git a/M3U8/scrapers/webcast.py b/M3U8/scrapers/webcast.py index ad8ccc10..19039d84 100644 --- a/M3U8/scrapers/webcast.py +++ b/M3U8/scrapers/webcast.py @@ -108,7 +108,7 @@ async def get_events(cached_keys: list[str]) -> list[dict[str, str]]: if not start_ts <= v["event_ts"] <= end_ts: continue - live.append({**v}) + live.append(v) return live diff --git a/M3U8/scrapers/xstreameast.py b/M3U8/scrapers/xstreameast.py index 418ea6d8..83f34d4e 100644 --- a/M3U8/scrapers/xstreameast.py +++ b/M3U8/scrapers/xstreameast.py @@ -30,7 +30,7 @@ SPORT_ENDPOINTS = [ async def process_event(url: str, url_num: int) -> tuple[str | None, str | None]: - valid_m3u8 = re.compile(r'(var|const)\s+(\w+)\s*=\s*"([^"]*)"', re.IGNORECASE) + valid_m3u8 = re.compile(r'(var|const)\s+(\w+)\s*=\s*"([^"]*)"', re.I) nones = None, None