diff --git a/M3U8/fetch.py b/M3U8/fetch.py index 530a986..8c7908c 100644 --- a/M3U8/fetch.py +++ b/M3U8/fetch.py @@ -9,6 +9,7 @@ from scrapers import ( pixel, ppv, roxie, + shark, streambtw, streameast, streamfree, @@ -49,6 +50,7 @@ async def main() -> None: asyncio.create_task(pixel.scrape()), asyncio.create_task(ppv.scrape(network.client)), asyncio.create_task(roxie.scrape(network.client)), + asyncio.create_task(shark.scrape(network.client)), asyncio.create_task(streambtw.scrape(network.client)), asyncio.create_task(streameast.scrape(network.client)), asyncio.create_task(streamfree.scrape(network.client)), @@ -66,6 +68,7 @@ async def main() -> None: | pixel.urls | ppv.urls | roxie.urls + | shark.urls | streambtw.urls | streameast.urls | strmd.urls diff --git a/M3U8/scrapers/pixel.py b/M3U8/scrapers/pixel.py index 9929538..d0650d5 100644 --- a/M3U8/scrapers/pixel.py +++ b/M3U8/scrapers/pixel.py @@ -11,15 +11,10 @@ urls: dict[str, dict[str, str | float]] = {} CACHE_FILE = Cache("pixel.json", exp=86_400) -API_FILE = Cache("pixel-api.json", exp=86_400) - BASE_URL = "https://pixelsport.tv/backend/livetv/events" -async def refresh_api_cache( - url: str, - ts: float, -) -> dict[str, list[dict, str, str]]: +async def get_api_data(url: str) -> dict[str, list[dict, str, str]]: log.info("Refreshing API cache") async with async_playwright() as p: @@ -44,23 +39,13 @@ async def refresh_api_cache( finally: await browser.close() - data = json.loads(raw_json) - - data["timestamp"] = ts - - return data + return json.loads(raw_json) -async def get_events(cached_keys: set[str]) -> dict[str, str | float]: +async def get_events() -> dict[str, dict[str, str | float]]: now = Time.clean(Time.now()) - if not (api_data := API_FILE.load(per_entry=False)): - api_data = await refresh_api_cache( - BASE_URL, - now.timestamp(), - ) - - API_FILE.write(api_data) + api_data = await get_api_data(BASE_URL) events = {} @@ -88,9 +73,6 @@ async def get_events(cached_keys: set[str]) -> dict[str, str | float]: if pattern.search(stream_link): key = f"[{sport}] {event_name} {z} (PIXL)" - if cached_keys & {key}: - continue - tvg_id, logo = leagues.get_tvg_info(sport, event_name) events[key] = { @@ -105,23 +87,17 @@ async def get_events(cached_keys: set[str]) -> dict[str, str | float]: async def scrape() -> None: - cached_urls = CACHE_FILE.load() - cached_count = len(cached_urls) - urls.update(cached_urls) - - log.info(f"Loaded {cached_count} event(s) from cache") + if cached := CACHE_FILE.load(): + urls.update(cached) + log.info(f"Loaded {len(urls)} event(s) from cache") + return log.info(f'Scraping from "{BASE_URL}"') - events = await get_events(set(cached_urls.keys())) + events = await get_events() - if events: - for d in (urls, cached_urls): - d |= events + urls.update(events) - if new_count := len(cached_urls) - cached_count: - log.info(f"Collected and cached {new_count} new event(s)") - else: - log.info("No new events found") + CACHE_FILE.write(urls) - CACHE_FILE.write(cached_urls) + log.info(f"Collected and cached {len(urls)} new event(s)") diff --git a/M3U8/scrapers/roxie.py b/M3U8/scrapers/roxie.py index 71793b5..3635dde 100644 --- a/M3U8/scrapers/roxie.py +++ b/M3U8/scrapers/roxie.py @@ -58,7 +58,7 @@ async def refresh_html_cache( url: str, sport: str, now_ts: float, -) -> dict[str, str | float]: +) -> dict[str, dict[str, str | float]]: try: r = await client.get(url) @@ -73,9 +73,7 @@ async def refresh_html_cache( events = {} for row in soup.css("table#eventsTable tbody tr"): - a_tag = row.css_first("td a") - - if not a_tag: + if not (a_tag := row.css_first("td a")): continue event = a_tag.text(strip=True) diff --git a/M3U8/scrapers/shark.py b/M3U8/scrapers/shark.py new file mode 100644 index 0000000..40f5863 --- /dev/null +++ b/M3U8/scrapers/shark.py @@ -0,0 +1,183 @@ +import re +from functools import partial + +import httpx +from selectolax.parser import HTMLParser + +from .utils import Cache, Time, get_logger, leagues, network + +log = get_logger(__name__) + +urls: dict[str, dict[str, str | float]] = {} + +CACHE_FILE = Cache("shark.json", exp=10_800) + +HTML_CACHE = Cache("shark-html.json", exp=28_800) + +BASE_URL = "https://sharkstreams.net" + + +async def process_event( + client: httpx.AsyncClient, + url: str, + url_num: int, +) -> str | None: + + try: + r = await client.get(url) + r.raise_for_status() + except Exception as e: + log.error(f'URL {url_num}) Failed to fetch "{url}": {e}') + return + + data: dict[str, list[str]] = r.json() + + if not data.get("urls"): + log.info(f"URL {url_num}) No M3U8 found") + + return + + log.info(f"URL {url_num}) Captured M3U8") + + return data["urls"][0] + + +async def refresh_html_cache( + client: httpx.AsyncClient, + url: str, + now_ts: float, +) -> dict[str, dict[str, str | float]]: + + try: + r = await client.get(url) + r.raise_for_status() + except Exception as e: + log.error(f'Failed to fetch "{url}": {e}') + + return {} + + soup = HTMLParser(r.text) + + events = {} + + for row in soup.css(".row"): + date_node = row.css_first(".ch-date") + sport_node = row.css_first(".ch-category") + name_node = row.css_first(".ch-name") + + if not (date_node and sport_node and name_node): + continue + + event_dt = Time.from_str(date_node.text(strip=True), timezone="EST") + sport = sport_node.text(strip=True) + event_name = name_node.text(strip=True) + + embed_btn = row.css_first("a.hd-link.secondary") + + if not embed_btn or not (onclick := embed_btn.attributes.get("onclick")): + continue + + pattern = re.compile(r"openEmbed\('([^']+)'\)", re.IGNORECASE) + + if not (match := pattern.search(onclick)): + continue + + link = match[1].replace("player.php", "get-stream.php") + + key = f"[{sport}] {event_name} (SHARK)" + + events[key] = { + "sport": sport, + "event": event_name, + "link": link, + "event_ts": event_dt.timestamp(), + "timestamp": now_ts, + } + + return events + + +async def get_events( + client: httpx.AsyncClient, + cached_keys: set[str], +) -> list[dict[str, str]]: + + now = Time.clean(Time.now()) + + if not (events := HTML_CACHE.load()): + events = await refresh_html_cache( + client, + BASE_URL, + now.timestamp(), + ) + + HTML_CACHE.write(events) + + live = [] + + start_ts = now.delta(hours=-1).timestamp() + end_ts = now.delta(minutes=10).timestamp() + + for k, v in events.items(): + if cached_keys & {k}: + continue + + if not start_ts <= v["event_ts"] <= end_ts: + continue + + live.append({**v}) + + return live + + +async def scrape(client: httpx.AsyncClient) -> None: + cached_urls = CACHE_FILE.load() + cached_count = len(cached_urls) + urls.update(cached_urls) + + log.info(f"Loaded {cached_count} event(s) from cache") + + log.info(f'Scraping from "{BASE_URL}"') + + events = await get_events(client, set(cached_urls.keys())) + + log.info(f"Processing {len(events)} new URL(s)") + + if events: + for i, ev in enumerate(events, start=1): + handler = partial( + process_event, + client=client, + url=ev["link"], + url_num=i, + ) + + url = await network.safe_process( + handler, + url_num=i, + log=log, + ) + + if url: + sport, event, ts = ev["sport"], ev["event"], ev["event_ts"] + + tvg_id, logo = leagues.get_tvg_info(sport, event) + + key = f"[{sport}] {event} (SHARK)" + + entry = { + "url": url, + "logo": logo, + "base": BASE_URL, + "timestamp": ts, + "id": tvg_id or "Live.Event.us", + } + + urls[key] = cached_urls[key] = entry + + if new_count := len(cached_urls) - cached_count: + log.info(f"Collected and cached {new_count} new event(s)") + else: + log.info("No new events found") + + CACHE_FILE.write(cached_urls) diff --git a/M3U8/scrapers/tvpass.py b/M3U8/scrapers/tvpass.py index 27a0001..603b6d4 100644 --- a/M3U8/scrapers/tvpass.py +++ b/M3U8/scrapers/tvpass.py @@ -13,7 +13,7 @@ CACHE_FILE = Cache("tvpass.json", exp=86_400) BASE_URL = "https://tvpass.org/playlist/m3u" -async def fetch_m3u8(client: httpx.AsyncClient) -> list[str]: +async def get_data(client: httpx.AsyncClient) -> list[str]: try: r = await client.get(BASE_URL) r.raise_for_status() @@ -25,19 +25,12 @@ async def fetch_m3u8(client: httpx.AsyncClient) -> list[str]: return r.text.splitlines() -async def scrape(client: httpx.AsyncClient) -> None: - if cached := CACHE_FILE.load(): - urls.update(cached) - log.info(f"Loaded {len(urls)} event(s) from cache") - return - - log.info(f'Scraping from "{BASE_URL}"') - +async def get_events(client: httpx.AsyncClient) -> dict[str, dict[str, str | float]]: now = Time.now().timestamp() - if not (data := await fetch_m3u8(client)): - log.warning("No M3U8 data received") - return + events = {} + + data = await get_data(client) for i, line in enumerate(data): if line.startswith("#EXTINF"): @@ -59,7 +52,7 @@ async def scrape(client: httpx.AsyncClient) -> None: tvg_id, logo = leagues.info(sport) - entry = { + events[key] = { "url": f"http://origin.thetvapp.to/hls/{channel}/mono.m3u8", "logo": logo, "id": tvg_id or "Live.Event.us", @@ -67,8 +60,21 @@ async def scrape(client: httpx.AsyncClient) -> None: "timestamp": now, } - urls[key] = entry + return events + + +async def scrape(client: httpx.AsyncClient) -> None: + if cached := CACHE_FILE.load(): + urls.update(cached) + log.info(f"Loaded {len(urls)} event(s) from cache") + return + + log.info(f'Scraping from "{BASE_URL}"') + + events = await get_events(client) + + urls.update(events) CACHE_FILE.write(urls) - log.info(f"Cached {len(urls)} event(s)") + log.info(f"Collected and cached {len(urls)} new event(s)") diff --git a/M3U8/scrapers/volo.py b/M3U8/scrapers/volo.py index 1bc4dac..484b7e7 100644 --- a/M3U8/scrapers/volo.py +++ b/M3U8/scrapers/volo.py @@ -115,7 +115,7 @@ async def refresh_html_cache( client: httpx.AsyncClient, url: str, sport: str, -) -> dict[str, str | float]: +) -> dict[str, dict[str, str | float]]: try: r = await client.get(url)