From 00000d9f35e854628720fd26606f94d5fffa1238 Mon Sep 17 00:00:00 2001 From: doms9 <96013514+doms9@users.noreply.github.com> Date: Tue, 31 Mar 2026 22:01:42 -0400 Subject: [PATCH] e - add fsports.py - add streamtpnew.py - misc edits. --- M3U8/fetch.py | 6 ++ M3U8/scrapers/fsports.py | 137 +++++++++++++++++++++++++++++ M3U8/scrapers/streamcenter.py | 4 +- M3U8/scrapers/streamtpnew.py | 158 ++++++++++++++++++++++++++++++++++ 4 files changed, 303 insertions(+), 2 deletions(-) create mode 100644 M3U8/scrapers/fsports.py create mode 100644 M3U8/scrapers/streamtpnew.py diff --git a/M3U8/fetch.py b/M3U8/fetch.py index 7d153d85..27305904 100644 --- a/M3U8/fetch.py +++ b/M3U8/fetch.py @@ -8,6 +8,7 @@ from scrapers import ( cdnlivetv, embedhd, fawa, + fsports, istreameast, livetvsx, ovogoal, @@ -18,6 +19,7 @@ from scrapers import ( streamcenter, streamhub, streamsgate, + streamtpnew, totalsportek, tvapp, watchfooty, @@ -60,6 +62,7 @@ async def main() -> None: pw_tasks = [ asyncio.create_task(cdnlivetv.scrape(hdl_brwsr)), asyncio.create_task(embedhd.scrape(hdl_brwsr)), + asyncio.create_task(fsports.scrape(hdl_brwsr)), asyncio.create_task(ppv.scrape(xtrnl_brwsr)), asyncio.create_task(roxie.scrape(hdl_brwsr)), asyncio.create_task(streamcenter.scrape(hdl_brwsr)), @@ -73,6 +76,7 @@ async def main() -> None: # asyncio.create_task(ovogoal.scrape()), asyncio.create_task(pawa.scrape()), asyncio.create_task(shark.scrape()), + asyncio.create_task(streamtpnew.scrape()), asyncio.create_task(totalsportek.scrape()), asyncio.create_task(tvapp.scrape()), asyncio.create_task(webcast.scrape()), @@ -95,6 +99,7 @@ async def main() -> None: cdnlivetv.urls | embedhd.urls | fawa.urls + | fsports.urls | istreameast.urls | livetvsx.urls | ovogoal.urls @@ -105,6 +110,7 @@ async def main() -> None: | streamcenter.urls | streamhub.urls | streamsgate.urls + | streamtpnew.urls | totalsportek.urls | tvapp.urls | watchfooty.urls diff --git a/M3U8/scrapers/fsports.py b/M3U8/scrapers/fsports.py new file mode 100644 index 00000000..1f5dfb88 --- /dev/null +++ b/M3U8/scrapers/fsports.py @@ -0,0 +1,137 @@ +import asyncio +from functools import partial +from urllib.parse import urljoin + +from playwright.async_api import Browser +from selectolax.parser import HTMLParser + +from .utils import Cache, Time, get_logger, leagues, network + +log = get_logger(__name__) + +urls: dict[str, dict[str, str | float]] = {} + +TAG = "FSPRTS" + +CACHE_FILE = Cache(TAG, exp=5_400) + +BASE_URL = "https://fsportshds.xyz" + +SPORT_URLS = { + # "Fighting": urljoin(BASE_URL, "mmastreams.php"), + "Basketball": urljoin(BASE_URL, "nbastreams.php"), + # "Ice Hockey": urljoin(BASE_URL, "nhlstreams.php"), + # "American Football": urljoin(BASE_URL, "nflstreams.php") +} | { + sport: urljoin(BASE_URL, f"{sport}streams.php".lower()) + for sport in [ + "Football", + # "Boxing", + # "F1", + # "MLB", + # "MotoGP", + ] +} + + +async def get_events(cached_keys: list[str]) -> list[dict[str, str]]: + tasks = [network.request(url, log=log) for url in SPORT_URLS.values()] + + results = await asyncio.gather(*tasks) + + events = [] + + if not ( + soups := [(HTMLParser(html.content), html.url) for html in results if html] + ): + return events + + for soup, url in soups: + sport = next((k for k, v in SPORT_URLS.items() if v == url), "Live Event") + + for card in soup.css(".media.btn.btn-default.btn-lg.btn-block"): + if not (name_elem := card.css_first("h4")): + continue + + if card.css_first('[id^="countdown-"]'): + continue + + if not (a_elem := card.css_first("a")) or not ( + href := a_elem.attributes.get("href") + ): + continue + + name = name_elem.text(strip=True) + + if f"[{sport}] {name} ({TAG})" in cached_keys: + continue + + events.append( + { + "sport": sport, + "event": name, + "link": urljoin(BASE_URL, href), + } + ) + + return events + + +async def scrape(browser: Browser) -> None: + cached_urls = CACHE_FILE.load() + + cached_count = len(cached_urls) + + urls.update(cached_urls) + + log.info(f"Loaded {cached_count} event(s) from cache") + + log.info(f'Scraping from "{BASE_URL}"') + + if events := await get_events(cached_urls.keys()): + log.info(f"Processing {len(events)} new URL(s)") + + now = Time.clean(Time.now()) + + async with network.event_context(browser, stealth=False) as context: + for i, ev in enumerate(events, start=1): + async with network.event_page(context) as page: + handler = partial( + network.process_event, + url=(link := ev["link"]), + url_num=i, + page=page, + log=log, + ) + + url = await network.safe_process( + handler, + url_num=i, + semaphore=network.PW_S, + log=log, + ) + + if url: + sport, event = ev["sport"], ev["event"] + + key = f"[{sport}] {event} ({TAG})" + + tvg_id, logo = leagues.get_tvg_info(sport, event) + + entry = { + "url": url, + "logo": logo, + "base": "https://vividmosaica.com/", + "timestamp": now.timestamp(), + "id": tvg_id or "Live.Event.us", + "link": link, + } + + urls[key] = cached_urls[key] = entry + + log.info(f"Collected and cached {len(cached_urls) - cached_count} new event(s)") + + else: + log.info("No new events found") + + CACHE_FILE.write(cached_urls) diff --git a/M3U8/scrapers/streamcenter.py b/M3U8/scrapers/streamcenter.py index 0d635a8b..6e9e3a24 100644 --- a/M3U8/scrapers/streamcenter.py +++ b/M3U8/scrapers/streamcenter.py @@ -14,7 +14,7 @@ CACHE_FILE = Cache(TAG, exp=10_800) API_FILE = Cache(f"{TAG}-api", exp=19_800) -BASE_URL = "https://backend.streamcenter.live/api/Parties" +API_URL = "https://backend.streamcenter.live/api/Parties" CATEGORIES = { 4: "Basketball", @@ -39,7 +39,7 @@ async def get_events(cached_keys: list[str]) -> list[dict[str, str]]: api_data = [{"timestamp": now.timestamp()}] if r := await network.request( - BASE_URL, + API_URL, log=log, params={"pageNumber": 1, "pageSize": 500}, ): diff --git a/M3U8/scrapers/streamtpnew.py b/M3U8/scrapers/streamtpnew.py new file mode 100644 index 00000000..4c98d9eb --- /dev/null +++ b/M3U8/scrapers/streamtpnew.py @@ -0,0 +1,158 @@ +import ast +import base64 +import re +from functools import partial + +from .utils import Cache, Time, get_logger, leagues, network + +log = get_logger(__name__) + +urls: dict[str, dict[str, str | float]] = {} + +TAG = "STP" + +CACHE_FILE = Cache(TAG, exp=19_800) + +API_FILE = Cache(f"{TAG}-api", exp=19_800) + +API_URL = "https://streamtpnew.com/eventos.json" + + +async def process_event(url: str, url_num: int) -> str | None: + if not (event_data := await network.request(url, log=log)): + log.warning(f"URL {url_num}) Failed to load url.") + return + + digit_func_ptrn = re.compile(r"{return\s+(\d*);}", re.I) + + if not (digit_list := digit_func_ptrn.findall(event_data.text)): + log.warning(f"URL {url_num}) Unable to decode url.") + return + + embed_list_ptrn = re.compile(r"\w*=\[\[(.*)\]\];") + + if not (embed_list := embed_list_ptrn.search(event_data.text)): + log.warning(f"URL {url_num}) Unable to decode url.") + return + + embed_list_str = embed_list[0].split("=", 1)[-1].strip(";") + + embed_list: list[tuple[int, str]] = ast.literal_eval(embed_list_str) + + embed_list.sort(key=lambda i: i[0]) + + m3u8 = "".join( + chr( + int("".join(c for c in base64.b64decode(v).decode("utf-8") if c.isdigit())) + - sum(map(int, digit_list)) + ) + for _, v in embed_list + ) + + log.info(f"URL {url_num}) Captured M3U8") + + return m3u8.split("&ip")[0] + + +async def get_events(cached_keys: list[str]) -> list[dict[str, str]]: + now = Time.clean(Time.now()) + + if not (api_data := API_FILE.load(per_entry=False, index=-1)): + log.info("Refreshing API cache") + + api_data = [{"timestamp": now.timestamp()}] + + if r := await network.request(API_URL, log=log): + api_data: list[dict[str, str]] = r.json() + + api_data[-1]["timestamp"] = now.timestamp() + + API_FILE.write(api_data) + + events = [] + + for event in api_data: + name = event.get("title") + + link = event.get("link") + + if not (name and link): + continue + + if (sport := event.get("category")) and sport == "Other": + sport = "Live Event" + + if f"[{sport}] {name} ({TAG})" in cached_keys: + continue + + events.append( + { + "sport": sport, + "event": name, + "link": link, + } + ) + + return events + + +async def scrape() -> None: + cached_urls = CACHE_FILE.load() + + valid_urls = {k: v for k, v in cached_urls.items() if v["url"]} + + valid_count = cached_count = len(valid_urls) + + urls.update(valid_urls) + + log.info(f"Loaded {cached_count} event(s) from cache") + + log.info('Scraping from "https://streamtpnew.com"') + + if events := await get_events(cached_urls.keys()): + log.info(f"Processing {len(events)} new URL(s)") + + now = Time.clean(Time.now()) + + for i, ev in enumerate(events, start=1): + handler = partial( + process_event, + url=(link := ev["link"]), + url_num=i, + ) + + url = await network.safe_process( + handler, + url_num=i, + semaphore=network.HTTP_S, + log=log, + ) + + sport, event = ev["sport"], ev["event"] + + key = f"[{sport}] {event} ({TAG})" + + tvg_id, logo = leagues.get_tvg_info(sport, event) + + entry = { + "url": url, + "logo": logo, + "base": link, + "timestamp": now.timestamp(), + "id": tvg_id or "Live.Event.us", + "link": link, + } + + cached_urls[key] = entry + + if url: + valid_count += 1 + + urls[key] = entry + + log.info(f"Collected and cached {valid_count - cached_count} new event(s)") + + else: + log.info("No new events found") + + CACHE_FILE.write(cached_urls)