From 00000d934ab2c566aee96b8aa50c13d28b4371d1 Mon Sep 17 00:00:00 2001 From: doms9 <96013514+doms9@users.noreply.github.com> Date: Sat, 13 Sep 2025 04:42:55 -0400 Subject: [PATCH] e --- M3U8/fetch.py | 53 ++++++++---- M3U8/scrapers/caches/livetvsx.json | 27 +++++- M3U8/scrapers/caches/ppv.json | 7 ++ M3U8/scrapers/caches/streambtw.json | 19 +++++ M3U8/scrapers/caches/streameast.json | 10 +++ M3U8/scrapers/caches/tvpass.json | 20 +++++ M3U8/scrapers/fstv.py | 118 +++++++++++++++++++-------- M3U8/scrapers/livetvsx.py | 13 ++- M3U8/scrapers/ppv.py | 9 +- M3U8/scrapers/streambtw.py | 5 +- M3U8/scrapers/streameast.py | 11 +-- M3U8/scrapers/tvpass.py | 7 +- 12 files changed, 231 insertions(+), 68 deletions(-) diff --git a/M3U8/fetch.py b/M3U8/fetch.py index 41fa612..673b997 100644 --- a/M3U8/fetch.py +++ b/M3U8/fetch.py @@ -3,7 +3,7 @@ import asyncio from pathlib import Path import httpx -from scrapers import livetvsx, ppv, streameast, tvpass +from scrapers import fstv, livetvsx, ppv, streambtw, streameast, tvpass from scrapers.utils import UA, get_logger log = get_logger(__name__) @@ -36,8 +36,10 @@ async def vanilla_fetch() -> tuple[list[str], int]: async def main() -> None: tasks = [ + asyncio.create_task(fstv.main(CLIENT)), asyncio.create_task(livetvsx.main(CLIENT)), asyncio.create_task(ppv.main(CLIENT)), + asyncio.create_task(streambtw.main(CLIENT)), asyncio.create_task(streameast.main(CLIENT)), asyncio.create_task(tvpass.main(CLIENT)), vanilla_fetch(), @@ -47,25 +49,40 @@ async def main() -> None: base_m3u8, tvg_chno = results[-1] - additions = livetvsx.urls | ppv.urls | streameast.urls | tvpass.urls - - lines = [ - f'#EXTINF:-1 tvg-chno="{chnl_num}" tvg-id="(N/A)" tvg-name="{event}" tvg-logo="{info["logo"]}" group-title="Live Events",{event}\n{info["url"]}' - for chnl_num, (event, info) in enumerate( - sorted(additions.items()), - start=tvg_chno + 1, - ) - ] - - M3U8_FILE.write_text( - '#EXTM3U url-tvg="https://raw.githubusercontent.com/doms9/iptv/refs/heads/default/EPG/TV.xml"\n' - + "\n".join(base_m3u8) - + "\n" - + "\n".join(lines) - + "\n", - encoding="utf-8", + additions = ( + fstv.urls + | livetvsx.urls + | ppv.urls + | streambtw.urls + | streameast.urls + | tvpass.urls ) + live_events = [] + + for chnl_num, (event, info) in enumerate( + sorted(additions.items()), + start=tvg_chno + 1, + ): + live_events.extend( + ( + f'#EXTINF:-1 tvg-chno="{chnl_num}" tvg-id="(N/A)" tvg-name="{event}" tvg-logo="{info["logo"]}" group-title="Live Events",{event}', + f'#EXTVLCOPT:http-referrer={info["base"]}', + f'#EXTVLCOPT:http-origin={info["base"]}', + info["url"], + ) + ) + + m3u8_content = "\n".join( + [ + '#EXTM3U url-tvg="https://raw.githubusercontent.com/doms9/iptv/refs/heads/default/EPG/TV.xml"' + ] + + base_m3u8 + + live_events + ) + + M3U8_FILE.write_text(m3u8_content, encoding="utf-8") + log.info(f"M3U8 saved to {M3U8_FILE.name}") diff --git a/M3U8/scrapers/caches/livetvsx.json b/M3U8/scrapers/caches/livetvsx.json index 1a92549..d26f377 100644 --- a/M3U8/scrapers/caches/livetvsx.json +++ b/M3U8/scrapers/caches/livetvsx.json @@ -2,126 +2,151 @@ "[Football: USA MLS Next Pro] St. Louis City 2 - Minnesota United II": { "url": "https://ddt2.dbcast.live/5Be237bdjS9FWNQwHcCsdg/1757750718/3230392e362e3137392e313937/ff429/playlist.m3u8", "logo": "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", + "base": "https://livetv.sx/enx/", "timestamp": 1757721904.604711 }, "[Football: Ecuador Serie A] Libertad - LDU Quito": { "url": "https://ddt2.dbcast.live/pSL4febu-wntL1ioWtTOSw/1757750737/3230392e362e3137392e313937/ff427/playlist.m3u8", "logo": "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", + "base": "https://livetv.sx/enx/", "timestamp": 1757721904.604711 }, "[Football: Mexico Women Liga MX] Toluca (W) - Monterrey (W)": { "url": "https://ddt2.dbcast.live/fISUs4UFoUpseEFP2g0BGQ/1757750756/3230392e362e3137392e313937/ff428/playlist.m3u8", "logo": "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", + "base": "https://livetv.sx/enx/", "timestamp": 1757721904.604711 }, "[Combat Sport: Wrestling] WWE Smackdown": { "url": "https://ddt2.dbcast.live/16kXWMV2o1G9PDOzuzy9zw/1757750786/3230392e362e3137392e313937/ff423/playlist.m3u8", "logo": "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", + "base": "https://livetv.sx/enx/", "timestamp": 1757721904.604711 }, "[Football: Bolivia Primera Division] Bolivar - Guabira": { "url": "https://ddt1.dbcast.live/BXT1oEzK_d--tnM8M1Jlww/1757750795/3230392e362e3137392e313937/ff426/playlist.m3u8", "logo": "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", + "base": "https://livetv.sx/enx/", "timestamp": 1757721904.604711 }, "[Baseball: MLB] Minnesota - Arizona": { "url": "https://ddt1.dbcast.live/uXCcBCLFE048Gzc7_2eeow/1757750806/3230392e362e3137392e313937/ff430/playlist.m3u8", "logo": "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", + "base": "https://livetv.sx/enx/", "timestamp": 1757721904.604711 }, "[Baseball: MLB] Milwaukee - Saint-Louis": { "url": "https://ddt2.dbcast.live/lK_ipHVXlxIea8PfedXQYQ/1757750815/3230392e362e3137392e313937/ff432/playlist.m3u8", "logo": "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", + "base": "https://livetv.sx/enx/", "timestamp": 1757721904.604711 }, "[Football: Argentina Liga Profesional] Lanus - Independiente R": { "url": "https://ddt2.dbcast.live/7frpo5ftDsAJ7MvWmS_96w/1757750825/3230392e362e3137392e313937/ff434/playlist.m3u8", "logo": "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", + "base": "https://livetv.sx/enx/", "timestamp": 1757721904.604711 }, "[Football: Argentina Liga Profesional] Newell`s Old Boys - Atletico Tucuman": { "url": "https://ddt1.dbcast.live/xVdG6q7epaQWM7cskH1AGQ/1757750836/3230392e362e3137392e313937/ff435/playlist.m3u8", "logo": "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", + "base": "https://livetv.sx/enx/", "timestamp": 1757721904.604711 }, "[Football: Colombia Primera A] Atletico Junior - La Equidad": { "url": "https://ddt1.dbcast.live/XEyD2nj0XMWEI-k6cBSc3g/1757750846/3230392e362e3137392e313937/ff437/playlist.m3u8", "logo": "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", + "base": "https://livetv.sx/enx/", "timestamp": 1757721904.604711 }, "[Football: Brazil Serie B] Coritiba - Goias": { "url": "https://ddt1.dbcast.live/4M6HouhhU_e8f_qDjoRSsg/1757750856/3230392e362e3137392e313937/ff436/playlist.m3u8", "logo": "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", + "base": "https://livetv.sx/enx/", "timestamp": 1757721904.604711 }, "[American Football: NCAA] Arizona Wildcats - Kansas Wildcats": { "url": "https://broadcasthub.click/hls/zujm0b9gNEQv/index.m3u8?st=p_j7iqvtXPU7RaF9NBZlVQ&e=1757736361", "logo": "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", + "base": "https://livetv.sx/enx/", "timestamp": 1757725505.510486 }, "[Football: Mexico Liga MX] Necaxa - Juarez": { "url": "https://ddt2.dbcast.live/nVc2u4EdQmOx6Put9G230g/1757754411/3230392e362e3137392e313937/ff442/playlist.m3u8", "logo": "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", + "base": "https://livetv.sx/enx/", "timestamp": 1757725505.510486 }, "[Football: Mexico Liga de Expansion] Correcaminos - Jaiba Brava": { "url": "https://ddt1.dbcast.live/-FtPoznDD1Ia-NlCgpI1Qw/1757754443/3230392e362e3137392e313937/ff443/playlist.m3u8", "logo": "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", + "base": "https://livetv.sx/enx/", "timestamp": 1757725505.510486 }, "[Baseball: MLB] San Diego - Colorado": { "url": "https://ddt1.dbcast.live/gerrR2rVEH4BiZ8aQRmITg/1757757921/3230392e362e3137392e313937/ff446/playlist.m3u8", "logo": "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", + "base": "https://livetv.sx/enx/", "timestamp": 1757729104.691189 }, "[Football: Mexico Women Liga MX] UANL (W) - Pumas (W)": { "url": "https://ddt1.dbcast.live/9Otngt3sItnlN5LBOgOBdw/1757757977/3230392e362e3137392e313937/ff464/playlist.m3u8", "logo": "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", + "base": "https://livetv.sx/enx/", "timestamp": 1757729104.691189 }, "[American Football: Canada CFL] Lions - Ottawa": { "url": "https://ab1.dbcast.live/ergfl2.m3u8?m=uFta8EPiYYjm86iZVpvT6g&c=fl2&p=3230392e362e3137392e313937&e=1757758017", "logo": "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", + "base": "https://livetv.sx/enx/", "timestamp": 1757729104.691189 }, "[Football: USA MLS Next Pro] Tacoma Defiance - Austin II": { "url": "https://ddt2.dbcast.live/2VfyYEJq3yI8yZKgJd_pOg/1757758027/3230392e362e3137392e313937/ff452/playlist.m3u8", "logo": "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", + "base": "https://livetv.sx/enx/", "timestamp": 1757729104.691189 }, "[Combat Sport: MMA] FFC": { "url": "https://ddt1.dbcast.live/jKP0IYVgcD31y0QPeulY5g/1757758058/3230392e362e3137392e313937/ff448/playlist.m3u8", "logo": "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", + "base": "https://livetv.sx/enx/", "timestamp": 1757729104.691189 }, "[American Football: NCAA] UCLA Bruins - New Mexico Lobos": { "url": "https://broadcasthub.click/hls/M1Vc77MgylFf/index.m3u8?st=ToU1LS4VLHm6xAopRJtUng&e=1757740092", "logo": "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", + "base": "https://livetv.sx/enx/", "timestamp": 1757729104.691189 }, "[Baseball: MLB] Seattle - LA Angels": { "url": "https://ddt2.dbcast.live/GFbW6FDJMLCUVWT1R1pw-Q/1757758172/3230392e362e3137392e313937/ff459/playlist.m3u8", "logo": "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", + "base": "https://livetv.sx/enx/", "timestamp": 1757729104.691189 }, "[Baseball: MLB] San Francisco - LA Dodgers": { "url": "https://ddt1.dbcast.live/EiIAyYYHqIE8goTQI21VXQ/1757758182/3230392e362e3137392e313937/ff461/playlist.m3u8", "logo": "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", + "base": "https://livetv.sx/enx/", "timestamp": 1757729104.691189 }, "[Tennis: Davis Cup Qualification] Alex De Minaur - Raphael Collignon": { "url": "https://ddt2.dbcast.live/yLO5aMKrc9dAF4occoQTnw/1757761570/3230392e362e3137392e313937/ff974/playlist.m3u8", "logo": "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", + "base": "https://livetv.sx/enx/", "timestamp": 1757732705.52622 }, "[Golf: Asian Tour] Shinhan Donghae Open": { "url": "https://ddt2.dbcast.live/6MGqrT83ehv_jJ9LITLHhg/1757761606/3230392e362e3137392e313937/ff465/playlist.m3u8", "logo": "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", + "base": "https://livetv.sx/enx/", "timestamp": 1757732705.52622 }, "[Tennis: Davis Cup] Australia - Belgium": { "url": "https://ddt2.dbcast.live/0Sqw4xZ9yiRaGqfWVDYXyg/1757761617/3230392e362e3137392e313937/ff974/playlist.m3u8", "logo": "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", + "base": "https://livetv.sx/enx/", "timestamp": 1757732705.52622 } -} \ No newline at end of file +} diff --git a/M3U8/scrapers/caches/ppv.json b/M3U8/scrapers/caches/ppv.json index 7265fa4..fdbc3ca 100644 --- a/M3U8/scrapers/caches/ppv.json +++ b/M3U8/scrapers/caches/ppv.json @@ -2,36 +2,43 @@ "[Baseball] Arizona Diamondbacks vs. Minnesota Twins": { "url": "https://gg.poocloud.in/twins/index.m3u8", "logo": "https://ppvs.su/assets/thumb/f3a9af92177c789c2803e8404325120d-thumbnail.jpg", + "base": "https://ppvs.su/", "timestamp": 1757721904.604711 }, "[Baseball] St. Louis Cardinals vs. Milwaukee Brewers": { "url": "https://gg.poocloud.in/brewers/index.m3u8", "logo": "https://ppvs.su/assets/thumb/c28dd89542bcd64d85b25b86807529a4-thumbnail.jpg", + "base": "https://ppvs.su/", "timestamp": 1757721904.604711 }, "[Wrestling] WWE: Friday Night Smackdown": { "url": "https://gg.poocloud.in/wrestling/index.m3u8", "logo": "https://ppvs.su/assets/thumb/ec7810b68fc39115ce3510bcf9687a63-thumbnail.jpg", + "base": "https://ppvs.su/", "timestamp": 1757721904.604711 }, "[Baseball] Colorado Rockies vs. San Diego Padres": { "url": "https://gg.poocloud.in/padres/index.m3u8", "logo": "https://ppvs.su/assets/thumb/ae2b1d17c96d982c0a6e920e0ff5579f-thumbnail.jpg", + "base": "https://ppvs.su/", "timestamp": 1757729104.691189 }, "[Baseball] Cincinnati Reds vs. Athletics": { "url": "https://gg.poocloud.in/athletics/index.m3u8", "logo": "https://ppvs.su/assets/thumb/acd57c8a41de2705e77f550ac7ececf4-thumbnail.jpg", + "base": "https://ppvs.su/", "timestamp": 1757729104.691189 }, "[Baseball] Los Angeles Angels vs. Seattle Mariners": { "url": "https://gg.poocloud.in/mariners/index.m3u8", "logo": "https://ppvs.su/assets/thumb/8971b329d6f619350f3b42915d3d548c-thumbnail.jpg", + "base": "https://ppvs.su/", "timestamp": 1757729104.691189 }, "[Baseball] Los Angeles Dodgers vs. San Francisco Giants": { "url": "https://gg.poocloud.in/giants/index.m3u8", "logo": "https://ppvs.su/assets/thumb/25adb7fd6882369cd4c08e9b0757623b-thumbnail.jpg", + "base": "https://ppvs.su/", "timestamp": 1757729104.691189 } } \ No newline at end of file diff --git a/M3U8/scrapers/caches/streambtw.json b/M3U8/scrapers/caches/streambtw.json index 0142203..08ce2d1 100644 --- a/M3U8/scrapers/caches/streambtw.json +++ b/M3U8/scrapers/caches/streambtw.json @@ -2,96 +2,115 @@ "[Soccer] Cyprus vs Romania": { "url": "https://streameast247.com/playlist/stream_4.m3u8", "logo": "https://artwork.espncdn.com/categories/119cfa41-71d4-39bf-a790-6273a52b0259/logo/original_201809051718.svg", + "base": "https://streambtw.com/", "timestamp": 1757682303.329149 }, "[Soccer] Albania vs Latvia": { "url": "https://streameast247.com/playlist/stream_5.m3u8", "logo": "https://artwork.espncdn.com/categories/119cfa41-71d4-39bf-a790-6273a52b0259/logo/original_201809051718.svg", + "base": "https://streambtw.com/", "timestamp": 1757682303.329149 }, "[Soccer] France vs Iceland": { "url": "https://streameast247.com/playlist/stream_6.m3u8", "logo": "https://artwork.espncdn.com/categories/119cfa41-71d4-39bf-a790-6273a52b0259/logo/original_201809051718.svg", + "base": "https://streambtw.com/", "timestamp": 1757682303.329149 }, "[Soccer] Hungary vs Portugal": { "url": "https://streameast247.com/playlist/stream_7.m3u8", "logo": "https://artwork.espncdn.com/categories/119cfa41-71d4-39bf-a790-6273a52b0259/logo/original_201809051718.svg", + "base": "https://streambtw.com/", "timestamp": 1757682303.329149 }, "[MLB] Pittsburgh vs Baltimore": { "url": "https://smsbusinesshub.com/playlist/stream_mlb1.m3u8", "logo": "https://artwork.espncdn.com/categories/b38f959b-7865-31ac-8841-b88355519e10/logo/original_201811152008.svg", + "base": "https://streambtw.com/", "timestamp": 1757682303.329149 }, "[MLB] Kansas City vs Cleveland": { "url": "https://smsbusinesshub.com/playlist/stream_mlb2.m3u8", "logo": "https://artwork.espncdn.com/categories/b38f959b-7865-31ac-8841-b88355519e10/logo/original_201811152008.svg", + "base": "https://streambtw.com/", "timestamp": 1757682303.329149 }, "[MLB] Washington vs Miami": { "url": "https://smsbusinesshub.com/playlist/stream_mlb3.m3u8", "logo": "https://artwork.espncdn.com/categories/b38f959b-7865-31ac-8841-b88355519e10/logo/original_201811152008.svg", + "base": "https://streambtw.com/", "timestamp": 1757682303.329149 }, "[MLB] N.Y. Mets vs Philadelphia": { "url": "https://smsbusinesshub.com/playlist/stream_mlb4.m3u8", "logo": "https://artwork.espncdn.com/categories/b38f959b-7865-31ac-8841-b88355519e10/logo/original_201811152008.svg", + "base": "https://streambtw.com/", "timestamp": 1757682303.329149 }, "[MLB] Detroit vs N.Y. Yankees": { "url": "https://smsbusinesshub.com/playlist/stream_mlb5.m3u8", "logo": "https://artwork.espncdn.com/categories/b38f959b-7865-31ac-8841-b88355519e10/logo/original_201811152008.svg", + "base": "https://streambtw.com/", "timestamp": 1757682303.329149 }, "[MLB] Houston vs Toronto": { "url": "https://smsbusinesshub.com/playlist/stream_mlb6.m3u8", "logo": "https://artwork.espncdn.com/categories/b38f959b-7865-31ac-8841-b88355519e10/logo/original_201811152008.svg", + "base": "https://streambtw.com/", "timestamp": 1757682303.329149 }, "[MLB] Chi. Cubs vs Atlanta": { "url": "https://smsbusinesshub.com/playlist/stream_mlb7.m3u8", "logo": "https://artwork.espncdn.com/categories/b38f959b-7865-31ac-8841-b88355519e10/logo/original_201811152008.svg", + "base": "https://streambtw.com/", "timestamp": 1757682303.329149 }, "[MLB] Tampa Bay vs Chi. White Sox": { "url": "https://smsbusinesshub.com/playlist/stream_mlb8.m3u8", "logo": "https://artwork.espncdn.com/categories/b38f959b-7865-31ac-8841-b88355519e10/logo/original_201811152008.svg", + "base": "https://streambtw.com/", "timestamp": 1757682303.329149 }, "[MLB] Milwaukee vs Texas": { "url": "https://smsbusinesshub.com/playlist/stream_mlb9.m3u8", "logo": "https://artwork.espncdn.com/categories/b38f959b-7865-31ac-8841-b88355519e10/logo/original_201811152008.svg", + "base": "https://streambtw.com/", "timestamp": 1757682303.329149 }, "[MLB] Minnesota vs L.A. Angels": { "url": "https://smsbusinesshub.com/playlist/stream_mlb10.m3u8", "logo": "https://artwork.espncdn.com/categories/b38f959b-7865-31ac-8841-b88355519e10/logo/original_201811152008.svg", + "base": "https://streambtw.com/", "timestamp": 1757682303.329149 }, "[MLB] St. Louis vs Seattle": { "url": "https://smsbusinesshub.com/playlist/stream_mlb11.m3u8", "logo": "https://artwork.espncdn.com/categories/b38f959b-7865-31ac-8841-b88355519e10/logo/original_201811152008.svg", + "base": "https://streambtw.com/", "timestamp": 1757682303.329149 }, "[MLB] Cincinnati vs San Diego": { "url": "https://smsbusinesshub.com/playlist/stream_mlb12.m3u8", "logo": "https://artwork.espncdn.com/categories/b38f959b-7865-31ac-8841-b88355519e10/logo/original_201811152008.svg", + "base": "https://streambtw.com/", "timestamp": 1757682303.329149 }, "[MLB] Arizona vs San Francisco": { "url": "https://smsbusinesshub.com/playlist/stream_mlb13.m3u8", "logo": "https://artwork.espncdn.com/categories/b38f959b-7865-31ac-8841-b88355519e10/logo/original_201811152008.svg", + "base": "https://streambtw.com/", "timestamp": 1757682303.329149 }, "[MLB] Boston vs Athletics": { "url": "https://smsbusinesshub.com/playlist/stream_mlb14.m3u8", "logo": "https://artwork.espncdn.com/categories/b38f959b-7865-31ac-8841-b88355519e10/logo/original_201811152008.svg", + "base": "https://streambtw.com/", "timestamp": 1757682303.329149 }, "[MLB] Colorado vs L.A. Dodgers": { "url": "https://smsbusinesshub.com/playlist/stream_mlb15.m3u8", "logo": "https://artwork.espncdn.com/categories/b38f959b-7865-31ac-8841-b88355519e10/logo/original_201811152008.svg", + "base": "https://streambtw.com/", "timestamp": 1757682303.329149 } } \ No newline at end of file diff --git a/M3U8/scrapers/caches/streameast.json b/M3U8/scrapers/caches/streameast.json index 279db57..8cd9e3f 100644 --- a/M3U8/scrapers/caches/streameast.json +++ b/M3U8/scrapers/caches/streameast.json @@ -2,51 +2,61 @@ "[MLB] St. Louis Cardinals - Milwaukee Brewers": { "url": "https://lb10.strmd.top/secure/pJawemMdGvTMviALZpuNBvjaFZYSVhBp/echo/stream/milwaukee-brewers-vs-stlouis-cardinals-baseball-173229/1/playlist.m3u8", "logo": "https://i.gyazo.com/0fe7865ef2f06c9507791b24f04dbca8.png", + "base": "https://streameast.ga", "timestamp": 1757721904.604711 }, "[MLB] Arizona Diamondbacks - Minnesota Twins": { "url": "https://lb10.strmd.top/secure/zBnEFkJErZYuzGUYJktohtdBHhXJkJOF/delta/stream/live_mlb_twins-diamondbacks-live-streaming-417853296/1/playlist.m3u8", "logo": "https://i.gyazo.com/0fe7865ef2f06c9507791b24f04dbca8.png", + "base": "https://streameast.ga", "timestamp": 1757721904.604711 }, "[UFC] BKFC 80: Leonardo Perdomo vs Arnold Adams": { "url": "https://lb10.strmd.top/secure/DgjCyseThNvpAptcnfZSLYmfOrPwKDya/alpha/stream/bkfc-80-leonardo-perdomo-vs-arnold-adams/1/playlist.m3u8", "logo": "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", + "base": "https://streameast.ga", "timestamp": 1757721904.604711 }, "[MLB] New York Yankees - Boston Red Sox": { "url": "https://lb10.strmd.top/secure/eAZxQJtKWvzZLCYQdaPiiILvvfZePXDw/echo/stream/boston-red-sox-vs-new-york-yankees-baseball-173224/1/playlist.m3u8", "logo": "https://i.gyazo.com/0fe7865ef2f06c9507791b24f04dbca8.png", + "base": "https://streameast.ga", "timestamp": 1757721904.604711 }, "[CFB] Kansas State Wildcats - Arizona Wildcats": { "url": "https://lb10.strmd.top/secure/KCJtWABXfBLkyOtYNFHMCkilExzaplcA/echo/stream/arizona-vs-kansas-state-american-football-17776/1/playlist.m3u8", "logo": "https://i.gyazo.com/ca63b40c86e757436de9d34d369b24f8.png", + "base": "https://streameast.ga", "timestamp": 1757725505.510486 }, "[WWE] WWE Friday Night Smackdown": { "url": "https://lb10.strmd.top/secure/KPLpdKZEFDgIMlAKnoRUWaTnMNIKfLii/alpha/stream/wwe-friday-night-smackdown/1/playlist.m3u8", "logo": "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", + "base": "https://streameast.ga", "timestamp": 1757725505.510486 }, "[CFB] New Mexico Lobos - UCLA Bruins": { "url": "https://lb10.strmd.top/secure/DVLLjIwkhkdoCTJNhmsoFeoktnAiFdDd/echo/stream/indiana-vs-indiana-state-american-football-17799/1/playlist.m3u8", "logo": "https://i.gyazo.com/ca63b40c86e757436de9d34d369b24f8.png", + "base": "https://streameast.ga", "timestamp": 1757729104.691189 }, "[MLB] Colorado Rockies - San Diego Padres": { "url": "https://lb10.strmd.top/secure/nDuhOdLPpdvzhAdggQiPtrkXWKqxbYNM/echo/stream/san-diego-padres-vs-colorado-rockies-baseball-173231/1/playlist.m3u8", "logo": "https://i.gyazo.com/0fe7865ef2f06c9507791b24f04dbca8.png", + "base": "https://streameast.ga", "timestamp": 1757729104.691189 }, "[MLB] Los Angeles Dodgers - San Francisco Giants": { "url": "https://mainstreams.pro/hls/hfzljfahzcax29.m3u8?st=6pxPf-zHFDQ-PSFJXxYVbub5B3unjacCOub30IoyL-4&e=1757743512", "logo": "https://i.gyazo.com/0fe7865ef2f06c9507791b24f04dbca8.png", + "base": "https://streameast.ga", "timestamp": 1757732705.52622 }, "[MLB] Cincinnati Reds - Athletics": { "url": "https://mainstreams.pro/hls/jzabfazkbf27.m3u8?st=WhD6cNVHQgvu5aWHjLos52gg8yg-mqDDX_3V-4v8hvA&e=1757743541", "logo": "https://i.gyazo.com/0fe7865ef2f06c9507791b24f04dbca8.png", + "base": "https://streameast.ga", "timestamp": 1757732705.52622 } } \ No newline at end of file diff --git a/M3U8/scrapers/caches/tvpass.json b/M3U8/scrapers/caches/tvpass.json index 5cfff75..b7cfa0e 100644 --- a/M3U8/scrapers/caches/tvpass.json +++ b/M3U8/scrapers/caches/tvpass.json @@ -2,101 +2,121 @@ "[MLB] Tampa Bay Rays @ Chicago Cubs": { "url": "http://origin.thetvapp.to/hls/MLB01/mono.m3u8", "logo": "https://i.gyazo.com/0fe7865ef2f06c9507791b24f04dbca8.png", + "base": "https://tvpass.org", "timestamp": 1757693103.550906 }, "[NCAAF] Indiana State Sycamores @ Indiana Hoosiers": { "url": "http://origin.thetvapp.to/hls/BTN/mono.m3u8", "logo": "https://i.gyazo.com/ca63b40c86e757436de9d34d369b24f8.png", + "base": "https://tvpass.org", "timestamp": 1757693103.550906 }, "[MLB] Pittsburgh Pirates @ Washington Nationals": { "url": "http://origin.thetvapp.to/hls/MLB02/mono.m3u8", "logo": "https://i.gyazo.com/0fe7865ef2f06c9507791b24f04dbca8.png", + "base": "https://tvpass.org", "timestamp": 1757693103.550906 }, "[MLB] Kansas City Royals @ Philadelphia Phillies": { "url": "http://origin.thetvapp.to/hls/MLB03/mono.m3u8", "logo": "https://i.gyazo.com/0fe7865ef2f06c9507791b24f04dbca8.png", + "base": "https://tvpass.org", "timestamp": 1757693103.550906 }, "[NCAAF] Colgate Raiders @ Syracuse Orange": { "url": "http://origin.thetvapp.to/hls/ACCNetwork/mono.m3u8", "logo": "https://i.gyazo.com/ca63b40c86e757436de9d34d369b24f8.png", + "base": "https://tvpass.org", "timestamp": 1757693103.550906 }, "[MLB] Baltimore Orioles @ Toronto Blue Jays": { "url": "http://origin.thetvapp.to/hls/MLB04/mono.m3u8", "logo": "https://i.gyazo.com/0fe7865ef2f06c9507791b24f04dbca8.png", + "base": "https://tvpass.org", "timestamp": 1757693103.550906 }, "[MLB] New York Yankees @ Boston Red Sox": { "url": "http://origin.thetvapp.to/hls/MLB05/mono.m3u8", "logo": "https://i.gyazo.com/0fe7865ef2f06c9507791b24f04dbca8.png", + "base": "https://tvpass.org", "timestamp": 1757693103.550906 }, "[MLB] Chicago White Sox @ Cleveland Guardians": { "url": "http://origin.thetvapp.to/hls/MLB06/mono.m3u8", "logo": "https://i.gyazo.com/0fe7865ef2f06c9507791b24f04dbca8.png", + "base": "https://tvpass.org", "timestamp": 1757693103.550906 }, "[MLB] Detroit Tigers @ Miami Marlins": { "url": "http://origin.thetvapp.to/hls/MLB07/mono.m3u8", "logo": "https://i.gyazo.com/0fe7865ef2f06c9507791b24f04dbca8.png", + "base": "https://tvpass.org", "timestamp": 1757693103.550906 }, "[MLB] Texas Rangers @ New York Mets": { "url": "http://origin.thetvapp.to/hls/MLB08/mono.m3u8", "logo": "https://i.gyazo.com/0fe7865ef2f06c9507791b24f04dbca8.png", + "base": "https://tvpass.org", "timestamp": 1757693103.550906 }, "[MLB] Houston Astros @ Atlanta Braves": { "url": "http://origin.thetvapp.to/hls/MLB09/mono.m3u8", "logo": "https://i.gyazo.com/0fe7865ef2f06c9507791b24f04dbca8.png", + "base": "https://tvpass.org", "timestamp": 1757693103.550906 }, "[NCAAF] Colorado Buffaloes @ Houston Cougars": { "url": "http://origin.thetvapp.to/hls/ESPN/mono.m3u8", "logo": "https://i.gyazo.com/ca63b40c86e757436de9d34d369b24f8.png", + "base": "https://tvpass.org", "timestamp": 1757693103.550906 }, "[MLB] St. Louis Cardinals @ Milwaukee Brewers": { "url": "http://origin.thetvapp.to/hls/MLB11/mono.m3u8", "logo": "https://i.gyazo.com/0fe7865ef2f06c9507791b24f04dbca8.png", + "base": "https://tvpass.org", "timestamp": 1757693103.550906 }, "[MLB] Arizona Diamondbacks @ Minnesota Twins": { "url": "http://origin.thetvapp.to/hls/MLB10/mono.m3u8", "logo": "https://i.gyazo.com/0fe7865ef2f06c9507791b24f04dbca8.png", + "base": "https://tvpass.org", "timestamp": 1757693103.550906 }, "[NCAAF] Kansas State Wildcats @ Arizona Wildcats": { "url": "http://origin.thetvapp.to/hls/WNYWDT1/mono.m3u8", "logo": "https://i.gyazo.com/ca63b40c86e757436de9d34d369b24f8.png", + "base": "https://tvpass.org", "timestamp": 1757693103.550906 }, "[MLB] Colorado Rockies @ San Diego Padres": { "url": "http://origin.thetvapp.to/hls/MLB12/mono.m3u8", "logo": "https://i.gyazo.com/0fe7865ef2f06c9507791b24f04dbca8.png", + "base": "https://tvpass.org", "timestamp": 1757693103.550906 }, "[NCAAF] New Mexico Lobos @ UCLA Bruins": { "url": "http://origin.thetvapp.to/hls/BTN/mono.m3u8", "logo": "https://i.gyazo.com/ca63b40c86e757436de9d34d369b24f8.png", + "base": "https://tvpass.org", "timestamp": 1757693103.550906 }, "[MLB] Cincinnati Reds @ Oakland Athletics": { "url": "http://origin.thetvapp.to/hls/MLB13/mono.m3u8", "logo": "https://i.gyazo.com/0fe7865ef2f06c9507791b24f04dbca8.png", + "base": "https://tvpass.org", "timestamp": 1757693103.550906 }, "[MLB] Los Angeles Angels @ Seattle Mariners": { "url": "http://origin.thetvapp.to/hls/MLB14/mono.m3u8", "logo": "https://i.gyazo.com/0fe7865ef2f06c9507791b24f04dbca8.png", + "base": "https://tvpass.org", "timestamp": 1757693103.550906 }, "[MLB] Los Angeles Dodgers @ San Francisco Giants": { "url": "http://origin.thetvapp.to/hls/MLB15/mono.m3u8", "logo": "https://i.gyazo.com/0fe7865ef2f06c9507791b24f04dbca8.png", + "base": "https://tvpass.org", "timestamp": 1757693103.550906 } } \ No newline at end of file diff --git a/M3U8/scrapers/fstv.py b/M3U8/scrapers/fstv.py index 1ab4012..dabb2ef 100644 --- a/M3U8/scrapers/fstv.py +++ b/M3U8/scrapers/fstv.py @@ -1,24 +1,31 @@ -import asyncio -from urllib.parse import urljoin +import json +from pathlib import Path +from urllib.parse import unquote, urljoin import httpx from selectolax.parser import HTMLParser -from .utils import LOGOS, get_base, get_logger +from .utils import LOGOS, get_base, get_logger, load_cache, now, safe_process_event log = get_logger(__name__) urls: dict[str, dict[str, str]] = {} MIRRORS = [ - "https://fstv.online", - "https://fstv.space", "https://fstv.zip", + "https://fstv.space", + "https://fstv.online", "https://fstv.us", ] +CACHE_FILE = Path(__file__).parent / "caches" / "fstv.json" -async def get_events(client: httpx.AsyncClient, base_url: str) -> list[tuple[str, str]]: + +async def get_events( + client: httpx.AsyncClient, + base_url: str, + cached_hrefs: set[str], +) -> list[dict[str, str]]: log.info(f'Scraping from "{base_url}"') try: @@ -31,7 +38,7 @@ async def get_events(client: httpx.AsyncClient, base_url: str) -> list[tuple[str soup = HTMLParser(r.text) - events = {} + events = [] for wrpr in soup.css("div.fixtures-live-wrapper"): for games in wrpr.css(".match-table-item"): @@ -48,21 +55,40 @@ async def get_events(client: httpx.AsyncClient, base_url: str) -> list[tuple[str else: event_name = full_text - events[event_name] = urljoin( - base_url, league_match.attributes.get("href") + href = league_match.attributes.get("href") + + link = urljoin(base_url, href) + + if cached_hrefs & {href}: + continue + + events.append( + { + "sport": event_name, + "link": link, + "logo": LOGOS.get( + event_name, + "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", + ), + "href": href, + } ) - return events.items() + return events -async def process_events(client: httpx.AsyncClient, url: str) -> tuple[str, list[str]]: +async def process_event( + client: httpx.AsyncClient, + url: str, + url_num: int, +) -> tuple[str, str]: try: r = await client.get(url) r.raise_for_status() except Exception as e: - log.error(f'Failed to fetch "{url}"\n{e}') + log.error(f'URL {url_num}) Failed to fetch "{url}"\n{e}') - return [] + return "", "" soup = HTMLParser(r.text) @@ -77,43 +103,65 @@ async def process_events(client: httpx.AsyncClient, url: str) -> tuple[str, list og_title.attributes.get("content", "").split(" start on")[0].strip() ) - btns = soup.css("button.btn-server") + if src := soup.css_first("iframe").attributes.get("src", ""): + log.info(f"URL {url_num}) Captured M3U8") + return match_name, unquote(src).split("link=")[-1] - return match_name, [ - btn.attributes.get("data-link") for btn in btns if "data-link" in btn.attributes - ] + log.info(f"URL {url_num}) No M3U8 found") + return "", "" async def main(client: httpx.AsyncClient) -> None: + cached_urls = load_cache(CACHE_FILE, exp=14400) + cached_hrefs = {entry["href"] for entry in cached_urls.values()} + cached_count = len(cached_urls) + urls.update(cached_urls) + + log.info(f"Collected {cached_count} event(s) from cache") + if not (base_url := await get_base(client, MIRRORS)): log.warning("No working FSTV mirrors") return - events = await get_events(client, base_url) + events = await get_events( + client, + base_url, + cached_hrefs, + ) - tasks = [process_events(client, href) for _, href in events if href] - results = await asyncio.gather(*tasks) + log.info(f"Processing {len(events)} new URL(s)") - for (event, _), (match_name, m3u8_urls) in zip(events, results): - event = event.strip() + for i, ev in enumerate(events, start=1): + match_name, url = await safe_process_event( + lambda: process_event( + client, + ev["link"], + url_num=i, + ), + url_num=i, + log=log, + ) - if not m3u8_urls: - continue - - for i, link in enumerate(m3u8_urls, start=1): + if url: key = ( - f"[{event}] {match_name} (S{i})" if match_name else f"[{event}] (S{i})" + f"[{ev['sport']}] {match_name} (FSTV)" + if match_name + else f"[{ev['sport']}] (FSTV)" ) - urls[key] = { - "url": link, - "logo": LOGOS.get( - event, - "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", - ), + entry = { + "url": url, + "logo": ev["logo"], + "base": base_url, + "timestamp": now.timestamp(), + "href": ev["href"], } - log.info(f"Collected {len(urls)} live event(s)") + urls[key] = cached_urls[key] = entry + if new_count := len(cached_urls) - cached_count: + log.info(f"Collected and cached {new_count} new event(s)") + else: + log.info("No new events found") -# add caching + CACHE_FILE.write_text(json.dumps(cached_urls, indent=2), encoding="utf-8") diff --git a/M3U8/scrapers/livetvsx.py b/M3U8/scrapers/livetvsx.py index 0c062b0..e0c102e 100644 --- a/M3U8/scrapers/livetvsx.py +++ b/M3U8/scrapers/livetvsx.py @@ -38,6 +38,8 @@ CERT_FILE = Path(__file__).parent / "utils" / "cached-ca.pem" CACHE_FILE = Path(__file__).parent / "caches" / "livetvsx.json" +exist_sprts = set(LOGOS.keys()) + async def write_to_cert( client: httpx.AsyncClient, @@ -248,9 +250,13 @@ async def get_events( else ("", "") ) - key = f"[{sport}: {event}] {title}" + key = f"[{sport}: {event}] {title} (LTVSX)" - if key in cached_keys: + if cached_keys & {key}: + elem.clear() + continue + + if exist_sprts & {sport, event}: elem.clear() continue @@ -293,7 +299,7 @@ async def main(client: httpx.AsyncClient) -> None: title = ev["title"] link = ev["link"] - key = f"[{sport}: {event}] {title}" + key = f"[{sport}: {event}] {title} (LTVSX)" url = await safe_process_event( lambda: process_event(link, url_num=i), @@ -308,6 +314,7 @@ async def main(client: httpx.AsyncClient) -> None: sport, "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", ), + "base": "https://livetv.sx/enx/", "timestamp": now.timestamp(), } diff --git a/M3U8/scrapers/ppv.py b/M3U8/scrapers/ppv.py index fbe1da5..4b2b655 100644 --- a/M3U8/scrapers/ppv.py +++ b/M3U8/scrapers/ppv.py @@ -145,9 +145,9 @@ async def get_events( event["uri_name"], ) - key = f"[{sport}] {name}" + key = f"[{sport}] {name} (PPV)" - if key in cached_keys: + if cached_keys & {key}: continue start_dt = datetime.fromtimestamp(start_ts, tz=TZ) - timedelta(minutes=30) @@ -198,14 +198,15 @@ async def main(client: httpx.AsyncClient) -> None: ) if url: + key = f"[{ev['sport']}] {ev['event']} (PPV)" + entry = { "url": url, "logo": ev["logo"], + "base": base_url, "timestamp": now.timestamp(), } - key = f"[{ev['sport']}] {ev['event']}" - urls[key] = cached_urls[key] = entry if new_count := len(cached_urls) - cached_count: diff --git a/M3U8/scrapers/streambtw.py b/M3U8/scrapers/streambtw.py index 0e75121..cd3b75a 100644 --- a/M3U8/scrapers/streambtw.py +++ b/M3U8/scrapers/streambtw.py @@ -98,13 +98,16 @@ async def main(client: httpx.AsyncClient) -> None: ) if url: + key = f"[{ev['sport']}] {ev['event']} (SBTW)" + entry = { "url": url, "logo": ev["logo"], + "base": BASE_URL, "timestamp": now.timestamp(), } - urls[f"[{ev['sport']}] {ev['event']}"] = entry + urls[key] = entry log.info(f"Collected {len(urls)} event(s)") diff --git a/M3U8/scrapers/streameast.py b/M3U8/scrapers/streameast.py index 706d3f3..1ff8818 100644 --- a/M3U8/scrapers/streameast.py +++ b/M3U8/scrapers/streameast.py @@ -104,7 +104,7 @@ async def process_event(url: str, url_num: int) -> str | None: async def get_events( client: httpx.AsyncClient, url: str, - cached_keys: list[str], + cached_keys: set[str], ) -> list[dict[str, str]]: try: r = await client.get(url) @@ -136,9 +136,9 @@ async def get_events( time_text = time_span.text(strip=True) timestamp = int(time_span.attributes.get("data-zaman")) - key = f"[{sport}] {name}" + key = f"[{sport}] {name} (SEAST)" - if key in cached_keys: + if cached_keys & {key}: continue event_dt = datetime.fromtimestamp(timestamp, TZ) @@ -188,14 +188,15 @@ async def main(client: httpx.AsyncClient) -> None: ) if url: + key = f"[{ev['sport']}] {ev['event']} (SEAST)" + entry = { "url": url, "logo": ev["logo"], + "base": base_url, "timestamp": now.timestamp(), } - key = f"[{ev['sport']}] {ev['event']}" - urls[key] = cached_urls[key] = entry if new_count := len(cached_urls) - cached_count: diff --git a/M3U8/scrapers/tvpass.py b/M3U8/scrapers/tvpass.py index 3bddf96..d28683c 100644 --- a/M3U8/scrapers/tvpass.py +++ b/M3U8/scrapers/tvpass.py @@ -51,15 +51,20 @@ async def main(client: httpx.AsyncClient) -> None: tvg_name = "(".join(tvg_name.split("(")[:-1]).strip() if url.endswith("/hd"): - urls[f"[{sport}] {tvg_name}"] = { + key = f"[{sport}] {tvg_name} (TVP)" + + entry = { "url": f"http://origin.thetvapp.to/hls/{url.split('/')[-2]}/mono.m3u8", "logo": LOGOS.get( sport, "https://i.gyazo.com/ec27417a9644ae517196494afa72d2b9.png", ), + "base": "https://tvpass.org", "timestamp": now.timestamp(), } + urls[key] = entry + CACHE_FILE.write_text(json.dumps(urls, indent=2), encoding="utf-8") log.info(f"Cached {len(urls)} event(s)")