doms/iptv
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 3

e

Browse source
This commit is contained in:
doms9 2025-10-01 11:57:49 -04:00
parent 7103b0f1c4
commit 00000d9937
17 changed files with 597 additions and 524 deletions
Download patch file Download diff file Expand all files Collapse all files

19
M3U8/scrapers/utils/__init__.py
View file

@ -1,19 +1,12 @@
from .cache import load_cache, write_cache
from .config import TZ, leagues, now
from .caching import Cache
from .config import Time, leagues
from .logger import get_logger
from .network import CLIENT, UA, capture_req, get_base, new_browser, safe_process_event
from .webwork import network
__all__ = [
"CLIENT",
"TZ",
"UA",
"capture_req",
"get_base",
"Cache",
"Time",
"get_logger",
"leagues",
"load_cache",
"new_browser",
"now",
"safe_process_event",
"write_cache",
"network",
]

50
M3U8/scrapers/utils/cache.py
View file

@ -1,50 +0,0 @@
import json
from datetime import datetime
from pathlib import Path
from .config import now
def near_hr(dt: datetime) -> float:
return dt.replace(minute=0, second=0, microsecond=0).timestamp()
def is_fresh(
entry: dict,
nearest_hr: bool,
exp: int,
) -> bool:
ts: float | int = entry.get("timestamp", 31496400)
if nearest_hr:
ts = near_hr(datetime.fromtimestamp(ts))
return now.timestamp() - ts < exp
def load_cache(
file: Path,
exp: int | float,
nearest_hr: bool = False,
per_entry: bool = True,
) -> dict[str, dict[str, str | float]]:
try:
data: dict = json.loads(file.read_text(encoding="utf-8"))
except (FileNotFoundError, json.JSONDecodeError):
return {}
if per_entry:
return {k: v for k, v in data.items() if is_fresh(v, nearest_hr, exp)}
ts: float | int = data.get("timestamp", 31496400)
if nearest_hr:
ts = near_hr(datetime.fromtimestamp(ts))
return data if now.timestamp() - ts < exp else {}
def write_cache(file: Path, data: dict) -> None:
file.parent.mkdir(parents=True, exist_ok=True)
file.write_text(json.dumps(data, indent=2), encoding="utf-8")

65
M3U8/scrapers/utils/caching.py Normal file
View file

@ -0,0 +1,65 @@
import json
from datetime import datetime
from pathlib import Path
from .config import Time
class Cache:
def __init__(self, file: Path, exp: int | float) -> None:
self.file = file
self.exp = exp
@staticmethod
def near_hr(dt: datetime) -> float:
return dt.replace(minute=0, second=0, microsecond=0).timestamp()
def is_fresh(
self,
entry: dict,
nearest_hr: bool,
) -> bool:
ts: float | int = entry.get("timestamp", 31496400)
if nearest_hr:
ts = self.near_hr(Time.from_ts(ts))
return Time.now().timestamp() - ts < self.exp
def load(
self,
nearest_hr: bool = False,
per_entry: bool = True,
) -> dict[str, dict[str, str | float]]:
try:
data: dict = json.loads(self.file.read_text(encoding="utf-8"))
except (FileNotFoundError, json.JSONDecodeError):
return {}
if not data:
return {}
if per_entry:
return {k: v for k, v in data.items() if self.is_fresh(v, nearest_hr)}
ts: float | int = data.get("timestamp", 31496400)
if nearest_hr:
ts = self.near_hr(Time.from_ts(ts))
return data if self.is_fresh({"timestamp": ts}, False) else {}
def write(self, data: dict) -> None:
self.file.parent.mkdir(parents=True, exist_ok=True)
self.file.write_text(
json.dumps(
data,
indent=2,
ensure_ascii=False,
),
encoding="utf-8",
)
__all__ = ["Cache"]

76
M3U8/scrapers/utils/config.py
View file

@ -1,27 +1,79 @@
import json
import re
from datetime import datetime
from datetime import datetime, timedelta, timezone
from pathlib import Path
import pytz
TZ = pytz.timezone("America/New_York")
ZONES = {"ET": pytz.timezone("America/New_York"), "UTC": timezone.utc}
now = datetime.now(TZ)
ZONES["EDT"] = ZONES["EST"] = ZONES["ET"]
live_img = "https://i.gyazo.com/978f2eb4a199ca5b56b447aded0cb9e3.png"
leagues_file = Path(__file__).parent / "leagues.json"
class Time(datetime):
TZ = timezone.utc
@classmethod
def now(cls) -> "Time":
dt = datetime.now(cls.TZ)
return cls.fromtimestamp(dt.timestamp(), tz=cls.TZ)
@classmethod
def from_ts(cls, ts: int | float) -> "Time":
return cls.fromtimestamp(ts, tz=cls.TZ)
def delta(self, **kwargs) -> "Time":
new_dt = super().__add__(timedelta(**kwargs))
return self.__class__.fromtimestamp(new_dt.timestamp(), tz=new_dt.tzinfo)
@classmethod
def from_str(cls, s: str, fmt: str | None = None) -> "Time":
pattern = r"\b(ET|UTC|EST|EDT)\b"
match = re.search(pattern, s)
tz = ZONES.get(match[1]) if match else cls.TZ
cleaned_str = re.sub(pattern, "", s).strip()
if fmt:
dt = datetime.strptime(cleaned_str, fmt)
else:
formats = [
"%Y-%m-%d %H:%M",
"%Y-%m-%d %H:%M:%S",
]
for frmt in formats:
try:
dt = datetime.strptime(cleaned_str, frmt)
break
except ValueError:
continue
else:
return cls.from_ts(31496400)
dt = tz.localize(dt) if hasattr(tz, "localize") else dt.replace(tzinfo=tz)
return cls.fromtimestamp(dt.astimezone(cls.TZ).timestamp(), tz=cls.TZ)
def to_tz(self, tzone: str) -> "Time":
dt = self.astimezone(ZONES[tzone])
return self.__class__.fromtimestamp(dt.timestamp(), tz=ZONES[tzone])
class Leagues:
def __init__(self) -> None:
self.data = json.loads(leagues_file.read_text(encoding="utf-8"))
self.data = json.loads(
(Path(__file__).parent / "leagues.json").read_text(encoding="utf-8")
)
self.live_img = "https://i.gyazo.com/978f2eb4a199ca5b56b447aded0cb9e3.png"
def teams(self, league: str) -> list[str]:
return self.data["teams"].get(league, [])
def info(self, name: str) -> tuple[str | str]:
def info(self, name: str) -> tuple[str | None, str]:
name = name.upper()
if match := next(
@ -36,9 +88,9 @@ class Leagues:
):
tvg_id, logo = match
return (tvg_id, logo or live_img)
return (tvg_id, logo or self.live_img)
return ("Live.Event.us", live_img)
return (None, self.live_img)
def is_valid(self, event: str, league: str) -> bool:
if match := re.search(r"(\-|vs.?)", event):
@ -48,5 +100,11 @@ class Leagues:
return event.lower() == "nfl redzone" if league == "NFL" else False
@property
def league_names(self) -> list[str]:
return self.data["teams"].keys()
leagues = Leagues()
__all__ = ["leagues", "Time"]

13
M3U8/scrapers/utils/logger.py
View file

@ -10,22 +10,23 @@ LOG_FMT = (
)
COLORS = {
"DEBUG": "\033[37m",
"DEBUG": "\033[36m",
"INFO": "\033[32m",
"WARNING": "\033[33m",
"ERROR": "\033[31m",
"CRITICAL": "\033[41m",
"CRITICAL": "\033[1;41m",
"reset": "\033[0m",
}
class ColorFormatter(logging.Formatter):
def format(self, record) -> str:
color = COLORS.get(record.levelname, "")
color = COLORS.get(record.levelname, COLORS["reset"])
levelname = record.levelname
record.levelname = f"{color}{levelname}{COLORS['reset']}"
record.levelname = f"{color}{levelname:<8}{COLORS['reset']}"
formatted = super().format(record)
record.levelname = levelname
return formatted
@ -41,5 +42,9 @@ def get_logger(name: str | None = None) -> logging.Logger:
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.setLevel(logging.INFO)
logger.propagate = False
return logger
__all__ = ["get_logger", "ColorFormatter"]

154
M3U8/scrapers/utils/network.py
View file

@ -1,154 +0,0 @@
import asyncio
import logging
import re
from collections.abc import Callable
from typing import Any
import httpx
from playwright.async_api import Browser, BrowserContext, Playwright, Request
UA = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0"
)
CLIENT = httpx.AsyncClient(
timeout=5,
follow_redirects=True,
headers={"User-Agent": UA},
)
async def check_status(client: httpx.AsyncClient, url: str) -> bool:
try:
r = await client.get(url)
r.raise_for_status()
except Exception:
return False
return r.status_code == 200
async def get_base(client: httpx.AsyncClient, mirrors: list[str]) -> str | None:
tasks = [check_status(client, link) for link in mirrors]
results = await asyncio.gather(*tasks)
try:
return [url for url, ok in zip(mirrors, results) if ok][0]
except IndexError:
return
async def safe_process_event(
fn: Callable,
url_num: int,
timeout: int | float = 15,
log: logging.Logger | None = None,
) -> Any | None:
if not log:
log = logging.getLogger(__name__)
task = asyncio.create_task(fn())
try:
return await asyncio.wait_for(task, timeout=timeout)
except asyncio.TimeoutError:
log.warning(f"URL {url_num}) Timed out after {timeout}s, skipping event")
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
except Exception as e:
log.debug(f"URL {url_num}) Ignore exception after timeout: {e}")
def capture_req(
req: Request,
captured: list[str],
got_one: asyncio.Event,
) -> None:
valid_m3u8 = re.compile(r"^(?!.*(amazonaws|knitcdn)).*\.m3u8")
if valid_m3u8.search(req.url):
captured.append(req.url)
got_one.set()
async def new_browser(
playwright: Playwright,
browser: str = "firefox",
ignore_https_errors: bool = False,
) -> tuple[Browser, BrowserContext]:
if browser == "brave":
brwsr = await playwright.chromium.connect_over_cdp("http://localhost:9222")
context = brwsr.contexts[0]
else:
brwsr = await playwright.firefox.launch(headless=True)
context = await brwsr.new_context(
user_agent=UA,
ignore_https_errors=ignore_https_errors,
viewport={"width": 1366, "height": 768},
device_scale_factor=1,
locale="en-US",
timezone_id="America/New_York",
color_scheme="dark",
permissions=["geolocation"],
extra_http_headers={
"Accept-Language": "en-US,en;q=0.9",
"Upgrade-Insecure-Requests": "1",
},
)
await context.add_init_script(
"""
Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
Object.defineProperty(navigator, 'languages', {
get: () => ['en-US', 'en']
});
Object.defineProperty(navigator, 'plugins', {
get: () => [1, 2, 3, 4]
});
const elementDescriptor = Object.getOwnPropertyDescriptor(HTMLElement.prototype, 'offsetHeight');
Object.defineProperty(HTMLDivElement.prototype, 'offsetHeight', {
...elementDescriptor,
get: function() {
if (this.id === 'modernizr') { return 24; }
return elementDescriptor.get.apply(this);
}
});
Object.defineProperty(window.screen, 'width', { get: () => 1366 });
Object.defineProperty(window.screen, 'height', { get: () => 768 });
const getParameter = WebGLRenderingContext.prototype. getParameter;
WebGLRenderingContext.prototype.getParameter = function (param) {
if (param === 37445) return "Intel Inc."; // UNMASKED_VENDOR_WEBGL
if (param === 37446) return "Intel Iris OpenGL Engine"; // UNMASKED_RENDERER_WEBGL
return getParameter.apply(this, [param]);
};
const observer = new MutationObserver(mutations => {
mutations.forEach(mutation => {
mutation.addedNodes.forEach(node => {
if (node.tagName === 'IFRAME' && node.hasAttribute('sandbox')) {
node.removeAttribute('sandbox');
}
});
});
});
observer.observe(document.documentElement, { childList: true, subtree: true });
"""
)
return brwsr, context

174
M3U8/scrapers/utils/webwork.py Normal file
View file

@ -0,0 +1,174 @@
import asyncio
import logging
import re
from collections.abc import Callable
from typing import Any
import httpx
from playwright.async_api import Browser, BrowserContext, Playwright, Request
from .logger import get_logger
class Network:
UA = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0"
)
def __init__(self) -> None:
self.client = httpx.AsyncClient(
timeout=5,
follow_redirects=True,
headers={"User-Agent": Network.UA},
)
self._logger = get_logger("network")
async def check_status(self, url: str) -> bool:
try:
r = await self.client.get(url)
r.raise_for_status()
return r.status_code == 200
except (httpx.HTTPError, httpx.TimeoutException) as e:
self._logger.debug(f"Status check failed for {url}: {e}")
return False
async def get_base(self, mirrors: list[str]) -> str | None:
tasks = [self.check_status(link) for link in mirrors]
results = await asyncio.gather(*tasks, return_exceptions=True)
working_mirrors = [
mirror for mirror, success in zip(mirrors, results) if success
]
return working_mirrors[0] if working_mirrors else None
@staticmethod
async def safe_process(
fn: Callable,
url_num: int,
timeout: int | float = 15,
log: logging.Logger | None = None,
) -> Any | None:
if not log:
log = logging.getLogger(__name__)
task = asyncio.create_task(fn())
try:
return await asyncio.wait_for(task, timeout=timeout)
except asyncio.TimeoutError:
log.warning(f"URL {url_num}) Timed out after {timeout}s, skipping event")
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
except Exception as e:
log.debug(f"URL {url_num}) Ignore exception after timeout: {e}")
return None
except Exception as e:
log.error(f"URL {url_num}) Unexpected error: {e}")
return None
@staticmethod
def capture_req(
req: Request,
captured: list[str],
got_one: asyncio.Event,
patterns: list[str] | None = None,
) -> None:
if not patterns:
patterns = ["amazonaws", "knitcdn"]
pattern = re.compile(rf"^.*\.m3u8(?!.*({'|'.join(patterns)}))")
if pattern.search(req.url):
captured.append(req.url)
got_one.set()
@staticmethod
async def browser(
playwright: Playwright,
browser: str = "firefox",
ignore_https_errors: bool = False,
) -> tuple[Browser, BrowserContext]:
if browser == "brave":
brwsr = await playwright.chromium.connect_over_cdp("http://localhost:9222")
context = brwsr.contexts[0]
else:
brwsr = await playwright.firefox.launch(headless=True)
context = await brwsr.new_context(
user_agent=Network.UA,
ignore_https_errors=ignore_https_errors,
viewport={"width": 1366, "height": 768},
device_scale_factor=1,
locale="en-US",
timezone_id="America/New_York",
color_scheme="dark",
permissions=["geolocation"],
extra_http_headers={
"Accept-Language": "en-US,en;q=0.9",
"Upgrade-Insecure-Requests": "1",
},
)
await context.add_init_script(
"""
Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
Object.defineProperty(navigator, 'languages', {
get: () => ['en-US', 'en']
});
Object.defineProperty(navigator, 'plugins', {
get: () => [1, 2, 3, 4]
});
const elementDescriptor = Object.getOwnPropertyDescriptor(HTMLElement.prototype, 'offsetHeight');
Object.defineProperty(HTMLDivElement.prototype, 'offsetHeight', {
...elementDescriptor,
get: function() {
if (this.id === 'modernizr') { return 24; }
return elementDescriptor.get.apply(this);
}
});
Object.defineProperty(window.screen, 'width', { get: () => 1366 });
Object.defineProperty(window.screen, 'height', { get: () => 768 });
const getParameter = WebGLRenderingContext.prototype. getParameter;
WebGLRenderingContext.prototype.getParameter = function (param) {
if (param === 37445) return "Intel Inc."; // UNMASKED_VENDOR_WEBGL
if (param === 37446) return "Intel Iris OpenGL Engine"; // UNMASKED_RENDERER_WEBGL
return getParameter.apply(this, [param]);
};
const observer = new MutationObserver(mutations => {
mutations.forEach(mutation => {
mutation.addedNodes.forEach(node => {
if (node.tagName === 'IFRAME' && node.hasAttribute('sandbox')) {
node.removeAttribute('sandbox');
}
});
});
});
observer.observe(document.documentElement, { childList: true, subtree: true });
"""
)
return brwsr, context
network = Network()
__all__ = ["network"]