e

M3U8/scrapers/utils/webwork.py

@@ -0,0 +1,174 @@
+import asyncio
+import logging
+import re
+from collections.abc import Callable
+from typing import Any
+
+import httpx
+from playwright.async_api import Browser, BrowserContext, Playwright, Request
+
+from .logger import get_logger
+
+
+class Network:
+    UA = (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0"
+    )
+
+    def __init__(self) -> None:
+        self.client = httpx.AsyncClient(
+            timeout=5,
+            follow_redirects=True,
+            headers={"User-Agent": Network.UA},
+        )
+
+        self._logger = get_logger("network")
+
+    async def check_status(self, url: str) -> bool:
+        try:
+            r = await self.client.get(url)
+            r.raise_for_status()
+            return r.status_code == 200
+        except (httpx.HTTPError, httpx.TimeoutException) as e:
+            self._logger.debug(f"Status check failed for {url}: {e}")
+            return False
+
+    async def get_base(self, mirrors: list[str]) -> str | None:
+        tasks = [self.check_status(link) for link in mirrors]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        working_mirrors = [
+            mirror for mirror, success in zip(mirrors, results) if success
+        ]
+
+        return working_mirrors[0] if working_mirrors else None
+
+    @staticmethod
+    async def safe_process(
+        fn: Callable,
+        url_num: int,
+        timeout: int | float = 15,
+        log: logging.Logger | None = None,
+    ) -> Any | None:
+
+        if not log:
+            log = logging.getLogger(__name__)
+
+        task = asyncio.create_task(fn())
+
+        try:
+            return await asyncio.wait_for(task, timeout=timeout)
+        except asyncio.TimeoutError:
+            log.warning(f"URL {url_num}) Timed out after {timeout}s, skipping event")
+
+            task.cancel()
+
+            try:
+                await task
+            except asyncio.CancelledError:
+                pass
+            except Exception as e:
+                log.debug(f"URL {url_num}) Ignore exception after timeout: {e}")
+
+            return None
+        except Exception as e:
+            log.error(f"URL {url_num}) Unexpected error: {e}")
+            return None
+
+    @staticmethod
+    def capture_req(
+        req: Request,
+        captured: list[str],
+        got_one: asyncio.Event,
+        patterns: list[str] | None = None,
+    ) -> None:
+        if not patterns:
+            patterns = ["amazonaws", "knitcdn"]
+
+        pattern = re.compile(rf"^.*\.m3u8(?!.*({'|'.join(patterns)}))")
+
+        if pattern.search(req.url):
+            captured.append(req.url)
+            got_one.set()
+
+    @staticmethod
+    async def browser(
+        playwright: Playwright,
+        browser: str = "firefox",
+        ignore_https_errors: bool = False,
+    ) -> tuple[Browser, BrowserContext]:
+
+        if browser == "brave":
+            brwsr = await playwright.chromium.connect_over_cdp("http://localhost:9222")
+            context = brwsr.contexts[0]
+        else:
+            brwsr = await playwright.firefox.launch(headless=True)
+
+            context = await brwsr.new_context(
+                user_agent=Network.UA,
+                ignore_https_errors=ignore_https_errors,
+                viewport={"width": 1366, "height": 768},
+                device_scale_factor=1,
+                locale="en-US",
+                timezone_id="America/New_York",
+                color_scheme="dark",
+                permissions=["geolocation"],
+                extra_http_headers={
+                    "Accept-Language": "en-US,en;q=0.9",
+                    "Upgrade-Insecure-Requests": "1",
+                },
+            )
+
+            await context.add_init_script(
+                """
+            Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
+
+            Object.defineProperty(navigator, 'languages', {
+            get: () => ['en-US', 'en']
+            });
+
+            Object.defineProperty(navigator, 'plugins', {
+            get: () => [1, 2, 3, 4]
+            });
+
+            const elementDescriptor = Object.getOwnPropertyDescriptor(HTMLElement.prototype, 'offsetHeight');
+            Object.defineProperty(HTMLDivElement.prototype, 'offsetHeight', {
+            ...elementDescriptor,
+            get: function() {
+                if (this.id === 'modernizr') { return 24; }
+                return elementDescriptor.get.apply(this);
+            }
+            });
+
+            Object.defineProperty(window.screen, 'width', { get: ()     => 1366 });
+            Object.defineProperty(window.screen, 'height', { get: ()    => 768 });
+
+            const getParameter = WebGLRenderingContext.prototype.   getParameter;
+            WebGLRenderingContext.prototype.getParameter = function (param) {
+            if (param === 37445) return "Intel Inc."; //  UNMASKED_VENDOR_WEBGL
+            if (param === 37446) return "Intel Iris OpenGL    Engine"; // UNMASKED_RENDERER_WEBGL
+            return getParameter.apply(this, [param]);
+            };
+
+            const observer = new MutationObserver(mutations => {
+                mutations.forEach(mutation => {
+                    mutation.addedNodes.forEach(node => {
+                        if (node.tagName === 'IFRAME' && node.hasAttribute('sandbox')) {
+                            node.removeAttribute('sandbox');
+                        }
+                    });
+                });
+            });
+
+            observer.observe(document.documentElement, { childList: true, subtree: true });
+            """
+            )
+
+        return brwsr, context
+
+
+network = Network()
+
+__all__ = ["network"]