This commit is contained in:
doms9 2025-09-03 00:39:49 -04:00
parent 00000d9d54
commit 00000d932b
6 changed files with 34 additions and 3639 deletions

View file

@ -10,12 +10,12 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v5
with: with:
fetch-depth: 0 fetch-depth: 0
- name: Cache venv - name: Cache venv
uses: actions/cache@v3 uses: actions/cache@v4
with: with:
path: .venv path: .venv
key: shared-venv-${{ runner.os }}-${{ hashFiles('uv.lock') }} key: shared-venv-${{ runner.os }}-${{ hashFiles('uv.lock') }}

View file

@ -10,7 +10,7 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v5
with: with:
fetch-depth: 0 fetch-depth: 0

View file

@ -18,19 +18,27 @@ jobs:
- name: Checkout - name: Checkout
if: steps.check_time.outputs.run == 'true' if: steps.check_time.outputs.run == 'true'
uses: actions/checkout@v4 uses: actions/checkout@v5
with: with:
fetch-depth: 0 fetch-depth: 0
- name: Cache venv - name: Cache venv
if: steps.check_time.outputs.run == 'true' if: steps.check_time.outputs.run == 'true'
uses: actions/cache@v3 uses: actions/cache@v4
with: with:
path: .venv path: .venv
key: shared-venv-${{ runner.os }}-${{ hashFiles('uv.lock') }} key: shared-venv-${{ runner.os }}-${{ hashFiles('uv.lock') }}
restore-keys: | restore-keys: |
shared-venv-${{ runner.os }}- shared-venv-${{ runner.os }}-
- name: Cache cert
uses: actions/cache@v4
with:
path: M3U8/scrape/cached-ca.pem
key: cert-cache-${{ runner.os }}-${{ hashFiles('M3U8/scrape/cached-ca.pem') }}
restore-keys: |
cert-cache-${{ runner.os }}-
- name: Install uv - name: Install uv
if: steps.check_time.outputs.run == 'true' if: steps.check_time.outputs.run == 'true'
uses: astral-sh/setup-uv@v6 uses: astral-sh/setup-uv@v6
@ -47,8 +55,9 @@ jobs:
python-version-file: "pyproject.toml" python-version-file: "pyproject.toml"
- name: Cache Playwright browsers - name: Cache Playwright browsers
id: cache-pw
if: steps.check_time.outputs.run == 'true' if: steps.check_time.outputs.run == 'true'
uses: actions/cache@v3 uses: actions/cache@v4
with: with:
path: ~/.cache/ms-playwright path: ~/.cache/ms-playwright
key: ${{ runner.os }}-playwright key: ${{ runner.os }}-playwright
@ -56,7 +65,7 @@ jobs:
${{ runner.os }}-playwright ${{ runner.os }}-playwright
- name: Install Playwright browsers - name: Install Playwright browsers
if: steps.check_time.outputs.run == 'true' if: steps.check_time.outputs.run == 'true' && steps.cache-pw.outputs.cache-hit != 'true'
run: | run: |
uv run playwright install uv run playwright install
uv run playwright install-deps uv run playwright install-deps

1
.gitignore vendored
View file

@ -12,3 +12,4 @@ wheels/
# Misc # Misc
.python-version .python-version
stuff/ stuff/
cached-ca.pem

File diff suppressed because it is too large Load diff

View file

@ -35,11 +35,21 @@ CACHE_FILE = Path(__file__).parent / "livetvsx.json"
async def safe_process_event(fn, url_num: int, timeout=20) -> Any | None: async def safe_process_event(fn, url_num: int, timeout=20) -> Any | None:
task = asyncio.create_task(fn())
try: try:
return await asyncio.wait_for(fn(), timeout=timeout) return await asyncio.wait_for(task, timeout=timeout)
except asyncio.TimeoutError: except asyncio.TimeoutError:
log.warning(f"URL {url_num}) Timed out after {timeout}s, skipping event") log.warning(f"URL {url_num}) Timed out after {timeout}s, skipping event")
return
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
except Exception as e:
log.debug(f"URL {url_num}) Ignore exception after timeout: {e}")
async def write_to_cert(client: httpx.AsyncClient, url: str, cert: Path) -> None: async def write_to_cert(client: httpx.AsyncClient, url: str, cert: Path) -> None:
@ -79,7 +89,7 @@ def load_cache() -> dict[str, dict[str, str | str]]:
try: try:
data = json.loads(CACHE_FILE.read_text(encoding="utf-8")) data = json.loads(CACHE_FILE.read_text(encoding="utf-8"))
now = datetime.now().timestamp() now = datetime.now(TZ).timestamp()
return { return {
k: v k: v
@ -169,7 +179,7 @@ async def parse_feed(
return events return events
async def process_event(url: str, url_num: int, max_wait_ms=15_000) -> str | None: async def process_event(url: str, url_num: int) -> str | None:
async with async_playwright() as p: async with async_playwright() as p:
browser = await p.firefox.launch(headless=True) browser = await p.firefox.launch(headless=True)
@ -199,7 +209,7 @@ async def process_event(url: str, url_num: int, max_wait_ms=15_000) -> str | Non
await ev_page.goto( await ev_page.goto(
url, url,
wait_until="domcontentloaded", wait_until="domcontentloaded",
timeout=30_000, timeout=10_000,
) )
btn = await ev_page.query_selector(".lnkhdr > tbody > tr > td:nth-child(2)") btn = await ev_page.query_selector(".lnkhdr > tbody > tr > td:nth-child(2)")
@ -248,7 +258,7 @@ async def process_event(url: str, url_num: int, max_wait_ms=15_000) -> str | Non
wait_task = asyncio.create_task(got_one.wait()) wait_task = asyncio.create_task(got_one.wait())
try: try:
await asyncio.wait_for(wait_task, timeout=max_wait_ms / 1000) await asyncio.wait_for(wait_task, timeout=1.5e1)
except asyncio.TimeoutError: except asyncio.TimeoutError:
log.warning(f"URL {url_num}) Timed out waiting for m3u8.") log.warning(f"URL {url_num}) Timed out waiting for m3u8.")
@ -308,7 +318,7 @@ async def main(client: httpx.AsyncClient) -> None:
log.info(f"Processing {len(events)} URLs") log.info(f"Processing {len(events)} URLs")
now_ts = datetime.now().timestamp() now_ts = datetime.now(TZ).timestamp()
for num, ev in enumerate(events, start=1): for num, ev in enumerate(events, start=1):
sport = ev["sport"] sport = ev["sport"]