e
This commit is contained in:
parent
00000d9d54
commit
00000d932b
6 changed files with 34 additions and 3639 deletions
4
.github/workflows/epg.yml
vendored
4
.github/workflows/epg.yml
vendored
|
|
@ -10,12 +10,12 @@ jobs:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v5
|
||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
|
|
||||||
- name: Cache venv
|
- name: Cache venv
|
||||||
uses: actions/cache@v3
|
uses: actions/cache@v4
|
||||||
with:
|
with:
|
||||||
path: .venv
|
path: .venv
|
||||||
key: shared-venv-${{ runner.os }}-${{ hashFiles('uv.lock') }}
|
key: shared-venv-${{ runner.os }}-${{ hashFiles('uv.lock') }}
|
||||||
|
|
|
||||||
2
.github/workflows/health.yml
vendored
2
.github/workflows/health.yml
vendored
|
|
@ -10,7 +10,7 @@ jobs:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v5
|
||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
|
|
||||||
|
|
|
||||||
17
.github/workflows/m3u8.yml
vendored
17
.github/workflows/m3u8.yml
vendored
|
|
@ -18,19 +18,27 @@ jobs:
|
||||||
|
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
if: steps.check_time.outputs.run == 'true'
|
if: steps.check_time.outputs.run == 'true'
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v5
|
||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
|
|
||||||
- name: Cache venv
|
- name: Cache venv
|
||||||
if: steps.check_time.outputs.run == 'true'
|
if: steps.check_time.outputs.run == 'true'
|
||||||
uses: actions/cache@v3
|
uses: actions/cache@v4
|
||||||
with:
|
with:
|
||||||
path: .venv
|
path: .venv
|
||||||
key: shared-venv-${{ runner.os }}-${{ hashFiles('uv.lock') }}
|
key: shared-venv-${{ runner.os }}-${{ hashFiles('uv.lock') }}
|
||||||
restore-keys: |
|
restore-keys: |
|
||||||
shared-venv-${{ runner.os }}-
|
shared-venv-${{ runner.os }}-
|
||||||
|
|
||||||
|
- name: Cache cert
|
||||||
|
uses: actions/cache@v4
|
||||||
|
with:
|
||||||
|
path: M3U8/scrape/cached-ca.pem
|
||||||
|
key: cert-cache-${{ runner.os }}-${{ hashFiles('M3U8/scrape/cached-ca.pem') }}
|
||||||
|
restore-keys: |
|
||||||
|
cert-cache-${{ runner.os }}-
|
||||||
|
|
||||||
- name: Install uv
|
- name: Install uv
|
||||||
if: steps.check_time.outputs.run == 'true'
|
if: steps.check_time.outputs.run == 'true'
|
||||||
uses: astral-sh/setup-uv@v6
|
uses: astral-sh/setup-uv@v6
|
||||||
|
|
@ -47,8 +55,9 @@ jobs:
|
||||||
python-version-file: "pyproject.toml"
|
python-version-file: "pyproject.toml"
|
||||||
|
|
||||||
- name: Cache Playwright browsers
|
- name: Cache Playwright browsers
|
||||||
|
id: cache-pw
|
||||||
if: steps.check_time.outputs.run == 'true'
|
if: steps.check_time.outputs.run == 'true'
|
||||||
uses: actions/cache@v3
|
uses: actions/cache@v4
|
||||||
with:
|
with:
|
||||||
path: ~/.cache/ms-playwright
|
path: ~/.cache/ms-playwright
|
||||||
key: ${{ runner.os }}-playwright
|
key: ${{ runner.os }}-playwright
|
||||||
|
|
@ -56,7 +65,7 @@ jobs:
|
||||||
${{ runner.os }}-playwright
|
${{ runner.os }}-playwright
|
||||||
|
|
||||||
- name: Install Playwright browsers
|
- name: Install Playwright browsers
|
||||||
if: steps.check_time.outputs.run == 'true'
|
if: steps.check_time.outputs.run == 'true' && steps.cache-pw.outputs.cache-hit != 'true'
|
||||||
run: |
|
run: |
|
||||||
uv run playwright install
|
uv run playwright install
|
||||||
uv run playwright install-deps
|
uv run playwright install-deps
|
||||||
|
|
|
||||||
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -12,3 +12,4 @@ wheels/
|
||||||
# Misc
|
# Misc
|
||||||
.python-version
|
.python-version
|
||||||
stuff/
|
stuff/
|
||||||
|
cached-ca.pem
|
||||||
File diff suppressed because it is too large
Load diff
|
|
@ -35,11 +35,21 @@ CACHE_FILE = Path(__file__).parent / "livetvsx.json"
|
||||||
|
|
||||||
|
|
||||||
async def safe_process_event(fn, url_num: int, timeout=20) -> Any | None:
|
async def safe_process_event(fn, url_num: int, timeout=20) -> Any | None:
|
||||||
|
task = asyncio.create_task(fn())
|
||||||
|
|
||||||
try:
|
try:
|
||||||
return await asyncio.wait_for(fn(), timeout=timeout)
|
return await asyncio.wait_for(task, timeout=timeout)
|
||||||
except asyncio.TimeoutError:
|
except asyncio.TimeoutError:
|
||||||
log.warning(f"URL {url_num}) Timed out after {timeout}s, skipping event")
|
log.warning(f"URL {url_num}) Timed out after {timeout}s, skipping event")
|
||||||
return
|
|
||||||
|
task.cancel()
|
||||||
|
|
||||||
|
try:
|
||||||
|
await task
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
pass
|
||||||
|
except Exception as e:
|
||||||
|
log.debug(f"URL {url_num}) Ignore exception after timeout: {e}")
|
||||||
|
|
||||||
|
|
||||||
async def write_to_cert(client: httpx.AsyncClient, url: str, cert: Path) -> None:
|
async def write_to_cert(client: httpx.AsyncClient, url: str, cert: Path) -> None:
|
||||||
|
|
@ -79,7 +89,7 @@ def load_cache() -> dict[str, dict[str, str | str]]:
|
||||||
try:
|
try:
|
||||||
data = json.loads(CACHE_FILE.read_text(encoding="utf-8"))
|
data = json.loads(CACHE_FILE.read_text(encoding="utf-8"))
|
||||||
|
|
||||||
now = datetime.now().timestamp()
|
now = datetime.now(TZ).timestamp()
|
||||||
|
|
||||||
return {
|
return {
|
||||||
k: v
|
k: v
|
||||||
|
|
@ -169,7 +179,7 @@ async def parse_feed(
|
||||||
return events
|
return events
|
||||||
|
|
||||||
|
|
||||||
async def process_event(url: str, url_num: int, max_wait_ms=15_000) -> str | None:
|
async def process_event(url: str, url_num: int) -> str | None:
|
||||||
async with async_playwright() as p:
|
async with async_playwright() as p:
|
||||||
browser = await p.firefox.launch(headless=True)
|
browser = await p.firefox.launch(headless=True)
|
||||||
|
|
||||||
|
|
@ -199,7 +209,7 @@ async def process_event(url: str, url_num: int, max_wait_ms=15_000) -> str | Non
|
||||||
await ev_page.goto(
|
await ev_page.goto(
|
||||||
url,
|
url,
|
||||||
wait_until="domcontentloaded",
|
wait_until="domcontentloaded",
|
||||||
timeout=30_000,
|
timeout=10_000,
|
||||||
)
|
)
|
||||||
|
|
||||||
btn = await ev_page.query_selector(".lnkhdr > tbody > tr > td:nth-child(2)")
|
btn = await ev_page.query_selector(".lnkhdr > tbody > tr > td:nth-child(2)")
|
||||||
|
|
@ -248,7 +258,7 @@ async def process_event(url: str, url_num: int, max_wait_ms=15_000) -> str | Non
|
||||||
wait_task = asyncio.create_task(got_one.wait())
|
wait_task = asyncio.create_task(got_one.wait())
|
||||||
|
|
||||||
try:
|
try:
|
||||||
await asyncio.wait_for(wait_task, timeout=max_wait_ms / 1000)
|
await asyncio.wait_for(wait_task, timeout=1.5e1)
|
||||||
|
|
||||||
except asyncio.TimeoutError:
|
except asyncio.TimeoutError:
|
||||||
log.warning(f"URL {url_num}) Timed out waiting for m3u8.")
|
log.warning(f"URL {url_num}) Timed out waiting for m3u8.")
|
||||||
|
|
@ -308,7 +318,7 @@ async def main(client: httpx.AsyncClient) -> None:
|
||||||
|
|
||||||
log.info(f"Processing {len(events)} URLs")
|
log.info(f"Processing {len(events)} URLs")
|
||||||
|
|
||||||
now_ts = datetime.now().timestamp()
|
now_ts = datetime.now(TZ).timestamp()
|
||||||
|
|
||||||
for num, ev in enumerate(events, start=1):
|
for num, ev in enumerate(events, start=1):
|
||||||
sport = ev["sport"]
|
sport = ev["sport"]
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue