- edit scraping for roxie.py
This commit is contained in:
doms9 2026-03-18 20:31:23 -04:00
parent d102ff040a
commit 00000d9595

View file

@ -82,18 +82,6 @@ async def process_event(
page: Page, page: Page,
) -> str | None: ) -> str | None:
captured: list[str] = []
got_one = asyncio.Event()
handler = partial(
network.capture_req,
captured=captured,
got_one=got_one,
)
page.on("request", handler)
try: try:
resp = await page.goto( resp = await page.goto(
url, url,
@ -105,55 +93,26 @@ async def process_event(
log.warning( log.warning(
f"URL {url_num}) Status Code: {resp.status if resp else 'None'}" f"URL {url_num}) Status Code: {resp.status if resp else 'None'}"
) )
return return
try: try:
if btn := await page.wait_for_selector( await page.wait_for_function(
"button.streambutton:nth-of-type(1)", "() => typeof clapprPlayer !== 'undefined'",
timeout=5_000, timeout=6_000,
): )
await btn.click(force=True, click_count=2)
stream = await page.evaluate("() => clapprPlayer.options.source")
except TimeoutError: except TimeoutError:
pass log.info(f"URL {url_num}) Could not find Clappr source")
try:
if player := await page.wait_for_selector(".play-wrapper", timeout=5_000):
await player.click(force=True, click_count=3)
except TimeoutError:
pass
wait_task = asyncio.create_task(got_one.wait())
try:
await asyncio.wait_for(wait_task, timeout=6)
except asyncio.TimeoutError:
log.warning(f"URL {url_num}) Timed out waiting for M3U8.")
return return
finally:
if not wait_task.done():
wait_task.cancel()
try:
await wait_task
except asyncio.CancelledError:
pass
if captured:
log.info(f"URL {url_num}) Captured M3U8") log.info(f"URL {url_num}) Captured M3U8")
return captured[0] return stream
log.warning(f"URL {url_num}) No M3U8 captured after waiting.")
return
except Exception as e: except Exception as e:
log.warning(f"URL {url_num}) {e}") log.warning(f"URL {url_num}) {e}")
return return
finally:
page.remove_listener("request", handler)
async def get_events(cached_keys: list[str]) -> list[dict[str, str]]: async def get_events(cached_keys: list[str]) -> list[dict[str, str]]:
now = Time.clean(Time.now()) now = Time.clean(Time.now())