This commit is contained in:
doms9 2025-12-23 13:28:56 -05:00
parent 4210d7dd18
commit 00000d9cb2
11 changed files with 342 additions and 321 deletions

View file

@ -64,12 +64,13 @@ async def main() -> None:
asyncio.create_task(streamsgate.scrape()), asyncio.create_task(streamsgate.scrape()),
asyncio.create_task(strmd.scrape()), asyncio.create_task(strmd.scrape()),
asyncio.create_task(tvpass.scrape()), asyncio.create_task(tvpass.scrape()),
# asyncio.create_task(watchfooty.scrape()),
asyncio.create_task(webcast.scrape()), asyncio.create_task(webcast.scrape()),
] ]
await asyncio.gather(*tasks) await asyncio.gather(*tasks)
await watchfooty.scrape()
additions = ( additions = (
embedhd.urls embedhd.urls
| fawa.urls | fawa.urls

View file

@ -89,6 +89,7 @@ async def scrape() -> None:
async with async_playwright() as p: async with async_playwright() as p:
browser, context = await network.browser(p) browser, context = await network.browser(p)
try:
for i, ev in enumerate(events, start=1): for i, ev in enumerate(events, start=1):
handler = partial( handler = partial(
network.process_event, network.process_event,
@ -128,6 +129,7 @@ async def scrape() -> None:
urls[key] = cached_urls[key] = entry urls[key] = cached_urls[key] = entry
finally:
await browser.close() await browser.close()
if new_count := len(cached_urls) - cached_count: if new_count := len(cached_urls) - cached_count:

View file

@ -88,6 +88,7 @@ async def scrape() -> None:
async with async_playwright() as p: async with async_playwright() as p:
browser, context = await network.browser(p) browser, context = await network.browser(p)
try:
handler = partial(get_events, context=context) handler = partial(get_events, context=context)
events = await network.safe_process( events = await network.safe_process(
@ -97,6 +98,7 @@ async def scrape() -> None:
log=log, log=log,
) )
finally:
await browser.close() await browser.close()
urls.update(events or {}) urls.update(events or {})

View file

@ -110,6 +110,7 @@ async def scrape() -> None:
async with async_playwright() as p: async with async_playwright() as p:
browser, context = await network.browser(p, browser="external") browser, context = await network.browser(p, browser="external")
try:
for i, ev in enumerate(events, start=1): for i, ev in enumerate(events, start=1):
handler = partial( handler = partial(
network.process_event, network.process_event,
@ -151,6 +152,7 @@ async def scrape() -> None:
urls[key] = cached_urls[key] = entry urls[key] = cached_urls[key] = entry
finally:
await browser.close() await browser.close()
if new_count := len(cached_urls) - cached_count: if new_count := len(cached_urls) - cached_count:

View file

@ -109,6 +109,7 @@ async def scrape() -> None:
async with async_playwright() as p: async with async_playwright() as p:
browser, context = await network.browser(p, browser="external") browser, context = await network.browser(p, browser="external")
try:
for i, ev in enumerate(events, start=1): for i, ev in enumerate(events, start=1):
handler = partial( handler = partial(
network.process_event, network.process_event,
@ -147,6 +148,7 @@ async def scrape() -> None:
urls[key] = cached_urls[key] = entry urls[key] = cached_urls[key] = entry
finally:
await browser.close() await browser.close()
if new_count := len(cached_urls) - cached_count: if new_count := len(cached_urls) - cached_count:

View file

@ -107,6 +107,7 @@ async def scrape() -> None:
async with async_playwright() as p: async with async_playwright() as p:
browser, context = await network.browser(p, browser="external") browser, context = await network.browser(p, browser="external")
try:
for i, ev in enumerate(events, start=1): for i, ev in enumerate(events, start=1):
handler = partial( handler = partial(
network.process_event, network.process_event,
@ -146,6 +147,7 @@ async def scrape() -> None:
urls[key] = cached_urls[key] = entry urls[key] = cached_urls[key] = entry
finally:
await browser.close() await browser.close()
if new_count := len(cached_urls) - cached_count: if new_count := len(cached_urls) - cached_count:

View file

@ -151,6 +151,7 @@ async def scrape() -> None:
async with async_playwright() as p: async with async_playwright() as p:
browser, context = await network.browser(p, browser="external") browser, context = await network.browser(p, browser="external")
try:
for i, ev in enumerate(events, start=1): for i, ev in enumerate(events, start=1):
handler = partial( handler = partial(
network.process_event, network.process_event,
@ -192,6 +193,7 @@ async def scrape() -> None:
urls[key] = cached_urls[key] = entry urls[key] = cached_urls[key] = entry
finally:
await browser.close() await browser.close()
if new_count := len(cached_urls) - cached_count: if new_count := len(cached_urls) - cached_count:

View file

@ -139,6 +139,7 @@ async def scrape() -> None:
async with async_playwright() as p: async with async_playwright() as p:
browser, context = await network.browser(p, browser="external") browser, context = await network.browser(p, browser="external")
try:
for i, ev in enumerate(events, start=1): for i, ev in enumerate(events, start=1):
handler = partial( handler = partial(
network.process_event, network.process_event,
@ -178,6 +179,7 @@ async def scrape() -> None:
urls[key] = cached_urls[key] = entry urls[key] = cached_urls[key] = entry
finally:
await browser.close() await browser.close()
if new_count := len(cached_urls) - cached_count: if new_count := len(cached_urls) - cached_count:

View file

@ -139,6 +139,7 @@ async def scrape() -> None:
async with async_playwright() as p: async with async_playwright() as p:
browser, context = await network.browser(p, browser="external") browser, context = await network.browser(p, browser="external")
try:
for i, ev in enumerate(events, start=1): for i, ev in enumerate(events, start=1):
handler = partial( handler = partial(
network.process_event, network.process_event,
@ -179,6 +180,7 @@ async def scrape() -> None:
urls[key] = cached_urls[key] = entry urls[key] = cached_urls[key] = entry
finally:
await browser.close() await browser.close()
if new_count := len(cached_urls) - cached_count: if new_count := len(cached_urls) - cached_count:

View file

@ -261,8 +261,9 @@ async def scrape() -> None:
if events: if events:
async with async_playwright() as p: async with async_playwright() as p:
browser, context = await network.browser(p) browser, context = await network.browser(p, browser="external")
try:
for i, ev in enumerate(events, start=1): for i, ev in enumerate(events, start=1):
handler = partial( handler = partial(
process_event, process_event,
@ -305,6 +306,7 @@ async def scrape() -> None:
valid_count += 1 valid_count += 1
urls[key] = entry urls[key] = entry
finally:
await browser.close() await browser.close()
if new_count := valid_count - cached_count: if new_count := valid_count - cached_count:

View file

@ -129,6 +129,7 @@ async def scrape() -> None:
async with async_playwright() as p: async with async_playwright() as p:
browser, context = await network.browser(p) browser, context = await network.browser(p)
try:
for i, ev in enumerate(events, start=1): for i, ev in enumerate(events, start=1):
handler = partial( handler = partial(
network.process_event, network.process_event,
@ -168,6 +169,7 @@ async def scrape() -> None:
urls[key] = cached_urls[key] = entry urls[key] = cached_urls[key] = entry
finally:
await browser.close() await browser.close()
if new_count := len(cached_urls) - cached_count: if new_count := len(cached_urls) - cached_count: