diff --git a/M3U8/scrapers/volokit.py b/M3U8/scrapers/volokit.py index 8637d184..11cbfe8e 100644 --- a/M3U8/scrapers/volokit.py +++ b/M3U8/scrapers/volokit.py @@ -13,9 +13,7 @@ urls: dict[str, dict[str, str | float]] = {} TAG = "VOLOKIT" -CACHE_FILE = Cache(TAG, exp=10_800) - -HTML_CACHE = Cache(f"{TAG}-html", exp=19_800) +CACHE_FILE = Cache(TAG, exp=19_800) BASE_URL = "http://volokit.xyz" @@ -71,98 +69,48 @@ async def process_event(url: str, url_num: int) -> str | None: return match[1] -async def refresh_html_cache( - url: str, - sport: str, - now: Time, -) -> dict[str, dict[str, str | float]]: +async def get_events(cached_keys: list[str]) -> list[dict[str, str]]: + sport_urls = { + sport.upper(): urljoin(BASE_URL, f"sport/{sport}/") for sport in SPORT_ENDPOINTS + } - events = {} + tasks = [network.request(url, log=log) for url in sport_urls.values()] - if not (html_data := await network.request(url, log=log)): + results = await asyncio.gather(*tasks) + + events = [] + + if not ( + soups := [(HTMLParser(html.content), html.url) for html in results if html] + ): return events - soup = HTMLParser(html_data.content) + for soup, url in soups: + for card in soup.css("#events .table .vevent.theevent"): + if not (href := card.css_first("a").attributes.get("href")): + continue - date = now.date() + if not (name_node := card.css_first(".teamtd.event")): + continue - if date_node := soup.css_first("tr.date"): - date = date_node.text(strip=True) + name = fix_event(name_node.text(strip=True)) - for card in soup.css("#events .table .vevent.theevent"): - if not (href := card.css_first("a").attributes.get("href")): - continue + sport = next((k for k, v in sport_urls.items() if v == url), "Live Event") - name_node = card.css_first(".teamtd.event") + if f"[{sport}] {name} ({TAG})" in cached_keys: + continue - time_node = card.css_first(".time") - - if not (name_node and time_node): - continue - - name, time = name_node.text(strip=True), time_node.text(strip=True) - - event_sport = SPORT_ENDPOINTS[sport] - - event_name = fix_event(name) - - event_dt = Time.from_str(f"{date} {time}", timezone="UTC") - - key = f"[{event_sport}] {event_name} ({TAG})" - - events[key] = { - "sport": event_sport, - "event": event_name, - "link": href, - "event_ts": event_dt.timestamp(), - "timestamp": now.timestamp(), - } + events.append( + { + "sport": sport, + "event": name, + "link": href, + } + ) return events -async def get_events(cached_keys: list[str]) -> list[dict[str, str]]: - now = Time.clean(Time.now()) - - if not (events := HTML_CACHE.load()): - log.info("Refreshing HTML cache") - - sport_urls = { - sport: urljoin(BASE_URL, f"sport/{sport}") for sport in SPORT_ENDPOINTS - } - - tasks = [ - refresh_html_cache( - url, - sport, - now, - ) - for sport, url in sport_urls.items() - ] - - results = await asyncio.gather(*tasks) - - events = {k: v for data in results for k, v in data.items()} - - HTML_CACHE.write(events) - - live = [] - - start_ts = now.delta(minutes=-30).timestamp() - end_ts = now.delta(minutes=30).timestamp() - - for k, v in events.items(): - if k in cached_keys: - continue - - if not start_ts <= v["event_ts"] <= end_ts: - continue - - live.append(v) - - return live - - async def scrape() -> None: cached_urls = CACHE_FILE.load() @@ -181,6 +129,8 @@ async def scrape() -> None: if events: log.info(f"Processing {len(events)} new URL(s)") + now = Time.clean(Time.now()) + for i, ev in enumerate(events, start=1): handler = partial( process_event, @@ -195,11 +145,7 @@ async def scrape() -> None: log=log, ) - sport, event, ts = ( - ev["sport"], - ev["event"], - ev["event_ts"], - ) + sport, event = ev["sport"], ev["event"] key = f"[{sport}] {event} ({TAG})" @@ -209,7 +155,7 @@ async def scrape() -> None: "url": url, "logo": logo, "base": link, - "timestamp": ts, + "timestamp": now.timestamp(), "id": tvg_id or "Live.Event.us", "link": link, }