e
This commit is contained in:
parent
b110aee1e8
commit
00000d9ba6
11 changed files with 121 additions and 135 deletions
|
|
@ -28,7 +28,7 @@ def validate_category(s: str) -> str:
|
|||
elif s == "fight":
|
||||
return "Fight (UFC/Boxing)"
|
||||
|
||||
return s.capitalize()
|
||||
return s.capitalize() if len(s) > 4 else s.upper()
|
||||
|
||||
|
||||
async def refresh_api_cache(
|
||||
|
|
@ -40,12 +40,12 @@ async def refresh_api_cache(
|
|||
r = await client.get(url)
|
||||
r.raise_for_status()
|
||||
except Exception as e:
|
||||
log.error(f'Failed to fetch "{url}"\n{e}')
|
||||
log.error(f'Failed to fetch "{url}": {e}')
|
||||
return {}
|
||||
|
||||
data = r.json()
|
||||
|
||||
data[0]["timestamp"] = Time.now().timestamp()
|
||||
data[-1]["timestamp"] = Time.now().timestamp()
|
||||
|
||||
return data
|
||||
|
||||
|
|
@ -113,7 +113,7 @@ async def get_events(
|
|||
cached_keys: set[str],
|
||||
) -> list[dict[str, str]]:
|
||||
|
||||
if not (api_data := API_FILE.load(per_entry=False, index=True)):
|
||||
if not (api_data := API_FILE.load(per_entry=False, index=-1)):
|
||||
api_data = await refresh_api_cache(
|
||||
client,
|
||||
urljoin(
|
||||
|
|
@ -211,15 +211,9 @@ async def scrape(client: httpx.AsyncClient) -> None:
|
|||
browser, context = await network.browser(p, "brave")
|
||||
|
||||
for i, ev in enumerate(events, start=1):
|
||||
url = await network.safe_process(
|
||||
lambda: process_event(
|
||||
ev["link"],
|
||||
url_num=i,
|
||||
context=context,
|
||||
),
|
||||
url_num=i,
|
||||
log=log,
|
||||
)
|
||||
handler = partial(process_event, url=ev["link"], url_num=i, context=context)
|
||||
|
||||
url = await network.safe_process(handler, url_num=i, log=log)
|
||||
|
||||
if url:
|
||||
sport, event, logo, ts = (
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue