diff --git a/M3U8/scrapers/livetvsx.py b/M3U8/scrapers/livetvsx.py index 4cd1424..133c073 100644 --- a/M3U8/scrapers/livetvsx.py +++ b/M3U8/scrapers/livetvsx.py @@ -10,12 +10,12 @@ log = get_logger(__name__) urls: dict[str, dict[str, str | float]] = {} -TAG = "LTVSX" - -XML_CACHE = Cache(f"{TAG}-xml", exp=28_000) +TAG = "LIVETVSX" CACHE_FILE = Cache(TAG, exp=10_800) +XML_CACHE = Cache(f"{TAG}-xml", exp=28_000) + BASE_URL = "https://cdn.livetv861.me/rss/upcoming_en.xml" VALID_SPORTS = {"NBA", "NHL", "NFL", "NCAA", "MLB"} @@ -193,7 +193,7 @@ async def scrape(browser: Browser) -> None: log.info(f"Processing {len(events)} new URL(s)") if events: - async with network.event_context(browser) as context: + async with network.event_context(browser, ignore_https=True) as context: for i, ev in enumerate(events, start=1): async with network.event_page(context) as page: handler = partial( diff --git a/M3U8/scrapers/utils/webwork.py b/M3U8/scrapers/utils/webwork.py index 19d3fdc..7bce5f8 100644 --- a/M3U8/scrapers/utils/webwork.py +++ b/M3U8/scrapers/utils/webwork.py @@ -129,12 +129,14 @@ class Network: async def event_context( browser: Browser, stealth: bool = True, + ignore_https: bool = False, ) -> AsyncGenerator[BrowserContext, None]: context: BrowserContext | None = None try: context = await browser.new_context( user_agent=Network.UA if stealth else None, + ignore_https_errors=ignore_https, viewport={"width": 1366, "height": 768}, device_scale_factor=1, locale="en-US", diff --git a/health.sh b/health.sh index f02c884..b781e26 100644 --- a/health.sh +++ b/health.sh @@ -9,21 +9,23 @@ STATUSLOG=$(mktemp) get_status() { local url="$1" local channel="$2" + local index="$3" + local total="$4" local attempt response status_code [[ "$url" != http* ]] && return - for attempt in $(seq 1 "$RETRY_COUNT"); do - echo "Checking '$url'" + printf '[%d/%d] Checking %s\n' "$((index + 1))" "$total" "$url" + for attempt in $(seq 1 "$RETRY_COUNT"); do response=$( curl -skL \ -A "$UA" \ -H "Accept: */*" \ -H "Accept-Language: en-US,en;q=0.9" \ - -H "Accept-Encoding: gzip, deflate, br" \ -H "Connection: keep-alive" \ -o /dev/null \ + --compressed \ --max-time 30 \ -w "%{http_code}" \ "$url" 2>&1 @@ -49,7 +51,7 @@ get_status() { status_code="$response" case "$status_code" in - 200) + 2* | 3*) echo "PASS" >>"$STATUSLOG" ;; @@ -73,6 +75,7 @@ get_status() { check_links() { echo "Checking links from: $base_file" + total_urls=$(grep -cE '^https?://' "$base_file") channel_num=0 name="" @@ -88,14 +91,14 @@ check_links() { elif [[ "$line" =~ ^https?:// ]]; then while (($(jobs -r | wc -l) >= MAX_JOBS)); do sleep 0.2; done - get_status "$line" "$name" & + get_status "$line" "$name" "$channel_num" "$total_urls" & ((channel_num++)) fi done < <(cat "$base_file") wait - echo "Done." + echo -e "\nDone." } write_readme() {