e

fix livetvsx.py scraping
edit health check script

M3U8/scrapers/livetvsx.py

@@ -10,12 +10,12 @@ log = get_logger(__name__)
 
 urls: dict[str, dict[str, str | float]] = {}
 
-TAG = "LTVSX"
-
-XML_CACHE = Cache(f"{TAG}-xml", exp=28_000)
+TAG = "LIVETVSX"
 
 CACHE_FILE = Cache(TAG, exp=10_800)
 
+XML_CACHE = Cache(f"{TAG}-xml", exp=28_000)
+
 BASE_URL = "https://cdn.livetv861.me/rss/upcoming_en.xml"
 
 VALID_SPORTS = {"NBA", "NHL", "NFL", "NCAA", "MLB"}
@@ -193,7 +193,7 @@ async def scrape(browser: Browser) -> None:
     log.info(f"Processing {len(events)} new URL(s)")
 
     if events:
-        async with network.event_context(browser) as context:
+        async with network.event_context(browser, ignore_https=True) as context:
             for i, ev in enumerate(events, start=1):
                 async with network.event_page(context) as page:
                     handler = partial(

M3U8/scrapers/utils/webwork.py

@@ -129,12 +129,14 @@ class Network:
     async def event_context(
         browser: Browser,
         stealth: bool = True,
+        ignore_https: bool = False,
     ) -> AsyncGenerator[BrowserContext, None]:
         context: BrowserContext | None = None
 
         try:
             context = await browser.new_context(
                 user_agent=Network.UA if stealth else None,
+                ignore_https_errors=ignore_https,
                 viewport={"width": 1366, "height": 768},
                 device_scale_factor=1,
                 locale="en-US",

health.sh

@@ -9,21 +9,23 @@ STATUSLOG=$(mktemp)
 get_status() {
     local url="$1"
     local channel="$2"
+    local index="$3"
+    local total="$4"
     local attempt response status_code
 
     [[ "$url" != http* ]] && return
 
-    for attempt in $(seq 1 "$RETRY_COUNT"); do
-        echo "Checking '$url'"
+    printf '[%d/%d] Checking %s\n' "$((index + 1))" "$total" "$url"
 
+    for attempt in $(seq 1 "$RETRY_COUNT"); do
         response=$(
             curl -skL \
                 -A "$UA" \
                 -H "Accept: */*" \
                 -H "Accept-Language: en-US,en;q=0.9" \
-                -H "Accept-Encoding: gzip, deflate, br" \
                 -H "Connection: keep-alive" \
                 -o /dev/null \
+                --compressed \
                 --max-time 30 \
                 -w "%{http_code}" \
                 "$url" 2>&1
@@ -49,7 +51,7 @@ get_status() {
     status_code="$response"
 
     case "$status_code" in
-    200)
+    2* | 3*)
         echo "PASS" >>"$STATUSLOG"
         ;;
 
@@ -73,6 +75,7 @@ get_status() {
 
 check_links() {
     echo "Checking links from: $base_file"
+    total_urls=$(grep -cE '^https?://' "$base_file")
     channel_num=0
     name=""
 
@@ -88,14 +91,14 @@ check_links() {
 
         elif [[ "$line" =~ ^https?:// ]]; then
             while (($(jobs -r | wc -l) >= MAX_JOBS)); do sleep 0.2; done
-            get_status "$line" "$name" &
+            get_status "$line" "$name" "$channel_num" "$total_urls" &
             ((channel_num++))
         fi
 
     done < <(cat "$base_file")
 
     wait
-    echo "Done."
+    echo -e "\nDone."
 }
 
 write_readme() {