diff --git a/addon.xml b/addon.xml
index 862704b..b52663b 100644
--- a/addon.xml
+++ b/addon.xml
@@ -1,7 +1,7 @@
@@ -29,8 +29,10 @@
https://watch.cbc.ca/
https://github.com/micahg/plugin.video.cbc
- - Updates for new Auth API
- - Fixes for live events
+ - Add back missing live channels
+ - Better live event handling
+ - Better EPG error handling
+ - Remove dead code and fix match bug
diff --git a/default.py b/default.py
index 99a60f3..25bbf1f 100644
--- a/default.py
+++ b/default.py
@@ -12,7 +12,7 @@
import routing
from resources.lib.cbc import CBC
-from resources.lib.utils import log, getAuthorizationFile, get_cookie_file, get_iptv_channels_file
+from resources.lib.utils import log, getAuthorizationFile, get_iptv_channels_file, is_pending, iso8601_to_local
from resources.lib.livechannels import LiveChannels
from resources.lib.gemv2 import GemV2
from resources.lib.iptvmanager import IPTVManager
@@ -119,7 +119,6 @@ def logout():
"""Remove authorization stuff."""
log('Logging out...', True)
os.remove(getAuthorizationFile())
- os.remove(get_cookie_file())
@plugin.route('/iptv/channels')
@@ -174,6 +173,7 @@ def play_live_channel():
def live_channels_menu():
"""Populate the menu with live channels."""
xbmcplugin.setContent(plugin.handle, 'videos')
+ xbmcplugin.addSortMethod(plugin.handle, xbmcplugin.SORT_METHOD_LABEL)
chans = LiveChannels()
chan_list = chans.get_live_channels()
cbc = CBC()
@@ -184,7 +184,10 @@ def live_channels_menu():
item = xbmcgui.ListItem(labels['title'])
item.setArt({'thumb': image, 'poster': image})
item.setInfo(type="Video", infoLabels=labels)
- item.setProperty('IsPlayable', 'true')
+ air_date = channel.get('airDate')
+ local_dt = iso8601_to_local(air_date) if air_date else None
+ if local_dt is None or not is_pending(local_dt):
+ item.setProperty('IsPlayable', 'true')
item.addContextMenuItems([
(getString(30014), 'RunPlugin({})'.format(plugin.url_for(live_channels_add_all))),
(getString(30015), 'RunPlugin({})'.format(plugin.url_for(live_channels_add, callsign))),
@@ -222,16 +225,16 @@ def layout_menu(path):
for f in items:
n = GemV2.normalized_format_item(f)
p = GemV2.normalized_format_path(f, path)
- item = xbmcgui.ListItem(n['label'])
+ item = xbmcgui.ListItem(n['title'])
if 'art' in n:
item.setArt(n['art'])
item.setInfo(type="Video", infoLabels=n['info_labels'])
- if n['playable']:
- item.setProperty('IsPlayable', 'true')
+ if 'app_code' in n and n['app_code']:
+ item.setProperty('IsPlayable', 'true' if 'playable' in n and n['playable'] else 'false')
url = plugin.url_for(play_live_channel, id=p, app_code=n['app_code'])
else:
url = plugin.url_for(layout_menu, p)
- xbmcplugin.addDirectoryItem(handle, url, item, not n['playable'])
+ xbmcplugin.addDirectoryItem(handle, url, item, not 'app_code' in n)
xbmcplugin.endOfDirectory(handle)
diff --git a/resources/lib/cbc.py b/resources/lib/cbc.py
index 3538ba2..b01565e 100644
--- a/resources/lib/cbc.py
+++ b/resources/lib/cbc.py
@@ -6,11 +6,10 @@
import json
# import http.client as http_client
from urllib.parse import urlparse, parse_qs, quote
-from xml.dom.minidom import parseString
import requests
-from .utils import save_cookies, loadCookies, saveAuthorization, log
+from .utils import saveAuthorization, log, iso8601_to_local, is_pending
# http_client.HTTPConnection.debuglevel = 1
@@ -65,9 +64,6 @@ def __init__(self):
"""Initialize the CBC class."""
# Create requests session object
self.session = requests.Session()
- session_cookies = loadCookies()
- if session_cookies is not None:
- self.session.cookies = session_cookies
@staticmethod
def azure_authorize_authorize(sess: requests.Session):
@@ -336,8 +332,8 @@ def get_labels(item):
'studio': 'Canadian Broadcasting Corporation',
'country': 'Canada'
}
- if 'cbc$callSign' in item:
- labels['title'] = '{} {}'.format(item['cbc$callSign'], item['title'])
+ if 'streamTitle' in item:
+ labels['title'] = item['streamTitle'].encode('utf-8')
else:
labels['title'] = item['title'].encode('utf-8')
@@ -373,14 +369,15 @@ def get_labels(item):
if item['cbc$audioVideo'].lower() == 'video':
labels['mediatype'] = 'video'
+ if 'airDate' in item:
+ local_dt = iso8601_to_local(item['airDate'])
+ if local_dt is not None:
+ is_pending(local_dt, item=labels)
+
return labels
@staticmethod
def get_session():
"""Get a requests session object with CBC cookies."""
- sess = requests.Session()
- cookies = loadCookies()
- if cookies is not None:
- sess.cookies = cookies
- return sess
+ return requests.Session()
diff --git a/resources/lib/epg.py b/resources/lib/epg.py
index 89bbf81..61ec52f 100644
--- a/resources/lib/epg.py
+++ b/resources/lib/epg.py
@@ -12,6 +12,8 @@
# Y/M/D
GUIDE_URL_FMT = 'https://www.cbc.ca/programguide/daily/{}/cbc_television'
+USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36'
+REQUEST_TIMEOUT = (10, 20)
# This is a combination of actual callsigns (NN for newsnet) and guid values
# for over-the-top-only services. No doubt, someone will come back here some
@@ -63,8 +65,16 @@ def map_channel_ids(unblocked):
"""Map channel IDs to guide names."""
url = get_guide_url(datetime.now())
data = call_guide_url(url)
+ if data is None:
+ log('Unable to map channel IDs: no data returned from {}'.format(url), True)
+ return { sign: None for sign in SPECIAL_GUIDES.keys()}
+
soup = BeautifulSoup(data, features="html.parser")
select = soup.find('select', id="selectlocation-tv")
+ if select is None:
+ log('Unable to map channel IDs: missing location selector at {}'.format(url), True)
+ return { sign: None for sign in SPECIAL_GUIDES.keys()}
+
options = select.find_all('option')
channel_map = { sign: None for sign in SPECIAL_GUIDES.keys()}
for option in options:
@@ -87,9 +97,14 @@ def get_guide_url(dttm, callsign=None):
def call_guide_url(url, location=None):
"""Call the guide URL and return the response body."""
cookies = {}
+ headers = {'User-Agent': USER_AGENT}
if location is not None:
cookies['pgTvLocation'] = location
- resp = requests.get(url, cookies=cookies)
+ try:
+ resp = requests.get(url, cookies=cookies, headers=headers, timeout=REQUEST_TIMEOUT)
+ except requests.RequestException as ex:
+ log(f'HTTP request failed for {url}: {ex}', True)
+ return None
if resp.status_code != 200:
log('{} returns status of {}'.format(url, resp.status_code), True)
return None
@@ -101,9 +116,21 @@ def get_channel_data(dttm, channel, callsign):
epg_data = []
url = get_guide_url(dttm, callsign)
data = call_guide_url(url, channel)
+ if data is None:
+ return epg_data
+
soup = BeautifulSoup(data, features="html.parser")
- select = soup.find('table', id="sched-table").find('tbody')
+ table = soup.find('table', id="sched-table")
+ if table is None:
+ log('Missing schedule table at "{}"'.format(url), True)
+ return epg_data
+
+ select = table.find('tbody')
+ if select is None:
+ log('Missing schedule body at "{}"'.format(url), True)
+ return epg_data
+
progs = select.find_all('tr')
for prog in progs:
prog_js = {}
@@ -122,7 +149,9 @@ def get_channel_data(dttm, channel, callsign):
prog_js = {}
break
prog_js['title'] = title_cell.get_text()
- prog_js['description'] = cell.find('dd').get_text()
+ description_cell = cell.find('dd')
+ if description_cell is not None:
+ prog_js['description'] = description_cell.get_text()
# skip the header row
if len(prog_js.items()) == 0:
diff --git a/resources/lib/gemv2.py b/resources/lib/gemv2.py
index 80d10f3..e721302 100644
--- a/resources/lib/gemv2.py
+++ b/resources/lib/gemv2.py
@@ -1,11 +1,10 @@
"""Module for the V2 Gem API."""
import json
-from datetime import datetime
import requests
from resources.lib.cbc import CBC
-from resources.lib.utils import loadAuthorization, log
+from resources.lib.utils import loadAuthorization, log, iso8601_to_local, is_pending
# api CONFIG IS AT https://services.radio-canada.ca/ott/catalog/v1/gem/settings?device=web
@@ -18,15 +17,6 @@
class GemV2:
"""V2 Gem API class."""
- @staticmethod
- def iso8601_to_local(dttm):
- """Convert an ISO 8601 timestamp (UTC or offset-aware) to local time string."""
- try:
- local_dt = datetime.fromisoformat(dttm.replace('Z', '+00:00')).astimezone()
- return local_dt.strftime('%Y-%m-%d %H:%M:%S')
- except (ValueError, TypeError, AttributeError):
- return dttm
-
@staticmethod
def scrape_json(uri, headers=None, params=None):
if headers is None:
@@ -40,7 +30,7 @@ def scrape_json(uri, headers=None, params=None):
try:
jsObj = json.loads(resp.content)
- except:
+ except (json.JSONDecodeError, ValueError, TypeError):
log(f'Unable to parse JSON from {uri} (status {resp.status_code})', True)
return None
return jsObj
@@ -198,7 +188,7 @@ def normalized_format_item(item):
images = item['images'] if 'images' in item else None
title = item['label'] if 'label' in item else item['title']
retval = {
- 'label': title,
+ 'title': title,
'playable': 'idMedia' in item,
'info_labels': {
'tvshowtitle': title,
@@ -225,22 +215,23 @@ def normalized_format_item(item):
if 'credits' in meta:
retval['info_labels']['cast'] = meta['credits'][0]['peoples'].split(',')
if 'live' in meta and 'startDate' in meta['live']:
- dttm = GemV2.iso8601_to_local(meta['live']['startDate'])
- retval['label'] += f' [Live: {dttm}]'
+ local_dt = iso8601_to_local(meta['live']['startDate'])
+ if local_dt is not None and is_pending(local_dt, item=retval):
+ retval['playable'] = False
if 'idMedia' in item:
# logic in https://services.radio-canada.ca/ott/catalog/v1/gem/settings?device=web
if 'type' in item:
- match item['type'].lower():
- case 'media':
- retval['app_code'] = 'gem'
- case 'quickturn':
- retval['app_code'] = 'medianet'
- case 'liveevent' | 'replay':
- retval['app_code'] = 'medianetlive'
- case _:
- log(f'Unknown type {item["type"]} for item with idMedia {item["idMedia"]}, defaulting to app_code "gem"')
- retval['app_code'] = 'gem'
+ item_type = item['type'].lower()
+ if item_type == 'media':
+ retval['app_code'] = 'gem'
+ elif item_type == 'quickturn':
+ retval['app_code'] = 'medianet'
+ elif item_type == 'liveevent' or item_type == 'replay':
+ retval['app_code'] = 'medianetlive'
+ else:
+ log(f'Unknown type {item["type"]} for item with idMedia {item["idMedia"]}, defaulting to app_code "gem"')
+ retval['app_code'] = 'gem'
return retval
diff --git a/resources/lib/livechannels.py b/resources/lib/livechannels.py
index 68e2403..e79a760 100644
--- a/resources/lib/livechannels.py
+++ b/resources/lib/livechannels.py
@@ -1,16 +1,17 @@
"""Module for live channels."""
-from concurrent import futures
import json
+import re
from urllib.parse import urlencode
import requests
-from resources.lib.utils import save_cookies, loadCookies, log, get_iptv_channels_file
+from resources.lib.utils import log, get_iptv_channels_file
from resources.lib.cbc import CBC
from resources.lib.gemv2 import GemV2
-LIST_URL = 'https://services.radio-canada.ca/ott/catalog/v2/gem/home?device=web'
-LIST_ELEMENT = '2415871718'
+GEM_BASE_URL = 'https://gem.cbc.ca/'
+FALLBACK_BUILD_ID = '7ByKb_CElwT2xVJeTO43g'
+LIST_URL_TEMPLATE = 'https://gem.cbc.ca/_next/data/{}/live.json'
class LiveChannels:
"""Class for live channels."""
@@ -19,36 +20,97 @@ def __init__(self):
"""Initialize the live channels class."""
# Create requests session object
self.session = requests.Session()
- session_cookies = loadCookies()
- if session_cookies is not None:
- self.session.cookies = session_cookies
+
+ @staticmethod
+ def extract_build_id(html):
+ """Extract Next.js buildId from page HTML."""
+ script_start = ''
+ start_pos = html.find(script_start)
+ if start_pos >= 0:
+ start_pos += len(script_start)
+ end_pos = html.find(script_end, start_pos)
+ if end_pos > start_pos:
+ try:
+ next_data = json.loads(html[start_pos:end_pos])
+ build_id = next_data.get('buildId')
+ if build_id:
+ return build_id
+ except (ValueError, TypeError):
+ pass
+
+ match = re.search(r'/_next/static/([^/]+)/_buildManifest\\.js', html)
+ if match:
+ return match.group(1)
+
+ return None
+
+ def get_live_list_url(self):
+ """Build the live channel JSON URL dynamically from current Next.js buildId."""
+ try:
+ resp = self.session.get(GEM_BASE_URL)
+ if resp.status_code == 200:
+ build_id = self.extract_build_id(resp.text)
+ if build_id:
+ return LIST_URL_TEMPLATE.format(build_id)
+ log('WARNING: Unable to find buildId in {} response'.format(GEM_BASE_URL), True)
+ else:
+ log('WARNING: {} returns status of {}'.format(GEM_BASE_URL, resp.status_code), True)
+ except requests.RequestException as err:
+ log('WARNING: Error fetching {}: {}'.format(GEM_BASE_URL, err), True)
+
+ return LIST_URL_TEMPLATE.format(FALLBACK_BUILD_ID)
def get_live_channels(self):
"""Get the list of live channels."""
- resp = self.session.get(LIST_URL)
+ list_url = self.get_live_list_url()
+ resp = self.session.get(list_url)
if not resp.status_code == 200:
- log('ERROR: {} returns status of {}'.format(LIST_URL, resp.status_code), True)
+ log('ERROR: {} returns status of {}'.format(list_url, resp.status_code), True)
return None
- save_cookies(self.session.cookies)
-
- ret = None
- for result in json.loads(resp.content)['lineups']['results']:
- if result['key'] == LIST_ELEMENT:
- ret = result['items']
-
- future_to_callsign = {}
- with futures.ThreadPoolExecutor(max_workers=20) as executor:
- for i, channel in enumerate(ret):
- callsign = CBC.get_callsign(channel)
- future = executor.submit(self.get_channel_metadata, callsign)
- future_to_callsign[future] = i
-
- for future in futures.as_completed(future_to_callsign):
- i = future_to_callsign[future]
- metadata = future.result()
- ret[i]['image'] = metadata['Metas']['imageHR']
- return ret
+
+ data = json.loads(resp.content)
+ page_data = data.get('pageProps', {}).get('data', {})
+ streams = page_data.get('streams', [])
+ free_tv_items = page_data.get('freeTv', {}).get('items', [])
+
+ channels = []
+ for stream in streams:
+ items = stream.get('items', [])
+ if len(items) == 0:
+ continue
+
+ for item in items:
+ channel = dict(item)
+ if 'title' not in channel or not channel['title']:
+ channel['title'] = stream.get('title')
+ if 'genericImage' in channel and 'image' not in channel:
+ channel['image'] = channel['genericImage']
+ channels.append(channel)
+
+ for item in free_tv_items:
+ channel = dict(item)
+ if 'genericImage' in channel and 'image' not in channel:
+ channel['image'] = channel['genericImage']
+ channels.append(channel)
+
+ unique_channels = []
+ seen_ids = set()
+ for channel in channels:
+ id_media = channel.get('idMedia')
+
+ if id_media is None:
+ unique_channels.append(channel)
+ continue
+
+ if id_media in seen_ids:
+ continue
+
+ seen_ids.add(id_media)
+ unique_channels.append(channel)
+
+ return unique_channels
def get_iptv_channels(self):
"""Get the channels in a IPTV Manager compatible list."""
@@ -99,7 +161,6 @@ def get_channel_metadata(self, id):
if not resp.status_code == 200:
log('ERROR: {} returns status of {}'.format(LIST_URL, resp.status_code), True)
return None
- save_cookies(self.session.cookies)
return json.loads(resp.content)
@staticmethod
diff --git a/resources/lib/utils.py b/resources/lib/utils.py
index 2858b64..e45938b 100644
--- a/resources/lib/utils.py
+++ b/resources/lib/utils.py
@@ -1,18 +1,10 @@
"""Utilities module."""
import os
import pickle
+from datetime import datetime, timezone
from requests.utils import dict_from_cookiejar
from requests.cookies import cookiejar_from_dict
-def get_cookie_file():
- """Get the cookies file."""
- try:
- from xbmcvfs import translatePath
- base = translatePath('special://userdata/addon_data/plugin.video.cbc')
- except ModuleNotFoundError:
- base = os.getcwd()
- return os.path.join(base, 'cookies')
-
def get_iptv_channels_file():
"""Get the filename for the IPTV channels filter."""
@@ -23,31 +15,6 @@ def get_iptv_channels_file():
base = os.getcwd()
return os.path.join(base, 'iptvchannels')
-def save_cookies(session_cookies):
- """
- Write cookies to the cookie file
- @param session_cookies the session.cookies object to save
- """
- with open(get_cookie_file(), 'wb') as f:
- cookies = dict_from_cookiejar(session_cookies)
- pickle.dump(cookies, f)
-
-
-def loadCookies():
- """
- Load cookies from the cookie file into a session.cookies object
- @return a session.cookies object
- """
- try:
- with open(get_cookie_file(), 'rb') as f:
- cookies = pickle.load(f)
- return cookiejar_from_dict(cookies)
- except IOError as err:
- log('Unable to load cookies: {}'.format(err), True)
- return None
-
- return None
-
def getAuthorizationFile():
"""
@@ -95,3 +62,36 @@ def log(msg, error = False):
xbmc.log(full_msg, level=xbmc.LOGERROR if error else xbmc.LOGINFO)
except:
print(msg)
+
+
+def iso8601_to_local(dttm):
+ """Convert an ISO 8601 timestamp (UTC or offset-aware) to local datetime."""
+ try:
+ return datetime.fromisoformat(dttm.replace('Z', '+00:00')).astimezone()
+ except (ValueError, TypeError, AttributeError):
+ return None
+
+
+def is_pending(timestamp, item=None):
+ """Return true when the supplied datetime is in the future.
+
+ If item is supplied and has a string title, append a live marker when pending.
+ """
+ try:
+ if timestamp.tzinfo is None:
+ timestamp = timestamp.replace(tzinfo=timezone.utc)
+ pending = timestamp.astimezone(timezone.utc) > datetime.now(timezone.utc)
+
+ if pending and item is not None and 'title' in item:
+ title = item['title']
+ if isinstance(title, bytes):
+ title = title.decode('utf-8', errors='replace')
+ elif title is None:
+ title = ''
+ elif not isinstance(title, str):
+ title = str(title)
+ item['title'] = f'{title} [Live at {timestamp.strftime("%Y-%m-%d %H:%M:%S")}]'
+ return pending
+ except (ValueError, TypeError, AttributeError) as err:
+ log(f'is_pending failed: {err}', True)
+ return False
diff --git a/test.py b/test.py
index 9443505..59d2b9c 100755
--- a/test.py
+++ b/test.py
@@ -4,6 +4,7 @@
from optparse import OptionParser
from operator import itemgetter
from resources.lib.epg import get_iptv_epg
+from resources.lib.utils import iso8601_to_local, is_pending
# parse the options
parser = OptionParser()
@@ -98,17 +99,17 @@ def progress(x):
if options.format == "section/sports":
b = GemV2.get_format(options.format)
n = GemV2.normalized_format_item(b[0])
- # print(f'{n["label"]}')
+ # print(f'{n["title"]}')
p = GemV2.normalized_format_path(b[0], options.format)
b = GemV2.get_format(p)
# i think these are upcomming
- idx = 0
+ idx = 5
n = GemV2.normalized_format_item(b[idx])
- print(f'{n["label"]}')
+ print(f'{n["title"]}')
p = GemV2.normalized_format_path(b[idx], p)
b = GemV2.get_format(p)
n = GemV2.normalized_format_item(b[0])
- print(f'{n["label"]}')
+ print(f'{n["title"]}')
p = GemV2.normalized_format_path(b[0], p)
s = GemV2.get_stream(id=p, app_code=n['app_code'])
print(f"{s['type']} {s['url']}")
@@ -130,7 +131,9 @@ def progress(x):
print(stream)
elif options.chans:
res = chans.get_live_channels()
- print(res)
+ for channel in res:
+ labels = CBC.get_labels(channel)
+ print(labels['title'])
elif options.progs:
res = events.getLivePrograms()
elif options.video: