diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..69f2be2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +/.settings +/.project +/.pydevproject +*.pyc +*.pyo +/.idea diff --git a/README.md b/README.md index 5427d52..50d5547 100644 --- a/README.md +++ b/README.md @@ -1 +1,5 @@ -# script.module.python.twitch \ No newline at end of file +# python-twitch for Kodi (script.module.python.twitch) + +# Currently Staging * Untested/Not working + +This Kodi module is a based on python-twitch https://github.com/ingwinlu/python-twitch. Thanks to ingwinlu for his continued work on the project. diff --git a/addon.xml b/addon.xml new file mode 100644 index 0000000..a842bb6 --- /dev/null +++ b/addon.xml @@ -0,0 +1,16 @@ + + + + + + + + + all + A fork of python-twitch, for Kodi + python-twitch for Kodi is module for interaction with the Twitch.tv API based on python-twitch by ingwinlu. + GNU GENERAL PUBLIC LICENSE. Version 3, 29 June 2007 + + https://github.com/MrSprigster/script.module.python.twitch + + diff --git a/resources/lib/twitch/__init__.py b/resources/lib/twitch/__init__.py new file mode 100644 index 0000000..e2db6f4 --- /dev/null +++ b/resources/lib/twitch/__init__.py @@ -0,0 +1,4 @@ +# -*- encoding: utf-8 -*- + +VERSION = '1.3.0' +CLIENT_ID = 'conc17x2vpauvp2youhs3legi90c6jx' diff --git a/resources/lib/twitch/api/__init__.py b/resources/lib/twitch/api/__init__.py new file mode 100644 index 0000000..b9fcafd --- /dev/null +++ b/resources/lib/twitch/api/__init__.py @@ -0,0 +1,6 @@ +# -*- encoding: utf-8 -*- + +from twitch.api import v3 +from twitch.api import v3 as default + +__all__ = ['v3', 'default'] diff --git a/resources/lib/twitch/api/parameters.py b/resources/lib/twitch/api/parameters.py new file mode 100644 index 0000000..08a745b --- /dev/null +++ b/resources/lib/twitch/api/parameters.py @@ -0,0 +1,40 @@ +# -*- encoding: utf-8 -*- + + +class _Parameter(object): + _valid = [] + + @classmethod + def validate(cls, value): + if value in cls._valid: + return value + raise ValueError(value) + + +class Period(_Parameter): + WEEK = 'week' + MONTH = 'month' + ALL = 'all' + _valid = [WEEK, MONTH, ALL] + + +class Boolean(_Parameter): + TRUE = 'true' + FALSE = 'false' + + _valid = [TRUE, FALSE] + + +class Direction(_Parameter): + DESC = 'desc' + ASC = 'asc' + + _valid = [DESC, ASC] + + +class SortBy(_Parameter): + CREATED_AT = 'created_at' + LAST_BROADCAST = 'last_broadcast' + LOGIN = 'login' + + _valid = [CREATED_AT, LAST_BROADCAST, LOGIN] diff --git a/resources/lib/twitch/api/usher.py b/resources/lib/twitch/api/usher.py new file mode 100644 index 0000000..9478c03 --- /dev/null +++ b/resources/lib/twitch/api/usher.py @@ -0,0 +1,67 @@ +# -*- encoding: utf-8 -*- + +from twitch.logging import log # NOQA +log.warning('By using this module you are violating the Twitch TOS') # NOQA + +from twitch import keys +from twitch.api.parameters import Boolean +from twitch.parser import m3u8 +from twitch.queries import HiddenApiQuery, UsherQuery +from twitch.queries import query + + +@query +def channel_token(channel): + q = HiddenApiQuery('channels/{channel}/access_token') + q.add_urlkw(keys.CHANNEL, channel) + return q + + +@query +def vod_token(id): + q = HiddenApiQuery('vods/{vod}/access_token') + q.add_urlkw(keys.VOD, id) + return q + + +@query +def _legacy_video(id): + q = HiddenApiQuery('videos/{id}') + q.add_urlkw(keys.ID, id) + return q + + +@m3u8 +@query +def live(channel): + token = channel_token(channel) + + q = UsherQuery('api/channel/hls/{channel}.m3u8') + q.add_urlkw(keys.CHANNEL, channel) + q.add_param(keys.SIG, token[keys.SIG]) + q.add_param(keys.TOKEN, token[keys.TOKEN]) + q.add_param(keys.ALLOW_SOURCE, Boolean.FALSE) + return q + + +@m3u8 +@query +def _vod(id): + id = id[1:] + + token = vod_token(id) + + q = UsherQuery('vod/{id}') + q.add_urlkw(keys.ID, id) + q.add_param(keys.NAUTHSIG, token[keys.SIG]) + q.add_param(keys.NAUTH, token[keys.TOKEN]) + return q + + +def video(id): + if id.startswith('v'): + return _vod(id) + elif id.startswith(('a', 'c')): + return _legacy_video(id) + else: + raise NotImplementedError('Unknown Video Type') diff --git a/resources/lib/twitch/api/v2/__init__.py b/resources/lib/twitch/api/v2/__init__.py new file mode 100644 index 0000000..e6daec9 --- /dev/null +++ b/resources/lib/twitch/api/v2/__init__.py @@ -0,0 +1,2 @@ +# -*- encoding: utf-8 -*- +# https://github.com/justintv/Twitch-API/blob/master/v2_resources/ diff --git a/resources/lib/twitch/api/v3/__init__.py b/resources/lib/twitch/api/v3/__init__.py new file mode 100644 index 0000000..675cec5 --- /dev/null +++ b/resources/lib/twitch/api/v3/__init__.py @@ -0,0 +1,16 @@ +# -*- encoding: utf-8 -*- +# https://github.com/justintv/Twitch-API/blob/master/v3_resources/ + +from twitch.api.v3 import blocks # NOQA +from twitch.api.v3 import channels # NOQA +from twitch.api.v3 import chat # NOQA +from twitch.api.v3 import follows # NOQA +from twitch.api.v3 import games # NOQA +from twitch.api.v3 import ingests # NOQA +from twitch.api.v3 import search # NOQA +from twitch.api.v3 import streams # NOQA +from twitch.api.v3 import subscriptions # NOQA +from twitch.api.v3 import teams # NOQA +from twitch.api.v3 import users # NOQA +from twitch.api.v3 import videos # NOQA +from twitch.api.v3.root import root # NOQA diff --git a/resources/lib/twitch/api/v3/blocks.py b/resources/lib/twitch/api/v3/blocks.py new file mode 100644 index 0000000..9fc14f4 --- /dev/null +++ b/resources/lib/twitch/api/v3/blocks.py @@ -0,0 +1,22 @@ +# -*- encoding: utf-8 -*- +# https://github.com/justintv/Twitch-API/blob/master/v3_resources/blocks.md + +from twitch.queries import query + + +# Needs Authentification +@query +def by_name(user): + raise NotImplementedError + + +# Needs Authentification, needs PUT +@query +def add_block(user, target): + raise NotImplementedError + + +# Needs Authentification, needs DELETE +@query +def del_block(user, target): + raise NotImplementedError diff --git a/resources/lib/twitch/api/v3/channels.py b/resources/lib/twitch/api/v3/channels.py new file mode 100644 index 0000000..456e57e --- /dev/null +++ b/resources/lib/twitch/api/v3/channels.py @@ -0,0 +1,54 @@ +# -*- encoding: utf-8 -*- +# https://github.com/justintv/Twitch-API/blob/master/v3_resources/channels.md + +from twitch import keys +from twitch.queries import V3Query as Qry +from twitch.queries import query + +from .videos import by_channel + + +@query +def by_name(name): + q = Qry('channels/{channel}') + q.add_urlkw(keys.CHANNEL, name) + return q + + +@query +def channel(): + raise NotImplementedError + + +def get_videos(name, **kwargs): + return by_channel(name, **kwargs) + + +@query +def editors(name): + raise NotImplementedError + + +# TODO needs authentification and put requests +@query +def update(name, status=None, game=None, delay=0): + raise NotImplementedError + + +# TODO needs auth +@query +def delete(name): + raise NotImplementedError + + +# TODO needs auth, needs POST request +@query +def commercial(name, length=30): + raise NotImplementedError + + +@query +def teams(name): + q = Qry('channels/{channel}/teams') + q.add_urlkw('channel', name) + return q diff --git a/resources/lib/twitch/api/v3/chat.py b/resources/lib/twitch/api/v3/chat.py new file mode 100644 index 0000000..049dde7 --- /dev/null +++ b/resources/lib/twitch/api/v3/chat.py @@ -0,0 +1,26 @@ +# -*- encoding: utf-8 -*- +# https://github.com/justintv/Twitch-API/blob/master/v3_resources/chat.md + +from twitch import keys +from twitch.queries import V3Query as Qry +from twitch.queries import query + + +@query +def by_channel(name): + q = Qry('chat/{channel}') + q.add_urlkw(keys.CHANNEL, name) + return q + + +@query +def badges(name): + q = Qry('chat/{channel}/badges') + q.add_urlkw(keys.CHANNEL, name) + return q + + +@query +def emoticons(): + q = Qry('chat/emoticons') + return q diff --git a/resources/lib/twitch/api/v3/follows.py b/resources/lib/twitch/api/v3/follows.py new file mode 100644 index 0000000..0714eb7 --- /dev/null +++ b/resources/lib/twitch/api/v3/follows.py @@ -0,0 +1,55 @@ +# -*- encoding: utf-8 -*- +# https://github.com/justintv/Twitch-API/blob/master/v3_resources/follows.md + +from twitch import keys +from twitch.api.parameters import Direction, SortBy +from twitch.queries import V3Query as Qry +from twitch.queries import query + + +@query +def by_channel(name, limit=25, offset=0, direction=Direction.DESC): + q = Qry('channels/{channel}/follows') + q.add_urlkw(keys.CHANNEL, name) + q.add_param(keys.LIMIT, limit, 25) + q.add_param(keys.OFFSET, offset, 0) + q.add_param(keys.DIRECTION, direction, Direction.DESC) + return q + + +@query +def by_user(name, limit=25, offset=0, direction=Direction.DESC, + sort_by=SortBy.CREATED_AT): + q = Qry('users/{user}/follows/channels') + q.add_urlkw(keys.USER, name) + q.add_param(keys.LIMIT, limit, 25) + q.add_param(keys.OFFSET, offset, 0) + q.add_param(keys.DIRECTION, direction, Direction.DESC) + q.add_param(keys.SORT_BY, sort_by, SortBy.CREATED_AT) + return q + + +@query +def status(user, target): + q = Qry('users/{user}/follows/channels/{target}') + q.add_urlkw(keys.USER, user) + q.add_urlkw(keys.TARGET, target) + return q + + +# Needs Auth, needs PUT +@query +def follow(user, target): + raise NotImplementedError + + +# Needs Auth, needs DELETE +@query +def unfollow(user, target): + raise NotImplementedError + + +# Needs Auth +@query +def streams(): + raise NotImplementedError diff --git a/resources/lib/twitch/api/v3/games.py b/resources/lib/twitch/api/v3/games.py new file mode 100644 index 0000000..f14ed10 --- /dev/null +++ b/resources/lib/twitch/api/v3/games.py @@ -0,0 +1,14 @@ +# -*- encoding: utf-8 -*- +# https://github.com/justintv/Twitch-API/blob/master/v3_resources/games.md + +from twitch import keys +from twitch.queries import V3Query as Qry +from twitch.queries import query + + +@query +def top(limit=10, offset=0): + q = Qry('games/top') + q.add_param(keys.LIMIT, limit, 10) + q.add_param(keys.OFFSET, offset, 0) + return q diff --git a/resources/lib/twitch/api/v3/ingests.py b/resources/lib/twitch/api/v3/ingests.py new file mode 100644 index 0000000..ddfebe1 --- /dev/null +++ b/resources/lib/twitch/api/v3/ingests.py @@ -0,0 +1,11 @@ +# -*- encoding: utf-8 -*- +# https://github.com/justintv/Twitch-API/blob/master/v3_resources/ingests.md + +from twitch.queries import V3Query as Qry +from twitch.queries import query + + +@query +def get(): + q = Qry('ingests') + return q diff --git a/resources/lib/twitch/api/v3/root.py b/resources/lib/twitch/api/v3/root.py new file mode 100644 index 0000000..b0fc249 --- /dev/null +++ b/resources/lib/twitch/api/v3/root.py @@ -0,0 +1,11 @@ +# -*- encoding: utf-8 -*- +# https://github.com/justintv/Twitch-API/blob/master/v3_resources/root.md + +from twitch.queries import V3Query as Qry +from twitch.queries import query + + +# TODO token as parameter +@query +def root(): + return Qry('') diff --git a/resources/lib/twitch/api/v3/search.py b/resources/lib/twitch/api/v3/search.py new file mode 100644 index 0000000..b9268db --- /dev/null +++ b/resources/lib/twitch/api/v3/search.py @@ -0,0 +1,36 @@ +# -*- encoding: utf-8 -*- +# https://github.com/justintv/Twitch-API/blob/master/v3_resources/search.md + +from twitch import keys +from twitch.api.parameters import Boolean +from twitch.queries import V3Query as Qry +from twitch.queries import query + + +@query +def channels(query, limit=25, offset=0): + q = Qry('search/channels') + q.add_param(keys.QUERY, query) + q.add_param(keys.LIMIT, limit, 25) + q.add_param(keys.OFFSET, offset, 0) + return q + + +@query +def streams(query, limit=25, offset=0, hls=Boolean.FALSE): + q = Qry('search/streams') + q.add_param(keys.QUERY, query) + q.add_param(keys.LIMIT, limit, 25) + q.add_param(keys.OFFSET, offset, 0) + q.add_param(keys.HLS, hls, Boolean.FALSE) + return q + + +# 'type' can currently only be suggest, so it is hardcoded +@query +def games(query, live=Boolean.FALSE): + q = Qry('search/games') + q.add_param(keys.QUERY, query) + q.add_param(keys.TYPE, 'suggest') + q.add_param(keys.LIVE, live, Boolean.FALSE) + return q diff --git a/resources/lib/twitch/api/v3/streams.py b/resources/lib/twitch/api/v3/streams.py new file mode 100644 index 0000000..0a5db83 --- /dev/null +++ b/resources/lib/twitch/api/v3/streams.py @@ -0,0 +1,44 @@ +# -*- encoding: utf-8 -*- +# https://github.com/justintv/Twitch-API/blob/master/v3_resources/streams.md + +from twitch import keys +from twitch.queries import V3Query as Qry +from twitch.queries import query + + +@query +def by_channel(name): + q = Qry('streams/{channel}') + q.add_urlkw(keys.CHANNEL, name) + return q + + +@query +def all(game=None, channel=None, limit=25, offset=0, client_id=None): + q = Qry('streams') + q.add_param(keys.GAME, game) + q.add_param(keys.CHANNEL, channel) + q.add_param(keys.LIMIT, limit, 25) + q.add_param(keys.OFFSET, offset, 0) + q.add_param(keys.CLIENT_ID, client_id) + return q + + +@query +def featured(limit=25, offset=0): + q = Qry('streams/featured') + q.add_param(keys.LIMIT, limit, 25) + q.add_param(keys.OFFSET, offset, 0) + return q + + +@query +def summary(game=None): + q = Qry('streams/summary') + q.add_param(keys.GAME, game) + return q + + +@query +def followed(): + raise NotImplementedError diff --git a/resources/lib/twitch/api/v3/subscriptions.py b/resources/lib/twitch/api/v3/subscriptions.py new file mode 100644 index 0000000..4cb5690 --- /dev/null +++ b/resources/lib/twitch/api/v3/subscriptions.py @@ -0,0 +1,23 @@ +# -*- encoding: utf-8 -*- +# https://github.com/justintv/Twitch-API/blob/master/v3_resources/subscriptions.md + +from twitch.api.parameters import Direction +from twitch.queries import query + + +# Needs Authentification +@query +def by_channel(channel, limit=25, offset=0, direction=Direction.ASC): + raise NotImplementedError + + +# Needs Authentification +@query +def channel_to_user(channel, user): + raise NotImplementedError + + +# Needs Authentification +@query +def user_to_channel(user, channel): + raise NotImplementedError diff --git a/resources/lib/twitch/api/v3/teams.py b/resources/lib/twitch/api/v3/teams.py new file mode 100644 index 0000000..4a2794c --- /dev/null +++ b/resources/lib/twitch/api/v3/teams.py @@ -0,0 +1,21 @@ +# -*- encoding: utf-8 -*- +# https://github.com/justintv/Twitch-API/blob/master/v3_resources/teams.md + +from twitch import keys +from twitch.queries import V3Query as Qry +from twitch.queries import query + + +@query +def active(limit=25, offset=0): + q = Qry('teams') + q.add_param(keys.LIMIT, limit, 25) + q.add_param(keys.OFFSET, offset, 0) + return q + + +@query +def by_name(name): + q = Qry('teams/{team}') + q.add_urlkw(keys.TEAM, name) + return q diff --git a/resources/lib/twitch/api/v3/users.py b/resources/lib/twitch/api/v3/users.py new file mode 100644 index 0000000..800409a --- /dev/null +++ b/resources/lib/twitch/api/v3/users.py @@ -0,0 +1,31 @@ +# -*- encoding: utf-8 -*- +# https://github.com/justintv/Twitch-API/blob/master/v3_resources/users.md + +from twitch import keys +from twitch.queries import V3Query as Qry +from twitch.queries import query + + +@query +def by_name(name): + q = Qry('users/{user}') + q.add_urlkw(keys.USER, name) + return q + + +# Needs Authentification +@query +def user(): + raise NotImplementedError + + +# Needs Authentification +@query +def streams(): + raise NotImplementedError + + +# Needs Authentification +@query +def videos(): + raise NotImplementedError diff --git a/resources/lib/twitch/api/v3/videos.py b/resources/lib/twitch/api/v3/videos.py new file mode 100644 index 0000000..2f2080e --- /dev/null +++ b/resources/lib/twitch/api/v3/videos.py @@ -0,0 +1,43 @@ +# -*- encoding: utf-8 -*- +# https://github.com/justintv/Twitch-API/blob/master/v3_resources/videos.md + +from twitch import keys +from twitch.api.parameters import Boolean, Period +from twitch.queries import V3Query as Qry +from twitch.queries import query + +from .users import videos + + +@query +def by_id(identification): + q = Qry('videos/{id}') + q.add_urlkw(keys.ID, identification) + return q + + +@query +def top(limit=10, offset=0, game=None, period=Period.WEEK): + q = Qry('videos/top') + q.add_param(keys.LIMIT, limit, 10) + q.add_param(keys.OFFSET, offset, 0) + q.add_param(keys.GAME, game) + q.add_param(keys.PERIOD, Period.validate(period), Period.WEEK) + return q + + +@query +def by_channel(name, limit=10, offset=0, + broadcasts=Boolean.FALSE, hls=Boolean.FALSE): + q = Qry('channels/{channel}/videos') + q.add_urlkw(keys.CHANNEL, name) + q.add_param(keys.LIMIT, limit, 10) + q.add_param(keys.OFFSET, offset, 0) + q.add_param(keys.BROADCASTS, Boolean.validate(broadcasts), Boolean.FALSE) + q.add_param(keys.HLS, Boolean.validate(hls), Boolean.FALSE) + return q + + +# Needs Auth +def followed(*args, **kwargs): + videos(*args, **kwargs) diff --git a/resources/lib/twitch/exceptions.py b/resources/lib/twitch/exceptions.py new file mode 100644 index 0000000..ac42919 --- /dev/null +++ b/resources/lib/twitch/exceptions.py @@ -0,0 +1,5 @@ +# -*- encoding: utf-8 -*- + + +class ResourceUnavailableException(Exception): + pass diff --git a/resources/lib/twitch/keys.py b/resources/lib/twitch/keys.py new file mode 100644 index 0000000..5b7d062 --- /dev/null +++ b/resources/lib/twitch/keys.py @@ -0,0 +1,37 @@ +# -*- encoding: utf-8 -*- +""" + keys + + string constants +""" + +ALLOW_SOURCE = 'allow_source' +BROADCASTS = 'broadcasts' +CHANNEL = 'channel' +CLIENT_ID = 'client_id' +DIRECTION = 'direction' +ERROR = 'error' +FEATURED = 'featured' +FOLLOWS = 'follows' +GAME = 'game' +HLS = 'hls' +ID = 'id' +LIMIT = 'limit' +LIVE = 'live' +MESSAGE = 'message' +NAUTH = 'nauth' +NAUTHSIG = 'nauthsig' +OFFSET = 'offset' +PERIOD = 'period' +QUERY = 'query' +SIG = 'sig' +SORT_BY = 'sortby' +TARGET = 'target' +TEAM = 'team' +TOKEN = 'token' +TYPE = 'type' +USER = 'user' +USER_AGENT = 'User-Agent' +USER_AGENT_STRING = ('Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) ' + 'Gecko/20100101 Firefox/6.0') +VOD = 'vod' diff --git a/resources/lib/twitch/logging.py b/resources/lib/twitch/logging.py new file mode 100644 index 0000000..c7cf7f2 --- /dev/null +++ b/resources/lib/twitch/logging.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import + +import logging + +log = logging.getLogger('twitch') +log.addHandler(logging.NullHandler()) + + +def deprecation_warning(logger, old, new): + logger.warning("DEPRECATED call to '%s\' detected, " + "please use '%s' instead", + old, new) diff --git a/resources/lib/twitch/parser.py b/resources/lib/twitch/parser.py new file mode 100644 index 0000000..21c843a --- /dev/null +++ b/resources/lib/twitch/parser.py @@ -0,0 +1,27 @@ +# -*- encoding: utf-8 -*- +import re + +from twitch.logging import log + +_m3u_pattern = re.compile( + r'#EXT-X-MEDIA:.*' + r'GROUP-ID="(?P.\w*)",' + r'NAME="(?P\w*)"[,=\w]*\n' + r'#EXT-X-STREAM-INF:.*\n(' + r'?Phttp.*)') + + +def m3u8(f): + def m3u8_wrapper(*args, **kwargs): + return m3u8_to_dict(f(*args, **kwargs)) + return m3u8_wrapper + + +def m3u8_to_dict(string): + log.debug('m3u8_to_dict called for:\n{}'.format(string)) + d = dict() + matches = re.finditer(_m3u_pattern, string) + for m in matches: + d[m.group('group_name')] = m.group('url') + log.debug('m3u8_to_dict result:\n{}'.format(d)) + return d diff --git a/resources/lib/twitch/queries.py b/resources/lib/twitch/queries.py new file mode 100644 index 0000000..15eff40 --- /dev/null +++ b/resources/lib/twitch/queries.py @@ -0,0 +1,128 @@ +# -*- encoding: utf-8 -*- + +from six.moves.urllib.parse import urljoin + +from twitch import CLIENT_ID +from twitch.exceptions import ResourceUnavailableException +from twitch.logging import log +from twitch.scraper import download, get_json + +_kraken_baseurl = 'https://api.twitch.tv/kraken/' +_hidden_baseurl = 'http://api.twitch.tv/api/' +_usher_baseurl = 'http://usher.twitch.tv/' + +_v2_headers = {'ACCEPT': 'application/vnd.twitchtv.v2+json'} +_v3_headers = {'ACCEPT': 'application/vnd.twitchtv.v3+json'} + + +class _Query(object): + def __init__(self, url, headers={}): + self._headers = headers + self._url = url + + self._params = dict() + self._urlkws = dict() + + @property + def url(self): + formatted_url = self._url.format(**self._urlkws) # throws KeyError + return formatted_url + + @property + def headers(self): + return self._headers + + @property + def params(self): + return self._params + + @property + def urlkws(self): + return self._urlkws + + def add_path(self, path): + self._url = urljoin(self._url, path) + return self + + def add_param(self, key, value, default=None): + assert_new(self._params, key) + if value != default: + self._params[key] = value + return self + + def add_urlkw(self, kw, replacement): + assert_new(self._urlkws, kw) + self._urlkws[kw] = replacement + return self + + def __str__(self): + return 'Query to {url}, params {params}, headers {headers}'.format( + url=self.url, params=self.params, headers=self.headers) + + def execute(self, f): + try: + return f(self.url, self.params, self.headers) + except: + raise ResourceUnavailableException(str(self)) + + +class DownloadQuery(_Query): + def execute(self): + # TODO implement download completely here + return super(DownloadQuery, self).execute(download) + + +class JsonQuery(_Query): + def execute(self): + # TODO implement get_json completely here + return super(JsonQuery, self).execute(get_json) + + +class ApiQuery(JsonQuery): + def __init__(self, path, headers={}): + headers.setdefault('Client-Id', CLIENT_ID) + super(ApiQuery, self).__init__(_kraken_baseurl, headers) + self.add_path(path) + + +class HiddenApiQuery(JsonQuery): + def __init__(self, path, headers={}): + super(HiddenApiQuery, self).__init__(_hidden_baseurl, headers) + self.add_path(path) + + +class UsherQuery(DownloadQuery): + def __init__(self, path, headers={}): + super(UsherQuery, self).__init__(_usher_baseurl, headers) + self.add_path(path) + + +class V3Query(ApiQuery): + def __init__(self, path): + super(V3Query, self).__init__(path, _v3_headers) + + +class V2Query(ApiQuery): + def __init__(self, path): + super(V2Query, self).__init__(path, _v2_headers) + + +def assert_new(d, k): + if k in d: + v = d.get(k) + raise ValueError("Key '{}' already set to '{}'".format( + k, v)) + + +# TODO maybe rename +def query(f): + def wrapper(*args, **kwargs): + qry = f(*args, **kwargs) + if not isinstance(qry, _Query): + raise ValueError('{} did not return a Query, was: {}'.format( + f.__name__, repr(qry))) + log.debug('QUERY: url: %s, params: %s, ' + 'headers: %r, target_func: %r', + qry.url, qry.params, qry.headers, f.__name__) + return qry.execute() + return wrapper diff --git a/resources/lib/twitch/scraper.py b/resources/lib/twitch/scraper.py new file mode 100644 index 0000000..8aa6377 --- /dev/null +++ b/resources/lib/twitch/scraper.py @@ -0,0 +1,60 @@ +# -*- encoding: utf-8 -*- +import six +from six.moves.urllib.error import URLError +from six.moves.urllib.parse import quote_plus # NOQA +from six.moves.urllib.parse import urlencode +from six.moves.urllib.request import Request, urlopen + +from twitch.keys import USER_AGENT, USER_AGENT_STRING +from twitch.logging import log + +try: + import json +except: + import simplejson as json # @UnresolvedImport + +MAX_RETRIES = 5 + + +def get_json(baseurl, parameters={}, headers={}): + '''Download Data from an URL and returns it as JSON + @param url Url to download from + @param parameters Parameter dict to be encoded with url + @param headers Headers dict to pass with Request + @returns JSON Object with data from URL + ''' + jsonString = download(baseurl, parameters, headers) + jsonDict = json.loads(jsonString) + log.debug(json.dumps(jsonDict, indent=4, sort_keys=True)) + return jsonDict + + +def download(baseurl, parameters={}, headers={}): + '''Download Data from an url and returns it as a String + @param baseurl Url to download from (e.g. http://www.google.com) + @param parameters Parameter dict to be encoded with url + @param headers Headers dict to pass with Request + @returns String of data from URL + ''' + url = '?'.join([baseurl, urlencode(parameters)]) + log.debug('Downloading: ' + url) + data = "" + for _ in range(MAX_RETRIES): + try: + req = Request(url, headers=headers) + req.add_header(USER_AGENT, USER_AGENT_STRING) + response = urlopen(req) + if six.PY2: + data = response.read() + else: + data = response.read().decode('utf-8') + response.close() + break + except Exception as err: + if not isinstance(err, URLError): + log.debug("Error %s during HTTP Request, abort", repr(err)) + raise # propagate non-URLError + log.debug("Error %s during HTTP Request, retrying", repr(err)) + else: + raise + return data