add method and data to queries

Author: anxdpanic

resources/lib/twitch/queries.py

@@ -17,9 +17,11 @@
 
 
 class _Query(object):
-    def __init__(self, url, headers={}):
+    def __init__(self, url, headers={}, data={}, method='GET'):
         self._headers = headers
+        self._data = data
         self._url = url
+        self._method = method
 
         self._params = dict()
         self._urlkws = dict()
@@ -33,10 +35,18 @@ def url(self):
     def headers(self):
         return self._headers
 
+    @property
+    def data(self):
+        return self._data
+
     @property
     def params(self):
         return self._params
 
+    @property
+    def method(self):
+        return self._method
+
     @property
     def urlkws(self):
         return self._urlkws
@@ -45,6 +55,12 @@ def add_path(self, path):
         self._url = urljoin(self._url, path)
         return self
 
+    def add_data(self, key, value, default=None):
+        assert_new(self._data, key)
+        if value != default:
+            self._data[key] = value
+        return self
+
     def add_param(self, key, value, default=None):
         assert_new(self._params, key)
         if value != default:
@@ -57,12 +73,12 @@ def add_urlkw(self, kw, replacement):
         return self
 
     def __str__(self):
-        return 'Query to {url}, params {params}, headers {headers}'.format(
-                url=self.url, params=self.params, headers=self.headers)
+        return '{method} Query to {url}, params {params}, data {data},  headers {headers}'.format(
+                url=self.url, params=self.params, headers=self.headers, data=self.data, method=self.method)
 
     def execute(self, f):
         try:
-            return f(self.url, self.params, self.headers)
+            return f(self.url, self.params, self.headers, self.data, self.method)
         except:
             raise ResourceUnavailableException(str(self))
 
@@ -80,42 +96,42 @@ def execute(self):
 
 
 class ApiQuery(JsonQuery):
-    def __init__(self, path, headers={}):
+    def __init__(self, path, headers={}, data={}, method='GET'):
         headers.setdefault('Client-ID', CLIENT_ID)
         headers.setdefault('Authorization', 'OAuth {access_token}'.format(access_token=OAUTH_TOKEN))
-        super(ApiQuery, self).__init__(_kraken_baseurl, headers)
+        super(ApiQuery, self).__init__(_kraken_baseurl, headers, data, method)
         self.add_path(path)
 
 
 class HiddenApiQuery(JsonQuery):
-    def __init__(self, path, headers={}):
+    def __init__(self, path, headers={}, data={}, method='GET'):
         headers.setdefault('Client-Id', CLIENT_ID)
         headers.setdefault('Authorization', 'OAuth {access_token}'.format(access_token=OAUTH_TOKEN))
-        super(HiddenApiQuery, self).__init__(_hidden_baseurl, headers)
+        super(HiddenApiQuery, self).__init__(_hidden_baseurl, headers, data, method)
         self.add_path(path)
 
 
 class UsherQuery(DownloadQuery):
-    def __init__(self, path, headers={}):
+    def __init__(self, path, headers={}, data={}, method='GET'):
         headers.setdefault('Client-Id', CLIENT_ID)
         headers.setdefault('Authorization', 'OAuth {access_token}'.format(access_token=OAUTH_TOKEN))
-        super(UsherQuery, self).__init__(_usher_baseurl, headers)
+        super(UsherQuery, self).__init__(_usher_baseurl, headers, data, method)
         self.add_path(path)
 
 
 class V5Query(ApiQuery):
-    def __init__(self, path):
-        super(V5Query, self).__init__(path, _v5_headers)
+    def __init__(self, path, method='GET'):
+        super(V5Query, self).__init__(path, _v5_headers, method=method)
 
 
 class V3Query(ApiQuery):
-    def __init__(self, path):
-        super(V3Query, self).__init__(path, _v3_headers)
+    def __init__(self, path, method='GET'):
+        super(V3Query, self).__init__(path, _v3_headers, method=method)
 
 
 class V2Query(ApiQuery):
-    def __init__(self, path):
-        super(V2Query, self).__init__(path, _v2_headers)
+    def __init__(self, path, method='GET'):
+        super(V2Query, self).__init__(path, _v2_headers, method=method)
 
 
 def assert_new(d, k):
@@ -132,8 +148,8 @@ def wrapper(*args, **kwargs):
         if not isinstance(qry, _Query):
             raise ValueError('{} did not return a Query, was: {}'.format(
                              f.__name__, repr(qry)))
-        log.debug('QUERY: url: %s, params: %s, '
+        log.debug('%s QUERY: url: %s, params: %s, data: %s, '
                   'headers: %r, target_func: %r',
-                  qry.url, qry.params, qry.headers, f.__name__)
+                  qry.method, qry.url, qry.params, qry.data, qry.headers, f.__name__)
         return qry.execute()
     return wrapper

resources/lib/twitch/scraper.py

@@ -17,34 +17,41 @@
 MAX_RETRIES = 5
 
 
-def get_json(baseurl, parameters={}, headers={}):
+def get_json(baseurl, parameters={}, headers={}, data={}, method='GET'):
     '''Download Data from an URL and returns it as JSON
     @param url Url to download from
     @param parameters Parameter dict to be encoded with url
     @param headers Headers dict to pass with Request
+    @param data Request body
+    @param method Request method
     @returns JSON Object with data from URL
     '''
-    jsonString = download(baseurl, parameters, headers)
+    jsonString = download(baseurl, parameters, headers, data, method)
     jsonDict = json.loads(jsonString)
     log.debug(json.dumps(jsonDict, indent=4, sort_keys=True))
     return jsonDict
 
 
-def download(baseurl, parameters={}, headers={}):
+def download(baseurl, parameters={}, headers={}, data={}, method='GET'):
     '''Download Data from an url and returns it as a String
+    @param method Request method
     @param baseurl Url to download from (e.g. http://www.google.com)
     @param parameters Parameter dict to be encoded with url
     @param headers Headers dict to pass with Request
+    @param data Request body
+    @param method Request method
     @returns String of data from URL
     '''
     url = '?'.join([baseurl, urlencode(parameters)])
     log.debug('Downloading: ' + url)
-    data = ""
+    content = ""
     for _ in range(MAX_RETRIES):
         try:
             headers.update({USER_AGENT: USER_AGENT_STRING})
-            response = requests.get(url, headers=headers)
-            data = response.content
+            response = requests.request(method=method, url=url, headers=headers, data=data)
+            content = response.content
+            if not content and response.status_code == 204:
+                content = '{"status": 204}'
             break
         except Exception as err:
             if not isinstance(err, URLError):
@@ -53,4 +60,4 @@ def download(baseurl, parameters={}, headers={}):
             log.debug("Error %s during HTTP Request, retrying", repr(err))
     else:
         raise
-    return data
+    return content