Add support for published files through native API

Author: rliebz

dataverse/dataset.py

@@ -7,7 +7,7 @@
 
 from exceptions import (
     MethodNotAllowedError, NoContainerError, OperationFailedError,
-    ConnectionError,
+    ConnectionError, MetadataNotFoundError
 )
 from file import DataverseFile
 from settings import SWORD_BOOTSTRAP
@@ -22,14 +22,16 @@ def __init__(self, entry=SWORD_BOOTSTRAP, dataverse=None, edit_uri=None,
         This can be specified in the atom entry or as kwargs
         """
         self.dataverse = dataverse
-        self._statement = None
-        self._state = None
 
         self.edit_uri = edit_uri
         self.edit_media_uri = edit_media_uri
         self.statement_uri = statement_uri
 
         self._entry = etree.XML(entry) if isinstance(entry, str) else entry
+        self._statement = None
+        self._state = None
+        self._json = None
+        self._id = None
 
         # Updates sword entry from keyword arguments
         for key, value in kwargs.iteritems():
@@ -72,8 +74,49 @@ def from_dataverse(cls, entry_element, dataverse):
 
     @property
     def doi(self):
+        if not self.dataverse:
+            raise NoContainerError('This dataset has not been added to a Dataverse.')
+
         # Note: This depends strongly on URL structure, and may break easily
-        return self.edit_media_uri.rsplit("/study/")[-1]
+        return self.edit_media_uri.rsplit("/study/", 1)[-1]
+
+    @property
+    def id(self):
+        if self._id:
+            return self._id
+
+        if not self.dataverse:
+            raise NoContainerError('This dataset has not been added to a Dataverse.')
+
+        for dataset in self.dataverse.get_contents(refresh=True):
+            doi = '{0}:{1}/{2}'.format(
+                dataset['protocol'],
+                dataset['authority'],
+                dataset['identifier'],
+            )
+            if doi == self.doi:
+                self._id = dataset['id']
+                return self._id
+
+        raise MetadataNotFoundError('The dataset ID could not be found.')
+
+    def get_contents(self, refresh=False):
+        if not refresh and self._contents_json:
+            return self._contents_json
+
+        content_uri = 'https://{0}/api/dataverses/{1}/contents'.format(
+            self.connection.host, self.alias
+        )
+        resp = requests.get(
+            content_uri,
+            params={'key': self.connection.token}
+        )
+
+        if resp.status_code != 200:
+            raise ConnectionError('Atom entry could not be retrieved.')
+
+        self._contents_json = resp.json()
+        return self._contents_json
 
     @property
     def citation(self):
@@ -145,21 +188,48 @@ def get_state(self, refresh=False):
         ).text
         return self._state
 
-    def get_file(self, file_name, published=False):
-        files = self.get_files(published)
+    def get_json(self, refresh=False):
+        if not refresh and self._json:
+            return self._json
+
+        if not self.dataverse:
+            raise NoContainerError('This dataset has not been added to a Dataverse.')
+
+        # TODO: Allow specification of other versions
+        json_url = 'https://{0}/api/datasets/{1}/versions/:latest-published'.format(
+            self.connection.host,
+            self.id
+        )
+
+        resp = requests.get(json_url, params={'key': self.connection.token})
+
+        if resp.status_code != 200:
+            raise ConnectionError('JSON metadata could not be retrieved.')
+
+        self._json = resp.json()['data']
+        return self._json
+
+    def get_file(self, file_name, published=False, refresh=True):
+        files = self.get_files(published, refresh)
         return next((f for f in files if f.name == file_name), None)
 
-    def get_file_by_id(self, file_id, published=False):
-        files = self.get_files(published)
+    def get_file_by_id(self, file_id, published=False, refresh=True):
+        files = self.get_files(published, refresh)
         return next((f for f in files if f.id == file_id), None)
 
     def get_files(self, published=False, refresh=True):
-        if self.get_state(refresh) == 'DRAFT' and published:
-            return []
-        elements = get_elements(self.get_statement(), 'entry')
-        return [DataverseFile.from_statement(element, self)
+        if published:
+            return self.get_published_files(refresh)
+
+        # TODO: Should the native API be preferred?
+        elements = get_elements(self.get_statement(refresh), 'entry')
+        return [DataverseFile.from_statement(self, element)
                 for element in elements]
 
+    def get_published_files(self, refresh=True):
+        return [DataverseFile.from_json(self, file_json)
+                for file_json in self.get_json(refresh)['files']]
+
     def add_file(self, filepath):
         self.add_files([filepath])
 
@@ -235,6 +305,7 @@ def delete_all_files(self):
             self.delete_file(f)
 
     # TODO: DANGEROUS! Will delete all unspecified fields! Deposit receipts only give SOME of the fields
+    # Can potentially be replaced with native API functionality
     # def update_metadata(self):
     #     depositReceipt = self.hostDataverse.connection.sword.update(
     #         dr=self.lastDepositReceipt,

dataverse/dataverse.py

@@ -3,6 +3,7 @@
 from dataset import Dataset
 from exceptions import (
     InsufficientMetadataError, MethodNotAllowedError, OperationFailedError,
+    ConnectionError
 )
 from utils import get_element, get_elements, sanitize
 
@@ -11,6 +12,7 @@ class Dataverse(object):
     def __init__(self, connection, collection):
         self.connection = connection
         self.collection = collection
+        self._contents_json = None
 
     @property
     def is_published(self):
@@ -41,6 +43,24 @@ def title(self):
             tag='title',
         ).text)
 
+    def get_contents(self, refresh=False):
+        if not refresh and self._contents_json:
+            return self._contents_json
+
+        content_uri = 'https://{0}/api/dataverses/{1}/contents'.format(
+            self.connection.host, self.alias
+        )
+        resp = requests.get(
+            content_uri,
+            params={'key': self.connection.token}
+        )
+
+        if resp.status_code != 200:
+            raise ConnectionError('Atom entry could not be retrieved.')
+
+        self._contents_json = resp.json()['data']
+        return self._contents_json
+
     def publish(self):
         edit_uri = 'https://{0}/dvn/api/data-deposit/v1.1/swordv2/edit/dataverse/{1}'.format(
             self.connection.host, self.alias

dataverse/exceptions.py

@@ -30,4 +30,8 @@ class ConnectionError(DataverseError):
 
 class OperationFailedError(DataverseError):
     """Raised when an operation fails for an unknown reason"""
+    pass
+
+class MetadataNotFoundError(DataverseError):
+    """Raised when metadata cannot be found for an unknown reason"""
     pass

dataverse/file.py

@@ -5,35 +5,36 @@
 
 
 class DataverseFile(object):
-    def __init__(self, name, dataset, edit_media_uri=None, download_url=None):
-        self.name = sanitize(name)
+    def __init__(self, dataset, name, file_id=None, edit_media_uri=None):
         self.dataset = dataset
+        self.name = sanitize(name)
 
         if edit_media_uri:
             self.is_published = False
             self.edit_media_uri = edit_media_uri
             self.id = edit_media_uri.split('/')[-2]
-            host = urlparse.urlparse(edit_media_uri).netloc
             self.download_url = 'http://{0}/api/access/datafile/{1}'.format(
-                host, self.id
+                dataset.connection.host, self.id
             )
-        elif download_url:
+        elif file_id:
             self.is_published = True
-            self.download_url = download_url
-            self.id = download_url.rsplit('=', 1)[-1]
+            self.id = file_id
+            self.download_url = 'http://{0}/api/access/datafile/{1}'.format(
+                dataset.connection.host, self.id
+            )
         else:
             raise InsufficientMetadataError(
-                'Files must have an edit media uri or download url.'
+                'Files must have a file id or edit media uri.'
             )
 
     @classmethod
-    def from_statement(cls, element, dataset):
+    def from_statement(cls, dataset, element):
         edit_media_uri = get_element(element, 'content').get('src')
         name = edit_media_uri.rsplit("/", 1)[-1]
-        return cls(name, dataset, edit_media_uri=edit_media_uri)
+        return cls(dataset, name, edit_media_uri=edit_media_uri)
 
     @classmethod
-    def from_metadata(cls, element, dataset):
-        name = element[0].text
-        download_url = element.attrib.get('URI')
-        return cls(name, dataset, download_url=download_url)
+    def from_json(cls, dataset, json):
+        name = json['datafile']['name']
+        file_id = json['datafile']['id']
+        return cls(dataset, name, file_id)