add support for loading zip file, load as tabular in case of tsv or csv formst, support read options

Author: OriHoch

README.md

@@ -56,6 +56,21 @@ Validate the data explicitly and get a list of errors
 Package::validate("tests/fixtures/simple_invalid_datapackage.json");  // array of validation errors
 ```
 
+Load a zip file
+
+```php
+$package = Package::load('http://datahub.io/opendatafortaxjustice/eucountrydatawb/r/datapackage_zip.zip');
+```
+
+Provide read options which are passed through to [tableschema-php](https://github.com/frictionlessdata/tableschema-php) Table::read method
+
+```php
+$package = Package::load('http://datahub.io/opendatafortaxjustice/eucountrydatawb/r/datapackage_zip.zip');
+foreach ($package as $resource) {
+    $resource->read(["cast" => false]);
+}
+```
+
 The package object has some useful methods to access and manipulate the resources
 
 ```php
@@ -106,7 +121,6 @@ Save the entire datapackage including any local data to a zip file
 $package->save("datapackage.zip");
 ```
 
-
 ### Resource
 
 Resource objects can be accessed from a Package as described above

src/Factory.php

@@ -4,6 +4,7 @@
 
 use frictionlessdata\datapackage\Datapackages\BaseDatapackage;
 use frictionlessdata\datapackage\Resources\BaseResource;
+use Alchemy\Zippy\Zippy;
 
 /**
  * datapackage and resource have different classes depending on the corresponding profile
@@ -23,6 +24,7 @@ class Factory
      *  - JSON encoded object
      *  - URL (must be in either 'http' or 'https' schemes)
      *  - local filesystem (POSIX) path.
+     *  - local or remote zip file
      *
      * @param mixed       $source
      * @param null|string $basePath optional, required only if you want to use relative paths
@@ -269,37 +271,45 @@ protected static function loadSource($source, $basePath)
                     );
                 }
             } elseif (static::isHttpSource($source)) {
-                try {
-                    $descriptor = json_decode(file_get_contents(static::normalizeHttpSource($source)));
-                } catch (\Exception $e) {
-                    throw new Exceptions\DatapackageInvalidSourceException(
-                        'Failed to load source: '.json_encode($source).': '.$e->getMessage()
-                    );
+                if (static::isHttpZipSource($source)) {
+                    return static::loadHttpZipSource($source);
+                } else {
+                    try {
+                        $descriptor = json_decode(file_get_contents(static::normalizeHttpSource($source)));
+                    } catch (\Exception $e) {
+                        throw new Exceptions\DatapackageInvalidSourceException(
+                            'Failed to load source: '.json_encode($source).': '.$e->getMessage()
+                        );
+                    }
+                    // http sources don't allow relative paths, hence basePath should remain null
+                    $basePath = null;
                 }
-                // http sources don't allow relative paths, hence basePath should remain null
-                $basePath = null;
             } else {
                 // not a json string and not a url - assume it's a file path
-                if (empty($basePath)) {
-                    // no basePath
-                    // - assume source is the absolute path of the file
-                    // - set it's directory as the basePath
-                    $basePath = dirname($source);
+                if (static::isFileZipSource($source)) {
+                    return static::loadFileZipSource($source);
                 } else {
-                    // got a basePath
-                    // - try to prepend it to the source and see if such a file exists
-                    // - if not - assume it's an absolute path
-                    $absPath = $basePath.DIRECTORY_SEPARATOR.$source;
-                    if (file_exists($absPath)) {
-                        $source = $absPath;
+                    if (empty($basePath)) {
+                        // no basePath
+                        // - assume source is the absolute path of the file
+                        // - set it's directory as the basePath
+                        $basePath = dirname($source);
+                    } else {
+                        // got a basePath
+                        // - try to prepend it to the source and see if such a file exists
+                        // - if not - assume it's an absolute path
+                        $absPath = $basePath.DIRECTORY_SEPARATOR.$source;
+                        if (file_exists($absPath)) {
+                            $source = $absPath;
+                        }
+                    }
+                    try {
+                        $descriptor = json_decode(file_get_contents($source));
+                    } catch (\Exception $e) {
+                        throw new Exceptions\DatapackageInvalidSourceException(
+                            'Failed to load source: '.json_encode($source).': '.$e->getMessage()
+                        );
                     }
-                }
-                try {
-                    $descriptor = json_decode(file_get_contents($source));
-                } catch (\Exception $e) {
-                    throw new Exceptions\DatapackageInvalidSourceException(
-                        'Failed to load source: '.json_encode($source).': '.$e->getMessage()
-                    );
                 }
             }
         } else {
@@ -310,4 +320,38 @@ protected static function loadSource($source, $basePath)
 
         return (object) ['descriptor' => $descriptor, 'basePath' => $basePath];
     }
+
+    protected static function isHttpZipSource($source)
+    {
+        return (strtolower(substr($source, -4)) == '.zip');
+    }
+
+    protected static function isFileZipSource($source)
+    {
+        return (strtolower(substr($source, -4)) == '.zip');
+    }
+
+    protected static function loadHttpZipSource($source)
+    {
+        $tempfile = tempnam(sys_get_temp_dir(), 'datapackage-php');
+        unlink($tempfile);
+        $tempfile.='.zip';
+        stream_copy_to_stream(fopen($source, 'r'), fopen($tempfile, 'w'));
+        register_shutdown_function(function() use ($tempfile) {unlink($tempfile);});
+        return self::loadFileZipSource($tempfile);
+    }
+
+    protected static function loadFileZipSource($source)
+    {
+        $zippy = Zippy::load();
+        $tempdir = tempnam(sys_get_temp_dir(), 'datapackage-php');
+        unlink($tempdir);
+        mkdir($tempdir);
+        register_shutdown_function(function() use ($tempdir) {Utils::removeDir($tempdir);});
+        $zippy->open($source)->extract($tempdir);
+        if (!file_exists($tempdir."/datapackage.json")) {
+            throw new Exceptions\DatapackageInvalidSourceException("zip file must contain a datappackage.json file");
+        }
+        return static::loadSource($tempdir."/datapackage.json", $tempdir);
+    }
 }

src/Resources/BaseResource.php

@@ -37,13 +37,31 @@ public static function handlesDescriptor($descriptor)
         return static::handlesProfile(Registry::getResourceValidationProfile($descriptor));
     }
 
-    public function read()
+    public function read($readOptions=null)
     {
+        $limit = ($readOptions && isset($readOptions["limit"])) ? $readOptions["limit"] : null;
         $rows = [];
-        foreach ($this as $row) {
-            $rows[] = $row;
+        foreach ($this->dataStreams() as $dataStream) {
+            if (isset($dataStream->table)) {
+                $readOptions["limit"] = $limit;
+                foreach ($dataStream->table->read($readOptions) as $row) {
+                    $rows[] = $row;
+                    if ($limit !== null) {
+                        $limit--;
+                        if ($limit < 0) break;
+                    }
+                };
+            } else {
+                foreach ($dataStream as $row) {
+                    $rows[] = $row;
+                    if ($limit !== null) {
+                        $limit--;
+                        if ($limit < 0) break;
+                    }
+                }
+            }
+            if ($limit !== null && $limit < 0) break;
         }
-
         return $rows;
     }
 

src/Resources/TabularResource.php

@@ -4,6 +4,7 @@
 
 use frictionlessdata\datapackage\DataStreams\TabularDataStream;
 use frictionlessdata\datapackage\DataStreams\TabularInlineDataStream;
+use frictionlessdata\datapackage\Registry;
 
 class TabularResource extends DefaultResource
 {
@@ -41,8 +42,11 @@ protected function getInlineDataStream($data)
         ]);
     }
 
-    protected static function handlesProfile($profile)
+    public static function handlesDescriptor($descriptor)
     {
-        return $profile == 'tabular-data-resource';
+        return (
+            Registry::getResourceValidationProfile($descriptor) == 'tabular-data-resource'
+            || (isset($descriptor->format) && in_array($descriptor->format, ['csv', 'tsv']))
+        );
     }
 }

tests/DatapackageTest.php

@@ -468,6 +468,23 @@ public function testCreateEditDatapackageDescriptor()
         $this->assertEquals("id,name\n1,one\n2,two\n3,three\n", file_get_contents($tempdir.'resource-1.csv'));
     }
 
+    public function testLoadDatapackageZip()
+    {
+        $package = Package::load(dirname(__FILE__).'/fixtures/datapackage_zip.zip');
+        // $package = Package::load('http://datahub.io/opendatafortaxjustice/eucountrydatawb/r/datapackage_zip.zip');
+        $this->assertEquals([[
+            'jurisdiction' => 'Austria',
+            'population in millions' => 8.7474000000000007,
+            'GDP in $Billions' => 386.42779999999999,
+            'GDP per cap' => 44176.519999999997
+        ], [
+            'jurisdiction' => 'Belgium',
+            'population in millions' => 11.3482,
+            'GDP in $Billions' => 466.3657,
+            'GDP per cap' => 41096.160000000003
+        ]], $package->resource('eucountrydatawb_csv')->read(["limit" => 2]));
+    }
+
     public function testStringPath()
     {
         $package = Package::create(['resources' => [
@@ -540,7 +557,7 @@ public function testDataHubCountryList()
             }
         }
         $this->assertEquals(['data_csv', 'data_json', 'datapackage_zip', 'data'], array_keys($resources));
-        $this->assertEquals('Name,Code', trim($resources['data_csv'][0]));
+        $this->assertEquals(["Name" => "Afghanistan", "Code" => "AF"], $resources['data_csv'][0]);
 
         // now, let's try to load it but get it as tabular data
         $descriptor = json_decode(file_get_contents(dirname(__FILE__).'/fixtures/datahub-country-list/datapackage.json'));
@@ -569,7 +586,7 @@ public function testCommitteesPackage()
             foreach ($resource as $row) {
                 ++$rowNum;
             }
-            $this->assertEquals(706, $rowNum);
+            $this->assertEquals(702, $rowNum);
             ++$resourceNum;
         }
     }