Add persistent HTTP cache.

Avoid exceeding Github limits and speed up fetching in development.

Author: mostlyobvious

Gemfile.lock

@@ -3,6 +3,7 @@ PATH
   specs:
     nanoc-github (1.0.1)
       concurrent-ruby (>= 1.1.6, < 2.0)
+      faraday-http-cache
       nanoc (~> 4.0)
       octokit (~> 4.0)
 
@@ -23,8 +24,10 @@ GEM
     ddplugin (1.0.2)
     diff-lcs (1.3)
     equatable (0.6.1)
-    faraday (1.0.0)
+    faraday (0.17.3)
       multipart-post (>= 1.2, < 3)
+    faraday-http-cache (2.0.0)
+      faraday (~> 0.8)
     hamster (3.0.0)
       concurrent-ruby (~> 1.0)
     hashdiff (1.0.1)
@@ -63,7 +66,7 @@ GEM
       nanoc-checking (~> 1.0)
       nanoc-cli (~> 4.11, >= 4.11.15)
       nanoc-core (~> 4.11, >= 4.11.15)
-    octokit (4.17.0)
+    octokit (4.18.0)
       faraday (>= 0.9)
       sawyer (~> 0.8.0, >= 0.5.3)
     parallel (1.19.1)
@@ -78,7 +81,7 @@ GEM
       addressable (>= 2.3.5)
       faraday (> 0.8, < 2.0)
     slow_enumerator_tools (1.1.0)
-    tomlrb (1.2.9)
+    tomlrb (1.3.0)
     tty-color (0.5.1)
     tty-command (0.9.0)
       pastel (~> 0.7.0)

README.md

@@ -1,6 +1,6 @@
 # Nanoc::Github
 
-Content source from git repository. A way to have your writing in public and open for editing while not being distracted 
+Content source from git repository. A way to have your writing in public and open for editing while not being distracted
 by static site generator trivia this content is usually mixed with.
 
 ## Usage
@@ -28,6 +28,7 @@ data_sources:
     access_token: secret123                        # github access token, not required for public repositories        (default: nil)
     path: posts/                                   # subdirectory of the content in given repository                  (default: nil)
     concurrency: 10                                # how many threads to spawn to fetch data                          (default: 5)
+    verbose: true                                  # show HTTP cache hit/miss on STDOUT                               (default: false)
 ```
 
 ## Status

lib/nanoc/github.rb

@@ -1,14 +1,54 @@
 require "nanoc"
 require "octokit"
 require "concurrent-ruby"
+require "faraday/http_cache"
+require "pstore"
 
 module Nanoc
   module Github
     REGEX = /^(?<metadata>---\s*\n.*?\n?)^(---\s*$\n?)/m
 
+    class Cache
+      def initialize(cache_dir)
+        @store = PStore.new(File.join(cache_dir, "nanoc-github.store"))
+      end
+
+      def write(name, value, options = nil)
+        store.transaction { store[name] = value }
+      end
+
+      def read(name, options = nil)
+        store.transaction(true) { store[name] }
+      end
+
+      def delete(name, options = nil)
+        store.transaction { store.delete(name) }
+      end
+
+      private
+      attr_reader :store
+    end
+
     class Source < Nanoc::DataSource
       identifier :github
 
+      def up
+        stack = Faraday::RackBuilder.new do |builder|
+          builder.use Faraday::HttpCache,
+            serializer: Marshal,
+            shared_cache: false,
+            store: Cache.new(tmp_dir),
+            logger: verbose ? logger : nil
+          builder.use Faraday::Request::Retry,
+            exceptions: [Octokit::ServerError]
+          builder.use Octokit::Middleware::FollowRedirects
+          builder.use Octokit::Response::RaiseError
+          builder.use Octokit::Response::FeedParser
+          builder.adapter Faraday.default_adapter
+        end
+        Octokit.middleware = stack
+      end
+
       def items
         @items ||= begin
           repository_items.map do |item|
@@ -51,7 +91,7 @@ def repository_items
       def encoding
         @config[:encoding] || "utf-8"
       end
-      
+
       def concurrency
         @config[:concurrency] || 5
       end
@@ -67,6 +107,18 @@ def path
       def repository
         @config[:repository]
       end
+
+      def verbose
+        @config[:verbose]
+      end
+
+      def logger
+        Logger.new(STDOUT)
+      end
+
+      def tmp_dir
+        File.join(@site_config.dir, "tmp")
+      end
     end
   end
 end

nanoc-github.gemspec

@@ -24,7 +24,8 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
   spec.extra_rdoc_files = Dir["README.md", "LICENSE.txt"]
 
-  spec.add_dependency "nanoc",           "~> 4.0"
-  spec.add_dependency "octokit",         "~> 4.0"
-  spec.add_dependency "concurrent-ruby", ">= 1.1.6", "< 2.0"
+  spec.add_dependency "nanoc",              "~> 4.0"
+  spec.add_dependency "octokit",            "~> 4.0"
+  spec.add_dependency "concurrent-ruby",    ">= 1.1.6", "< 2.0"
+  spec.add_dependency "faraday-http-cache"
 end