Pin versions in the offline buildpack

Christopher Frost · Christopher Frost · commit ad899698d271 · 2015-06-23T15:08:34.000+01:00
This commit allows a PINNED=true option to be set when building an
offline buildpack that will pin the versions of all buildpack
dependencies to the currently resolved versions. This will result in
changes to the contents of configuration files in the config
directory.

[#93987158]
diff --git a/README.md b/README.md
@@ -100,9 +100,11 @@ Creating build/java-buildpack-cfd6b17.zip
 ### Offline Package
 The offline package is a version of the buildpack designed to run without access to a network.  It packages the latest version of each dependency (as configured in the [`config/` directory][]) and [disables `remote_downloads`][]. This package is about 180M in size.  To create the offline package, use the `OFFLINE=true` argument:
 
+To pin the version of dependencies used by the buildpack to the ones currently resolvable use the `PINNED=true` argument. This will update the [`config/` directory][] to contain exact version of each dependency instead of version ranges.
+
 ```bash
 bundle install
-bundle exec rake package OFFLINE=true
+bundle exec rake package OFFLINE=true PINNED=true
 ...
 Creating build/java-buildpack-offline-cfd6b17.zip
 ```
diff --git a/config/app_dynamics_agent.yml b/config/app_dynamics_agent.yml
@@ -16,6 +16,6 @@
 # Configuration for the New Relic framework
 ---
 version: 4.0.+
-repository_root: "{default.repository.root}/app-dynamics"
-default_node_name: "$(expr \"$VCAP_APPLICATION\" : \'.*instance_index[\": ]*\\([[:digit:]]*\\).*\')"
+repository_root: ! '{default.repository.root}/app-dynamics'
+default_node_name: ! '$(expr "$VCAP_APPLICATION" : ''.*instance_index[": ]*\([[:digit:]]*\).*'')'
 default_tier_name: CloudFoundry
diff --git a/config/groovy.yml b/config/groovy.yml
@@ -16,4 +16,4 @@
 # Configuration for the Groovy container
 ---
 version: 2.4.+
-repository_root: "{default.repository.root}/groovy"
+repository_root: ! '{default.repository.root}/groovy'
diff --git a/config/jrebel_agent.yml b/config/jrebel_agent.yml
@@ -16,4 +16,4 @@
 # Configuration for the JRebel framework
 ---
 version: 6.+
-repository_root: "http://dl.zeroturnaround.com/jrebel"
+repository_root: http://dl.zeroturnaround.com/jrebel
diff --git a/config/maria_db_jdbc.yml b/config/maria_db_jdbc.yml
@@ -16,4 +16,4 @@
 # Configuration for the MariaDB JDBC framework
 ---
 version: 1.1.+
-repository_root: "{default.repository.root}/mariadb-jdbc"
+repository_root: ! '{default.repository.root}/mariadb-jdbc'
diff --git a/config/new_relic_agent.yml b/config/new_relic_agent.yml
@@ -16,4 +16,4 @@
 # Configuration for the New Relic framework
 ---
 version: 3.+
-repository_root: "{default.repository.root}/new-relic"
+repository_root: ! '{default.repository.root}/new-relic'
diff --git a/config/open_jdk_jre.yml b/config/open_jdk_jre.yml
@@ -18,10 +18,10 @@
 ---
 jre:
   version: 1.8.0_+
-  repository_root: "{default.repository.root}/openjdk/{platform}/{architecture}"
+  repository_root: ! '{default.repository.root}/openjdk/{platform}/{architecture}'
 memory_calculator:
   version: 1.+
-  repository_root: "{default.repository.root}/memory-calculator/{platform}/{architecture}"
+  repository_root: ! '{default.repository.root}/memory-calculator/{platform}/{architecture}'
   memory_sizes:
     metaspace: 64m..
     permgen: 64m..
diff --git a/config/play_framework_auto_reconfiguration.yml b/config/play_framework_auto_reconfiguration.yml
@@ -18,5 +18,5 @@
 # avoid conflicts.
 ---
 version: 1.+
-repository_root: "{default.repository.root}/auto-reconfiguration"
+repository_root: ! '{default.repository.root}/auto-reconfiguration'
 enabled: true
diff --git a/config/play_framework_jpa_plugin.yml b/config/play_framework_jpa_plugin.yml
@@ -16,5 +16,5 @@
 # Configuration for the Play JPA Plugin framework
 ---
 version: 1.+
-repository_root: "{default.repository.root}/play-jpa-plugin"
+repository_root: ! '{default.repository.root}/play-jpa-plugin'
 enabled: true
diff --git a/config/postgresql_jdbc.yml b/config/postgresql_jdbc.yml
@@ -16,4 +16,4 @@
 # Configuration for the Postgresql JDBC framework
 ---
 version: 9.4.+
-repository_root: "{default.repository.root}/postgresql-jdbc"
+repository_root: ! '{default.repository.root}/postgresql-jdbc'
diff --git a/config/spring_auto_reconfiguration.yml b/config/spring_auto_reconfiguration.yml
@@ -18,5 +18,5 @@
 # avoid conflicts.
 ---
 version: 1.+
-repository_root: "{default.repository.root}/auto-reconfiguration"
+repository_root: ! '{default.repository.root}/auto-reconfiguration'
 enabled: true
diff --git a/config/spring_boot_cli.yml b/config/spring_boot_cli.yml
@@ -18,4 +18,4 @@
 # avoid conflicts.
 ---
 version: 1.+
-repository_root: "{default.repository.root}/spring-boot-cli"
+repository_root: ! '{default.repository.root}/spring-boot-cli'
diff --git a/config/tomcat.yml b/config/tomcat.yml
@@ -17,39 +17,39 @@
 ---
 tomcat:
   version: 8.0.+
-  repository_root: "{default.repository.root}/tomcat"
+  repository_root: ! '{default.repository.root}/tomcat'
 lifecycle_support:
   version: 2.+
-  repository_root: "{default.repository.root}/tomcat-lifecycle-support"
+  repository_root: ! '{default.repository.root}/tomcat-lifecycle-support'
 logging_support:
   version: 2.+
-  repository_root: "{default.repository.root}/tomcat-logging-support"
+  repository_root: ! '{default.repository.root}/tomcat-logging-support'
 access_logging_support:
   version: 2.+
-  repository_root: "{default.repository.root}/tomcat-access-logging-support"
+  repository_root: ! '{default.repository.root}/tomcat-access-logging-support'
   access_logging: disabled
 redis_store:
   version: 1.+
-  repository_root: "{default.repository.root}/redis-store"
+  repository_root: ! '{default.repository.root}/redis-store'
   database: 0
   timeout: 2000
   connection_pool_size: 2
 gemfire_store:
   gemfire:
     version: 8.0.+
-    repository_root: "{default.repository.root}/gem-fire"
+    repository_root: ! '{default.repository.root}/gem-fire'
   gemfire_modules:
     version: 8.0.+
-    repository_root: "{default.repository.root}/gem-fire-modules"
+    repository_root: ! '{default.repository.root}/gem-fire-modules'
   gemfire_modules_tomcat7:
     version: 8.0.+
-    repository_root: "{default.repository.root}/gem-fire-modules-tomcat7"
+    repository_root: ! '{default.repository.root}/gem-fire-modules-tomcat7'
   gemfire_security:
     version: 8.0.+
-    repository_root: "{default.repository.root}/gem-fire-security"
+    repository_root: ! '{default.repository.root}/gem-fire-security'
   gemfire_logging:
     version: 1.5.8
-    repository_root: "{default.repository.root}/slf4j-jdk14"
+    repository_root: ! '{default.repository.root}/slf4j-jdk14'
   gemfire_logging_api:
     version: 1.5.8
-    repository_root: "{default.repository.root}/slf4j-api"
+    repository_root: ! '{default.repository.root}/slf4j-api'
diff --git a/lib/java_buildpack/util/configuration_utils.rb b/lib/java_buildpack/util/configuration_utils.rb
@@ -34,12 +34,12 @@ class << self
         # exist, returns an empty hash. Overlays configuration in a matching environment variable, on top of the loaded
         # configuration, if present. Will not add a new configuration key where an existing one does not exist.
         #
-        # @param [String] identifier the identifier of the configuration
+        # @param [String] identifier the identifier of the configuration to load
         # @param [Boolean] should_log whether the contents of the configuration file should be logged.  This value
         #                             should be left to its default and exists to allow the logger to use the utility.
         # @return [Hash] the configuration or an empty hash if the configuration file does not exist
         def load(identifier, should_log = true)
-          file = CONFIG_DIRECTORY + "#{identifier}.yml"
+          file = file_name(identifier)
 
           if file.exist?
             user_provided = ENV[environment_variable_name(identifier)]
@@ -51,6 +51,31 @@ def load(identifier, should_log = true)
           configuration || {}
         end
 
+        # Write a new configuration file to the buildpack configuration directory. Any existing file will be replaced.
+        #
+        # @param [String] identifier the identifier of the configuration to write
+        # @param [Boolean] should_log whether the contents of the configuration file should be logged.  This value
+        #                             should be left to its default and exists to allow the logger to use the utility.
+        def write(identifier, new_content, should_log = true)
+          file = file_name(identifier)
+
+          if file.exist?
+            logger.debug { "Writing configuration file #{file}" } if should_log
+            header = header(file)
+
+            File.open(file, 'w') do |f|
+              header.each { |line| f.write line}
+              # f.write(header.join('\n'))
+              # f.write('\n')
+              YAML.dump(new_content, f)
+            end
+          else
+            logger.debug { "No configuration file #{file} found" } if should_log
+          end
+
+        end
+
+
         private
 
         CONFIG_DIRECTORY = Pathname.new(File.expand_path('../../../config', File.dirname(__FILE__))).freeze
@@ -59,6 +84,22 @@ def load(identifier, should_log = true)
 
         private_constant :CONFIG_DIRECTORY, :ENVIRONMENT_VARIABLE_PATTERN
 
+        def file_name(identifier)
+          CONFIG_DIRECTORY + "#{identifier}.yml"
+        end
+
+        def header(file)
+          header = []
+          File.open(file, 'r') do |f|
+            f.each do |line|
+              break if line =~ /^---/
+              fail unless line =~ /^#/ || line =~ /^$/
+              header << line
+            end
+          end
+          header
+        end
+
         def load_configuration(file, user_provided, should_log)
           configuration = YAML.load_file(file)
           logger.debug { "Configuration from #{file}: #{configuration}" } if should_log
diff --git a/rakelib/dependency_cache_task.rb b/rakelib/dependency_cache_task.rb
@@ -21,6 +21,7 @@
 require 'java_buildpack/util/configuration_utils'
 require 'java_buildpack/util/cache/download_cache'
 require 'java_buildpack/util/snake_case'
+require 'monitor'
 require 'rake/tasklib'
 require 'rakelib/package'
 require 'pathname'
@@ -38,8 +39,9 @@ def initialize
 
       @default_repository_root = default_repository_root
       @cache                   = cache
+      @monitor                 = Monitor.new
 
-      configurations = component_ids.map { |component_id| configurations(configuration(component_id)) }.flatten
+      configurations = component_ids.map { |component_id| component_configuration(component_id) }.flatten
       uris(configurations).each { |uri| multitask PACKAGE_NAME => [cache_task(uri)] }
     end
 
@@ -100,7 +102,7 @@ def cache
 
     def cache_task(uri)
       task uri do |t|
-        rake_output_message "Caching #{t.name}"
+        @monitor.synchronize { rake_output_message "Caching #{t.name}" }
         cache.get(t.name) {}
       end
 
@@ -115,18 +117,24 @@ def configuration(id)
       JavaBuildpack::Util::ConfigurationUtils.load id, false
     end
 
-    def configurations(configuration)
+    def configurations(component_id, configuration, sub_component_id = nil)
       configurations = []
 
       if repository_configuration?(configuration)
+        configuration['component_id'] = component_id
+        configuration['sub_component_id'] = sub_component_id if sub_component_id
         configurations << configuration
       else
-        configuration.values.each { |v| configurations << configurations(v) if v.is_a? Hash }
+        configuration.each { |k, v| configurations << configurations(component_id, v, k) if v.is_a? Hash }
       end
 
       configurations
     end
 
+    def component_configuration(component_id)
+      configurations(component_id, configuration(component_id))
+    end
+
     def default_repository_root
       configuration('repository')['default_repository_root'].chomp('/')
     end
@@ -150,7 +158,6 @@ def uris(configurations)
       configurations.each do |configuration|
         index_configuration(configuration).each do |index_configuration|
           multitask PACKAGE_NAME => [cache_task(index_configuration[:uri])]
-
           get_from_cache(configuration, index_configuration, uris)
         end
       end
@@ -162,6 +169,7 @@ def get_from_cache(configuration, index_configuration, uris)
       @cache.get(index_configuration[:uri]) do |f|
         index         = YAML.load f
         found_version = version(configuration, index)
+        pin_version(configuration, found_version.to_s) if ENV['PINNED'].to_b
 
         if found_version.nil?
           rake_output_message "Unable to resolve version '#{configuration['version']}' for platform " \
@@ -172,6 +180,25 @@ def get_from_cache(configuration, index_configuration, uris)
       end
     end
 
+    def pin_version(old_configuration, version)
+      component_id = old_configuration['component_id']
+      sub_component_id = old_configuration['sub_component_id']
+      rake_output_message "Pinning #{sub_component_id ? sub_component_id : component_id} version to #{version}"
+      configuration_to_update = JavaBuildpack::Util::ConfigurationUtils.load(component_id, true)
+      update_configuration(configuration_to_update, version, sub_component_id)
+      JavaBuildpack::Util::ConfigurationUtils.write(component_id, configuration_to_update, true)
+    end
+
+    def update_configuration(config, version, sub_component)
+      if sub_component.nil?
+        config['version'] = version
+      elsif config.key?(sub_component)
+        config[sub_component]['version'] = version
+      else
+        config.values.each { |v| update_configuration(v, version, sub_component) if v.is_a? Hash }
+      end
+    end
+
     def version(configuration, index)
       JavaBuildpack::Repository::VersionResolver.resolve(
         JavaBuildpack::Util::TokenizedVersion.new(configuration['version']), index.keys)
diff --git a/spec/java_buildpack/util/configuration_utils_spec.rb b/spec/java_buildpack/util/configuration_utils_spec.rb
@@ -16,6 +16,7 @@
 
 require 'java_buildpack/util'
 require 'java_buildpack/util/configuration_utils'
+require 'fileutils'
 require 'logging_helper'
 require 'pathname'
 require 'spec_helper'
@@ -24,24 +25,32 @@
 describe JavaBuildpack::Util::ConfigurationUtils do
   include_context 'logging_helper'
 
+  let(:test_data) do
+    { 'foo'     => { 'one' => '1', 'two' => 2 },
+      'bar'     => { 'alpha' => { 'one' => 'cat', 'two' => 'dog' } },
+      'version' => '1.7.1' }
+  end
+
   it 'not load absent configuration file' do
     allow_any_instance_of(Pathname).to receive(:exist?).and_return(false)
     expect(described_class.load('test')).to eq({})
   end
 
+  it 'write configuration file' do
+    described_class.write('test', test_data)
+    expect(described_class.load('test')).to eq(test_data)
+    Pathname.new(File.expand_path('../../../config/test.yml', File.dirname(__FILE__))).delete
+  end
+
   context do
 
     before do
       allow_any_instance_of(Pathname).to receive(:exist?).and_return(true)
-      allow(YAML).to receive(:load_file).and_return('foo' => { 'one' => '1', 'two' => 2 },
-                                                    'bar' => { 'alpha' => { 'one' => 'cat', 'two' => 'dog' } },
-                                                    'version' => '1.7.1')
+      allow(YAML).to receive(:load_file).and_return(test_data)
     end
 
     it 'load configuration file' do
-      expect(described_class.load('test')).to eq('foo' => { 'one' => '1', 'two' => 2 },
-                                                 'bar' => { 'alpha' => { 'one' => 'cat', 'two' => 'dog' } },
-                                                 'version' => '1.7.1')
+      expect(described_class.load('test')).to eq(test_data)
     end
 
     context do
