diff --git a/migration/src/main/resources/org/dependencytrack/migration/V202608241029__osv_multi_source_config.sql b/migration/src/main/resources/org/dependencytrack/migration/V202608241029__osv_multi_source_config.sql new file mode 100644 index 0000000000..0a1a16726b --- /dev/null +++ b/migration/src/main/resources/org/dependencytrack/migration/V202608241029__osv_multi_source_config.sql @@ -0,0 +1,19 @@ +-- Convert legacy OSV config to multi-source config +-- example: {"enabled": true, ...} -> {"sources": [{"name":"default", "enabled": true, ...}]}. +UPDATE "EXTENSION_RUNTIME_CONFIG" + SET "CONFIG" = jsonb_build_object( + 'sources', + jsonb_build_array( "CONFIG" || jsonb_build_object('name', 'default') ) + ), + "UPDATED_AT" = now() +WHERE "EXTENSION_POINT" = 'vuln-data-source' + AND "EXTENSION" = 'osv' + AND NOT ("CONFIG" ? 'sources'); + +-- Watermarks are namespaced by source name. Move existing watermarks to the default source. +UPDATE "EXTENSION_KV_STORE" + SET "KEY" = 'watermark/default/' || replace("KEY", 'watermark/', '') +WHERE "EXTENSION_POINT" = 'vuln-data-source' + AND "EXTENSION" = 'osv' + AND "KEY" LIKE 'watermark/%' + AND NOT "KEY" LIKE 'watermark/default/%'; \ No newline at end of file diff --git a/vuln-data-source/osv/src/main/java/org/dependencytrack/vulndatasource/osv/OsvCompositeVulnDataSource.java b/vuln-data-source/osv/src/main/java/org/dependencytrack/vulndatasource/osv/OsvCompositeVulnDataSource.java new file mode 100644 index 0000000000..1d2d6992ca --- /dev/null +++ b/vuln-data-source/osv/src/main/java/org/dependencytrack/vulndatasource/osv/OsvCompositeVulnDataSource.java @@ -0,0 +1,106 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) OWASP Foundation. All Rights Reserved. + */ +package org.dependencytrack.vulndatasource.osv; + +import org.cyclonedx.proto.v1_7.Bom; +import org.dependencytrack.vulndatasource.api.VulnDataSource; +import org.jspecify.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; + +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; + +import static java.util.Objects.requireNonNull; + +/** + * @since 5.0.0 + */ +final class OsvCompositeVulnDataSource implements VulnDataSource { + + private static final Logger LOGGER = LoggerFactory.getLogger(OsvCompositeVulnDataSource.class); + private final List dataSources; + private @Nullable OsvVulnDataSource currentDataSource; + private int currentDataSourceIndex; + + /** + * Tracks the originating data source for a Bom instance so markProcessed can be + * delegated to the producer even if currentDataSource has moved on. + */ + private final Map originMap = Collections.synchronizedMap(new IdentityHashMap<>()); + + OsvCompositeVulnDataSource(final List dataSources) { + this.dataSources = requireNonNull(dataSources, "dataSources must not be null"); + } + + @Override + public boolean hasNext() { + while (currentDataSourceIndex < dataSources.size()) { + if (dataSources.get(currentDataSourceIndex).hasNext()) { + return true; + } + currentDataSourceIndex++; + } + return false; + } + + @Override + public Bom next() { + if (currentDataSourceIndex >= dataSources.size()) { + throw new NoSuchElementException(); + } + currentDataSource = dataSources.get(currentDataSourceIndex); + try (final var _ = MDC.putCloseable("osvSource", currentDataSource.getDataSourceName())) { + final Bom bom = currentDataSource.next(); + originMap.put(bom, currentDataSource); + return bom; + } + } + + @Override + public void markProcessed(final Bom bom) { + final var origin = originMap.remove(bom); + final var target = origin != null ? origin : currentDataSource; + if (target == null) { + throw new IllegalStateException("No data source available to mark processed"); + } + try (final var _ = MDC.putCloseable("osvSource", target.getDataSourceName())) { + target.markProcessed(bom); + } + } + + @Override + public void close() { + for (final var dataSource : dataSources) { + try { + dataSource.close(); + } catch (final Exception e) { + LOGGER.warn("Failed to close data source: {}", dataSource.getDataSourceName(), e); + } + } + } + + List getDataSources() { + return dataSources; + } +} diff --git a/vuln-data-source/osv/src/main/java/org/dependencytrack/vulndatasource/osv/OsvVulnDataSource.java b/vuln-data-source/osv/src/main/java/org/dependencytrack/vulndatasource/osv/OsvVulnDataSource.java index d892ab3d55..d0778ca1e3 100644 --- a/vuln-data-source/osv/src/main/java/org/dependencytrack/vulndatasource/osv/OsvVulnDataSource.java +++ b/vuln-data-source/osv/src/main/java/org/dependencytrack/vulndatasource/osv/OsvVulnDataSource.java @@ -62,6 +62,7 @@ final class OsvVulnDataSource implements VulnDataSource { private static final Logger LOGGER = LoggerFactory.getLogger(OsvVulnDataSource.class); private static final int MAX_INCREMENTAL_ADVISORY_DOWNLOADS = 250; + private final String dataSourceName; private final @Nullable WatermarkManager watermarkManager; private final ObjectMapper objectMapper; private final String dataUrl; @@ -78,12 +79,14 @@ final class OsvVulnDataSource implements VulnDataSource { private final boolean isAliasSyncEnabled; OsvVulnDataSource( + final String dataSourceName, final @Nullable WatermarkManager watermarkManager, final ObjectMapper objectMapper, final String dataUrl, final Collection ecosystems, final HttpClient httpClient, final boolean isAliasSyncEnabled) { + this.dataSourceName = requireNonNull(dataSourceName, "dataSourceName must not be null"); this.watermarkManager = watermarkManager; this.objectMapper = objectMapper; this.dataUrl = dataUrl; @@ -213,8 +216,9 @@ private void logCurrentEcosystemSummary() { } LOGGER.info( - "Finished ecosystem {}: processed {} advisories", + "Finished ecosystem {} of data source {}: processed {} advisories", currentEcosystem, + dataSourceName, currentEcosystemAdvisoriesProcessed); } @@ -223,7 +227,7 @@ private void openNextEcosystem() { currentEcosystemAdvisoriesProcessed = 0; currentAdvisorySource = openAdvisorySource(currentEcosystem); - LOGGER.info("Processing ecosystem {}", currentEcosystem); + LOGGER.info("Processing ecosystem {} of data source {}", currentEcosystem, dataSourceName); } private @Nullable OsvAdvisorySource openAdvisorySource(String ecosystem) { @@ -376,4 +380,8 @@ private Set getModifiedAdvisoryIds(String ecosystem, Instant watermark) WatermarkManager getWatermarkManager() { return watermarkManager; } + + String getDataSourceName() { + return dataSourceName; + } } diff --git a/vuln-data-source/osv/src/main/java/org/dependencytrack/vulndatasource/osv/OsvVulnDataSourceFactory.java b/vuln-data-source/osv/src/main/java/org/dependencytrack/vulndatasource/osv/OsvVulnDataSourceFactory.java index cc7fe705e9..fadace69a7 100644 --- a/vuln-data-source/osv/src/main/java/org/dependencytrack/vulndatasource/osv/OsvVulnDataSourceFactory.java +++ b/vuln-data-source/osv/src/main/java/org/dependencytrack/vulndatasource/osv/OsvVulnDataSourceFactory.java @@ -32,6 +32,9 @@ import java.net.URI; import java.net.http.HttpClient; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; import java.util.Set; import static java.util.Objects.requireNonNull; @@ -41,6 +44,7 @@ */ final class OsvVulnDataSourceFactory implements VulnDataSourceFactory, RuntimeConfigurable { + private static final String DEFAULT_SOURCE_NAME = "default"; private @Nullable ConfigRegistry configRegistry; private @Nullable KeyValueStore kvStore; private @Nullable ObjectMapper objectMapper; @@ -76,22 +80,34 @@ public void init(ExtensionContext context) { @Override public RuntimeConfigSpec runtimeConfigSpec() { - final var defaultConfig = new OsvVulnDataSourceConfigV1() + final var defaultSource = new OsvSourceConfigV1() + .withName(DEFAULT_SOURCE_NAME) .withIncrementalMirroringEnabled(true) .withEnabled(false) .withAliasSyncEnabled(false) .withDataUrl(URI.create("https://storage.googleapis.com/osv-vulnerabilities")) .withEcosystems(Set.of("Go", "Maven", "npm", "NuGet", "PyPI")); - return RuntimeConfigSpec.of(defaultConfig, config -> { - if (!config.isEnabled()) { - return; - } - if (config.getDataUrl() == null) { - throw new InvalidRuntimeConfigException("No data URL provided"); - } - if (config.getEcosystems() == null || config.getEcosystems().isEmpty()) { - throw new InvalidRuntimeConfigException("At least one ecosystem must be specified"); + final var defaultConfig = new OsvVulnDataSourceConfigV1().withFeeds(new LinkedHashSet<>(Set.of(defaultSource))); + + return RuntimeConfigSpec.of(defaultConfig, (OsvVulnDataSourceConfigV1 config) -> { + final Set seenNames = new LinkedHashSet<>(); + for (final var feed : config.getFeeds()) { + if (feed.getName() == null || feed.getName().isBlank()) { + throw new InvalidRuntimeConfigException("No data feed name provided"); + } + if (!seenNames.add(feed.getName())) { + throw new InvalidRuntimeConfigException("Duplicate data feed name provided: " + feed.getName()); + } + if (!feed.isEnabled()) { + continue; + } + if (feed.getDataUrl() == null) { + throw new InvalidRuntimeConfigException("No data URL provided"); + } + if (feed.getEcosystems() == null || feed.getEcosystems().isEmpty()) { + throw new InvalidRuntimeConfigException("At least one ecosystem must be specified"); + } } }); } @@ -99,7 +115,8 @@ public RuntimeConfigSpec runtimeConfigSpec() { @Override public boolean isDataSourceEnabled() { requireNonNull(configRegistry, "configRegistry must not be null"); - return configRegistry.getRuntimeConfig(OsvVulnDataSourceConfigV1.class).isEnabled(); + return !enabledFeeds(configRegistry.getRuntimeConfig(OsvVulnDataSourceConfigV1.class)) + .isEmpty(); } @Override @@ -109,20 +126,31 @@ public VulnDataSource create() { requireNonNull(objectMapper, "objectMapper must not be null"); requireNonNull(httpClient, "httpClient must not be null"); - final var config = configRegistry.getRuntimeConfig(OsvVulnDataSourceConfigV1.class); - if (!config.isEnabled()) { + final List feeds = + enabledFeeds(configRegistry.getRuntimeConfig(OsvVulnDataSourceConfigV1.class)); + if (feeds.isEmpty()) { throw new IllegalStateException("Vulnerability data source is disabled and cannot be created"); } - final WatermarkManager watermarkManager = - config.isIncrementalMirroringEnabled() ? new WatermarkManager(config.getEcosystems(), kvStore) : null; + final var dataSources = new ArrayList(feeds.size()); + for (final OsvSourceConfigV1 feed : feeds) { + final WatermarkManager watermarkManager = feed.isIncrementalMirroringEnabled() + ? new WatermarkManager(feed.getName(), feed.getEcosystems(), kvStore) + : null; + + dataSources.add(new OsvVulnDataSource( + feed.getName(), + watermarkManager, + objectMapper, + feed.getDataUrl().toString(), + feed.getEcosystems(), + httpClient, + feed.getAliasSyncEnabled())); + } + return new OsvCompositeVulnDataSource(dataSources); + } - return new OsvVulnDataSource( - watermarkManager, - objectMapper, - config.getDataUrl().toString(), - config.getEcosystems(), - httpClient, - config.getAliasSyncEnabled()); + private List enabledFeeds(final OsvVulnDataSourceConfigV1 config) { + return config.getFeeds().stream().filter(OsvSourceConfigV1::isEnabled).toList(); } } diff --git a/vuln-data-source/osv/src/main/java/org/dependencytrack/vulndatasource/osv/WatermarkManager.java b/vuln-data-source/osv/src/main/java/org/dependencytrack/vulndatasource/osv/WatermarkManager.java index 613bfc14f7..0acd43fd1d 100644 --- a/vuln-data-source/osv/src/main/java/org/dependencytrack/vulndatasource/osv/WatermarkManager.java +++ b/vuln-data-source/osv/src/main/java/org/dependencytrack/vulndatasource/osv/WatermarkManager.java @@ -39,8 +39,8 @@ final class WatermarkManager { private final Map pendingRecordByEcosystem; private final Map committedRecordByEcosystem; - WatermarkManager(final Collection ecosystems, final KeyValueStore kvStore) { - final var watermarkStore = new WatermarkStore(kvStore); + WatermarkManager(final String sourceName, final Collection ecosystems, final KeyValueStore kvStore) { + final var watermarkStore = new WatermarkStore(sourceName, kvStore); final Map recordByEcosystem = watermarkStore.getForEcosystems(ecosystems); this.store = watermarkStore; diff --git a/vuln-data-source/osv/src/main/java/org/dependencytrack/vulndatasource/osv/WatermarkStore.java b/vuln-data-source/osv/src/main/java/org/dependencytrack/vulndatasource/osv/WatermarkStore.java index 6d3ec98f7d..b53560418d 100644 --- a/vuln-data-source/osv/src/main/java/org/dependencytrack/vulndatasource/osv/WatermarkStore.java +++ b/vuln-data-source/osv/src/main/java/org/dependencytrack/vulndatasource/osv/WatermarkStore.java @@ -40,15 +40,17 @@ final class WatermarkStore { private static final Logger LOGGER = LoggerFactory.getLogger(WatermarkStore.class); + private final String sourceName; private final KeyValueStore kvStore; - WatermarkStore(final KeyValueStore kvStore) { + WatermarkStore(final String sourceName, final KeyValueStore kvStore) { + this.sourceName = requireNonNull(sourceName, "OSV sourceName must not be null"); this.kvStore = kvStore; } Map getForEcosystems(final Collection ecosystems) { final Map ecosystemByKey = - ecosystems.stream().collect(Collectors.toMap(WatermarkStore::getKey, Function.identity())); + ecosystems.stream().collect(Collectors.toMap(this::getKey, Function.identity())); final Map kvEntryByKey = kvStore.getMany(ecosystemByKey.keySet()); if (kvEntryByKey.isEmpty()) { @@ -92,7 +94,7 @@ WatermarkRecord save(final WatermarkRecord watermark) { }; } - private static String getKey(final String ecosystem) { - return "watermark/" + ecosystem; + private String getKey(final String ecosystem) { + return "watermark/" + sourceName + "/" + ecosystem; } } diff --git a/vuln-data-source/osv/src/main/resources/org/dependencytrack/vulndatasource/osv/osv-vuln-data-source-config-v1.schema.json b/vuln-data-source/osv/src/main/resources/org/dependencytrack/vulndatasource/osv/osv-vuln-data-source-config-v1.schema.json index 72d4cb4689..41abeb26a3 100644 --- a/vuln-data-source/osv/src/main/resources/org/dependencytrack/vulndatasource/osv/osv-vuln-data-source-config-v1.schema.json +++ b/vuln-data-source/osv/src/main/resources/org/dependencytrack/vulndatasource/osv/osv-vuln-data-source-config-v1.schema.json @@ -6,44 +6,66 @@ "org.dependencytrack.plugin.api.config.RuntimeConfig" ], "properties": { - "enabled": { - "type": "boolean", - "title": "Enabled", - "description": "Whether the OSV data source should be enabled.", - "existingJavaType": "boolean" - }, - "aliasSyncEnabled": { - "type": "boolean", - "title": "Alias Synchronization Enabled", - "description": "Whether to include alias information in vulnerability data." - }, - "incrementalMirroringEnabled": { - "type": "boolean", - "title": "Incremental Mirroring Enabled", - "description": "Whether incremental mirroring is enabled. When enabled (default), watermark logic will be used to only mirror new or updated vulnerabilities. When disabled, full mirroring will be performed on each run.", - "default": true, - "existingJavaType": "boolean" - }, - "dataUrl": { - "type": "string", - "title": "Data URL", - "description": "The URL to download OSV data from. \n**Must** be compatible with [OSV's data dumps](https://google.github.io/osv.dev/data/#data-dumps).", - "minLength": 1, - "format": "uri" - }, - "ecosystems": { + "feeds": { "type": "array", - "title": "Ecosystems", - "description": "The ecosystems to mirror vulnerability data for. \nA list of available ecosystems can be found in the [OSV documentation](https://ossf.github.io/osv-schema/#defined-ecosystems).", + "title": "Feeds", + "description": "The OSV feeds to mirror vulnerability data from. \nEach feed is configured, enabled and watermarked independently.", "uniqueItems": true, "items": { - "type": "string", - "title": "Ecosystem", - "minLength": 1 + "type": "object", + "title": "Source", + "javaType": "org.dependencytrack.vulndatasource.osv.OsvSourceConfigV1", + "properties": { + "name": { + "type": "string", + "title": "Name", + "description": "The name of the OSV data feed. Used in watermark keys and other identifiers; must be 1–255 characters and contain only letters, digits, space, dot (.), underscore (_) or hyphen (-).", + "minLength": 1, + "maxLength": 255, + "pattern": "^[A-Za-z0-9 ._-]+$" + }, + "enabled": { + "type": "boolean", + "title": "Enabled", + "description": "Whether the OSV data source should be enabled.", + "existingJavaType": "boolean" + }, + "aliasSyncEnabled": { + "type": "boolean", + "title": "Alias Synchronization Enabled", + "description": "Whether to include alias information in vulnerability data." + }, + "incrementalMirroringEnabled": { + "type": "boolean", + "title": "Incremental Mirroring Enabled", + "description": "Whether incremental mirroring is enabled. When enabled (default), watermark logic will be used to only mirror new or updated vulnerabilities. When disabled, full mirroring will be performed on each run.", + "default": true, + "existingJavaType": "boolean" + }, + "dataUrl": { + "type": "string", + "title": "Data URL", + "description": "The URL to download OSV data from. \n**Must** be compatible with [OSV's data dumps](https://google.github.io/osv.dev/data/#data-dumps).", + "minLength": 1, + "format": "uri" + }, + "ecosystems": { + "type": "array", + "title": "Ecosystems", + "description": "The ecosystems to mirror vulnerability data for. \nA list of available ecosystems can be found in the [OSV documentation](https://ossf.github.io/osv-schema/#defined-ecosystems).", + "uniqueItems": true, + "items": { + "type": "string", + "title": "Ecosystem", + "minLength": 1 + } + } + }, + "required": [ + "name", + "enabled" + ] } } - }, - "required": [ - "enabled" - ] -} \ No newline at end of file + } +} diff --git a/vuln-data-source/osv/src/test/java/org/dependencytrack/vulndatasource/osv/OsvCompositeVulnDataSourceTest.java b/vuln-data-source/osv/src/test/java/org/dependencytrack/vulndatasource/osv/OsvCompositeVulnDataSourceTest.java new file mode 100644 index 0000000000..778c73dcea --- /dev/null +++ b/vuln-data-source/osv/src/test/java/org/dependencytrack/vulndatasource/osv/OsvCompositeVulnDataSourceTest.java @@ -0,0 +1,79 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) OWASP Foundation. All Rights Reserved. + */ +package org.dependencytrack.vulndatasource.osv; + +import org.cyclonedx.proto.v1_7.Bom; +import org.cyclonedx.proto.v1_7.Vulnerability; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +class OsvCompositeVulnDataSourceTest { + + @Test + void shouldMirrorAllSources() { + final Bom BOV_A = Bom.newBuilder() + .addVulnerabilities(Vulnerability.newBuilder().setId("CVE-A").build()) + .build(); + + final Bom BOV_B = Bom.newBuilder() + .addVulnerabilities(Vulnerability.newBuilder().setId("CVE-B").build()) + .build(); + + final var dataSourceA = mock(OsvVulnDataSource.class); + doReturn(true, false).when(dataSourceA).hasNext(); + doReturn(BOV_A).when(dataSourceA).next(); + + final var dataSourceB = mock(OsvVulnDataSource.class); + doReturn(true, false).when(dataSourceB).hasNext(); + doReturn(BOV_B).when(dataSourceB).next(); + + final var bovs = new ArrayList(); + + try (var dataSource = new OsvCompositeVulnDataSource(List.of(dataSourceA, dataSourceB))) { + while (dataSource.hasNext()) { + final var bov = dataSource.next(); + bovs.add(bov); + dataSource.markProcessed(bov); + } + } + + assertThat(bovs).containsExactly(BOV_A, BOV_B); + verify(dataSourceA).markProcessed(BOV_A); + verify(dataSourceB).markProcessed(BOV_B); + verify(dataSourceA).close(); + verify(dataSourceB).close(); + } + + @Test + void markProcessedShouldThrowWhenNothingToProcess() { + final var compositeDataSource = new OsvCompositeVulnDataSource(List.of(mock(OsvVulnDataSource.class))); + assertThatExceptionOfType(IllegalStateException.class) + .isThrownBy( + () -> compositeDataSource.markProcessed(Bom.newBuilder().build())) + .withMessage("No data source available to mark processed"); + } +} diff --git a/vuln-data-source/osv/src/test/java/org/dependencytrack/vulndatasource/osv/OsvVulnDataSourceFactoryTest.java b/vuln-data-source/osv/src/test/java/org/dependencytrack/vulndatasource/osv/OsvVulnDataSourceFactoryTest.java index de055996e9..0f111e0ccc 100644 --- a/vuln-data-source/osv/src/test/java/org/dependencytrack/vulndatasource/osv/OsvVulnDataSourceFactoryTest.java +++ b/vuln-data-source/osv/src/test/java/org/dependencytrack/vulndatasource/osv/OsvVulnDataSourceFactoryTest.java @@ -18,6 +18,7 @@ */ package org.dependencytrack.vulndatasource.osv; +import org.dependencytrack.plugin.api.config.ConfigRegistry; import org.dependencytrack.plugin.testing.AbstractExtensionFactoryTest; import org.dependencytrack.plugin.testing.ExtensionContextBuilder; import org.dependencytrack.plugin.testing.MockConfigRegistry; @@ -27,6 +28,9 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import java.net.URI; +import java.util.Set; + import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; @@ -52,82 +56,116 @@ void priorityShouldBeZero() { assertThat(factory.priority()).isEqualTo(100); } + @Test + void defaultConfigShouldContainSingleDefaultSource() { + final OsvVulnDataSourceConfigV1 config = defaultConfig(); + assertThat(config.getFeeds()).satisfiesExactly(feed -> { + assertThat(feed.getName()).isEqualTo("default"); + assertThat(feed.isEnabled()).isFalse(); + assertThat(feed.getAliasSyncEnabled()).isFalse(); + assertThat(feed.isIncrementalMirroringEnabled()).isTrue(); + assertThat(feed.getDataUrl().toString()).isEqualTo("https://storage.googleapis.com/osv-vulnerabilities"); + assertThat(feed.getEcosystems()).containsExactlyInAnyOrder("npm", "PyPI", "NuGet", "Maven", "Go"); + }); + } + @ParameterizedTest @ValueSource(booleans = {true, false}) void isDataSourceEnabledShouldReturnTrueWhenEnabledAndFalseOtherwise(final boolean isEnabled) { - final var config = - (OsvVulnDataSourceConfigV1) factory.runtimeConfigSpec().defaultConfig(); - config.setEnabled(isEnabled); - - factory.init(new ExtensionContextBuilder() - .withConfigRegistry(new MockConfigRegistry(factory.runtimeConfigSpec(), config)) - .build()); + final OsvVulnDataSourceConfigV1 config = defaultConfig(); + config.getFeeds().forEach(feed -> feed.setEnabled(isEnabled)); + initFactory(config); assertThat(factory.isDataSourceEnabled()).isEqualTo(isEnabled); } @Test void createShouldReturnNullWhenDisabled() { - final var config = - (OsvVulnDataSourceConfigV1) factory.runtimeConfigSpec().defaultConfig(); - config.setEnabled(false); - - final var configRegistry = new MockConfigRegistry(factory.runtimeConfigSpec(), config); - - factory.init( - new ExtensionContextBuilder().withConfigRegistry(configRegistry).build()); - + final OsvVulnDataSourceConfigV1 config = defaultConfig(); + config.getFeeds().forEach(feed -> feed.setEnabled(false)); + initFactory(config); assertThatExceptionOfType(IllegalStateException.class).isThrownBy(factory::create); } @Test void createShouldReturnDataSource() { - final var config = - (OsvVulnDataSourceConfigV1) factory.runtimeConfigSpec().defaultConfig(); - config.setEnabled(true); - - final var configRegistry = new MockConfigRegistry(factory.runtimeConfigSpec(), config); - - factory.init( - new ExtensionContextBuilder().withConfigRegistry(configRegistry).build()); + final OsvVulnDataSourceConfigV1 config = defaultConfig(); + config.getFeeds().forEach(feed -> feed.setEnabled(true)); + initFactory(config); + final VulnDataSource dataSource = factory.create(); + assertThat(dataSource).isNotNull(); + assertThat(((OsvCompositeVulnDataSource) dataSource).getDataSources()).hasSize(1); + dataSource.close(); + } + @Test + void createShouldReturnDataSourcePerEnabledSource() { + final OsvVulnDataSourceConfigV1 config = defaultConfig(); + config.getFeeds().forEach(feed -> feed.setEnabled(true)); + config.getFeeds() + .add(new OsvSourceConfigV1() + .withName("Chainguard") + .withEnabled(true) + .withIncrementalMirroringEnabled(true) + .withAliasSyncEnabled(false) + .withDataUrl(URI.create("https://chainguard.com/osv-vulnerabilities")) + .withEcosystems(Set.of("Maven"))); + config.getFeeds() + .add(new OsvSourceConfigV1() + .withName("Red Hat") + .withEnabled(false) + .withIncrementalMirroringEnabled(false) + .withAliasSyncEnabled(false) + .withDataUrl(URI.create("https://redhat.com/osv-vulnerabilities")) + .withEcosystems(Set.of("Go"))); + initFactory(config); final VulnDataSource dataSource = factory.create(); assertThat(dataSource).isNotNull(); + assertThat(((OsvCompositeVulnDataSource) dataSource).getDataSources()).hasSize(2); dataSource.close(); } @Test void createWhenIncrementalMirroringDisabledShouldCreateDataSourceWithNullWatermarkManager() { - final var config = - (OsvVulnDataSourceConfigV1) factory.runtimeConfigSpec().defaultConfig(); - config.setEnabled(true); - config.setIncrementalMirroringEnabled(false); - - final var configRegistry = new MockConfigRegistry(factory.runtimeConfigSpec(), config); - - factory.init( - new ExtensionContextBuilder().withConfigRegistry(configRegistry).build()); - + final OsvVulnDataSourceConfigV1 config = defaultConfig(); + config.getFeeds().forEach(feed -> { + feed.setEnabled(true); + feed.setIncrementalMirroringEnabled(false); + }); + initFactory(config); try (VulnDataSource dataSource = factory.create()) { - assertThat(dataSource).isNotNull(); - assertThat(((OsvVulnDataSource) dataSource).getWatermarkManager()).isNull(); + assertThat(((OsvCompositeVulnDataSource) dataSource).getDataSources()) + .singleElement() + .satisfies( + source -> assertThat(source.getWatermarkManager()).isNull()); } } @Test void createWhenIncrementalMirroringEnabledShouldCreateDataSourceWithWatermarkManager() { - final var config = - (OsvVulnDataSourceConfigV1) factory.runtimeConfigSpec().defaultConfig(); - config.setEnabled(true); - config.setIncrementalMirroringEnabled(true); + final OsvVulnDataSourceConfigV1 config = defaultConfig(); + config.getFeeds().forEach(feed -> { + feed.setEnabled(true); + feed.setIncrementalMirroringEnabled(true); + }); + initFactory(config); + try (VulnDataSource dataSource = factory.create()) { + assertThat(((OsvCompositeVulnDataSource) dataSource).getDataSources()) + .singleElement() + .satisfies( + source -> assertThat(source.getWatermarkManager()).isNotNull()); + } + } + + private OsvVulnDataSourceConfigV1 defaultConfig() { + return (OsvVulnDataSourceConfigV1) factory.runtimeConfigSpec().defaultConfig(); + } - final var configRegistry = new MockConfigRegistry(factory.runtimeConfigSpec(), config); + private void initFactory(final OsvVulnDataSourceConfigV1 config) { + initFactory(new MockConfigRegistry(factory.runtimeConfigSpec(), config)); + } + private void initFactory(final ConfigRegistry configRegistry) { factory.init( new ExtensionContextBuilder().withConfigRegistry(configRegistry).build()); - - try (VulnDataSource dataSource = factory.create()) { - assertThat(dataSource).isNotNull(); - assertThat(((OsvVulnDataSource) dataSource).getWatermarkManager()).isNotNull(); - } } } diff --git a/vuln-data-source/osv/src/test/java/org/dependencytrack/vulndatasource/osv/OsvVulnDataSourceTest.java b/vuln-data-source/osv/src/test/java/org/dependencytrack/vulndatasource/osv/OsvVulnDataSourceTest.java index 4c9b9cf82f..95a5cabbc5 100644 --- a/vuln-data-source/osv/src/test/java/org/dependencytrack/vulndatasource/osv/OsvVulnDataSourceTest.java +++ b/vuln-data-source/osv/src/test/java/org/dependencytrack/vulndatasource/osv/OsvVulnDataSourceTest.java @@ -73,6 +73,7 @@ void beforeEach() { objectMapper = new ObjectMapper().registerModule(new JavaTimeModule()); vulnDataSource = new OsvVulnDataSource( + "default", watermarkManagerMock, objectMapper, "http://localhost", @@ -172,6 +173,7 @@ void shouldIterateAdvisoriesFromFullArchive(WireMockRuntimeInfo wmRuntimeInfo) t .withHeader("Content-Type", "application/zip"))); OsvVulnDataSource dataSource = new OsvVulnDataSource( + "default", watermarkManagerMock, objectMapper, wmRuntimeInfo.getHttpBaseUrl(), @@ -241,6 +243,7 @@ void shouldPercentEncodeSpacesInEcosystemNameForFullArchive(WireMockRuntimeInfo .withBody(zipBytes.toByteArray()))); try (var dataSource = new OsvVulnDataSource( + "default", null, objectMapper, wmRuntimeInfo.getHttpBaseUrl(), @@ -284,6 +287,7 @@ void shouldSkipDirectoryAndNonJsonEntriesInFullArchive(WireMockRuntimeInfo wmRun final var ids = new ArrayList(); try (var dataSource = new OsvVulnDataSource( + "default", null, objectMapper, wmRuntimeInfo.getHttpBaseUrl(), @@ -322,6 +326,7 @@ void nullWatermarkManagerPerformsFullDownload(WireMockRuntimeInfo wmRuntimeInfo) .withHeader("Content-Type", "application/zip"))); try (var dataSource = new OsvVulnDataSource( + "default", null, objectMapper, wmRuntimeInfo.getHttpBaseUrl(), @@ -364,6 +369,7 @@ void watermarkManagerReturnsNullPerformsFullDownload(WireMockRuntimeInfo wmRunti .withHeader("Content-Type", "application/zip"))); try (var dataSource = new OsvVulnDataSource( + "default", watermarkManagerMock, objectMapper, wmRuntimeInfo.getHttpBaseUrl(), @@ -405,6 +411,7 @@ void watermarkManagerReturnsInstantPerformsIncrementalDownload(WireMockRuntimeIn .withBody(advisoryJson))); try (var dataSource = new OsvVulnDataSource( + "default", watermarkManagerMock, objectMapper, wmRuntimeInfo.getHttpBaseUrl(), @@ -447,6 +454,7 @@ void shouldDownloadIncrementalAdvisoriesLazily(WireMockRuntimeInfo wmRuntimeInfo } try (var dataSource = new OsvVulnDataSource( + "default", watermarkManagerMock, objectMapper, wmRuntimeInfo.getHttpBaseUrl(), @@ -502,6 +510,7 @@ void shouldFallBackToFullDownloadWhenIncrementalThresholdExceeded(WireMockRuntim .withBody(zipBytes.toByteArray()))); try (var dataSource = new OsvVulnDataSource( + "default", watermarkManagerMock, objectMapper, wmRuntimeInfo.getHttpBaseUrl(), @@ -560,6 +569,7 @@ void shouldOnlyProcessChangedAdvisoriesFromFullArchive(WireMockRuntimeInfo wmRun final var ids = new ArrayList(); try (var dataSource = new OsvVulnDataSource( + "default", watermarkManagerMock, objectMapper, wmRuntimeInfo.getHttpBaseUrl(), @@ -622,6 +632,7 @@ void shouldContinueWithRemainingEcosystemsWhenEcosystemHasNoChanges(WireMockRunt final var ids = new ArrayList(); try (var dataSource = new OsvVulnDataSource( + "default", watermarkManagerMock, objectMapper, wmRuntimeInfo.getHttpBaseUrl(), @@ -669,6 +680,7 @@ void shouldContinueWithRemainingEcosystemsWhenFirstEcosystemHasNoChanges(WireMoc final var ids = new ArrayList(); try (var dataSource = new OsvVulnDataSource( + "default", watermarkManagerMock, objectMapper, wmRuntimeInfo.getHttpBaseUrl(), @@ -706,6 +718,7 @@ void shouldPercentEncodeSpacesInEcosystemNameForIncrementalAdvisories(WireMockRu } try (var dataSource = new OsvVulnDataSource( + "default", watermarkManagerMock, objectMapper, wmRuntimeInfo.getHttpBaseUrl(), diff --git a/vuln-data-source/osv/src/test/java/org/dependencytrack/vulndatasource/osv/WatermarkManagerTest.java b/vuln-data-source/osv/src/test/java/org/dependencytrack/vulndatasource/osv/WatermarkManagerTest.java index 2d99f582b7..c0d6c0912a 100644 --- a/vuln-data-source/osv/src/test/java/org/dependencytrack/vulndatasource/osv/WatermarkManagerTest.java +++ b/vuln-data-source/osv/src/test/java/org/dependencytrack/vulndatasource/osv/WatermarkManagerTest.java @@ -36,13 +36,13 @@ class WatermarkManagerTest { void createShouldInitializeWatermarkWhenAvailable() { kvStore.putMany(Map.ofEntries( Map.entry( - "watermark/maven", + "watermark/default/maven", String.valueOf(Instant.ofEpochSecond(666).toEpochMilli())), Map.entry( - "watermark/npm", + "watermark/default/npm", String.valueOf(Instant.ofEpochSecond(555).toEpochMilli())))); - final var watermarkManager = new WatermarkManager(List.of("maven", "npm"), kvStore); + final var watermarkManager = new WatermarkManager("default", List.of("maven", "npm"), kvStore); assertThat(watermarkManager).isNotNull(); assertThat(watermarkManager.getWatermark("maven")).isEqualTo(Instant.ofEpochSecond(666)); assertThat(watermarkManager.getWatermark("npm")).isEqualTo(Instant.ofEpochSecond(555)); @@ -50,14 +50,14 @@ void createShouldInitializeWatermarkWhenAvailable() { @Test void createShouldNotInitializeWatermarkWhenNotAvailable() { - final var watermarkManager = new WatermarkManager(List.of("maven"), kvStore); + final var watermarkManager = new WatermarkManager("default", List.of("maven"), kvStore); assertThat(watermarkManager).isNotNull(); assertThat(watermarkManager.getWatermark("maven")).isNull(); } @Test void shouldAdvanceWatermarkWhenInitialWatermarkIsNull() { - final var watermarkManager = new WatermarkManager(List.of("maven"), kvStore); + final var watermarkManager = new WatermarkManager("default", List.of("maven"), kvStore); watermarkManager.maybeAdvance("maven", Instant.ofEpochSecond(666)); assertThat(watermarkManager.getWatermark("maven")).isNull(); @@ -68,9 +68,11 @@ void shouldAdvanceWatermarkWhenInitialWatermarkIsNull() { @Test void shouldAdvanceWatermarkWhenInitialWatermarkIsEarlier() { - kvStore.put("watermark/maven", String.valueOf(Instant.ofEpochSecond(666).toEpochMilli())); + kvStore.put( + "watermark/default/maven", + String.valueOf(Instant.ofEpochSecond(666).toEpochMilli())); - final var watermarkManager = new WatermarkManager(List.of("maven"), kvStore); + final var watermarkManager = new WatermarkManager("default", List.of("maven"), kvStore); watermarkManager.maybeAdvance("maven", Instant.ofEpochSecond(667)); assertThat(watermarkManager.getWatermark("maven")).isEqualTo(Instant.ofEpochSecond(666));