diff --git a/server/opensharing-server-core/src/main/java/io/opensharing/runtime/OpenSharing.java b/server/opensharing-server-core/src/main/java/io/opensharing/runtime/OpenSharing.java new file mode 100644 index 000000000..d699b200e --- /dev/null +++ b/server/opensharing-server-core/src/main/java/io/opensharing/runtime/OpenSharing.java @@ -0,0 +1,45 @@ +package io.opensharing.runtime; + +import io.opensharing.catalog.CatalogConnector; + +/** + * OpenSharing assembled from what its host supplies. The standalone server builds one from its + * settings. An embedding host, such as Unity Catalog, builds one directly, serves it through + * its own HTTP stack, and authenticates callers itself. + */ +public final class OpenSharing { + + private final CatalogConnector catalog; + + private OpenSharing(CatalogConnector catalog) { + this.catalog = catalog; + } + + public static Builder builder() { + return new Builder(); + } + + public CatalogConnector catalog() { + return catalog; + } + + public static final class Builder { + + private CatalogConnector catalog; + + private Builder() {} + + /** The host's catalog integration (required). */ + public Builder catalog(CatalogConnector catalog) { + this.catalog = catalog; + return this; + } + + public OpenSharing build() { + if (catalog == null) { + throw new IllegalStateException("OpenSharing requires a CatalogConnector from the host"); + } + return new OpenSharing(catalog); + } + } +} diff --git a/server/opensharing-server/pom.xml b/server/opensharing-server/pom.xml index 99c5ad1f7..f5f18da67 100644 --- a/server/opensharing-server/pom.xml +++ b/server/opensharing-server/pom.xml @@ -22,21 +22,8 @@ - org.springframework.boot - spring-boot-starter-web - - - org.springframework.boot - spring-boot-starter-data-jpa - - - org.springframework.boot - spring-boot-starter-validation - - - org.springframework.boot - spring-boot-configuration-processor - true + com.linecorp.armeria + armeria @@ -45,30 +32,14 @@ - org.springdoc - springdoc-openapi-starter-webmvc-ui - ${springdoc.version} - - - - com.h2database - h2 - runtime - - - org.postgresql - postgresql - runtime - - - com.mysql - mysql-connector-j + ch.qos.logback + logback-classic runtime - org.springframework.boot - spring-boot-starter-test + org.junit.jupiter + junit-jupiter test @@ -76,17 +47,44 @@ - org.springframework.boot - spring-boot-maven-plugin + org.apache.maven.plugins + maven-jar-plugin + + + + io.opensharing.OpenSharingServer + + + + + + + org.apache.maven.plugins + maven-shade-plugin - io.opensharing.OpenSharingServer - exec - true + true + exec + false + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + package - repackage + shade diff --git a/server/opensharing-server/src/main/java/io/opensharing/OpenSharingServer.java b/server/opensharing-server/src/main/java/io/opensharing/OpenSharingServer.java new file mode 100644 index 000000000..a6a413a9b --- /dev/null +++ b/server/opensharing-server/src/main/java/io/opensharing/OpenSharingServer.java @@ -0,0 +1,92 @@ +package io.opensharing; + +import com.linecorp.armeria.common.HttpResponse; +import com.linecorp.armeria.common.HttpStatus; +import com.linecorp.armeria.server.Server; +import io.opensharing.catalog.CatalogConnector; +import io.opensharing.catalog.local.LocalCatalogConnector; +import io.opensharing.catalog.local.LocalCatalogLoader; +import io.opensharing.config.ServerProperties; +import io.opensharing.config.ServerProperties.Property; +import io.opensharing.runtime.OpenSharing; +import java.util.Locale; + +/** + * Runnable standalone OpenSharing server. It builds {@link OpenSharing} from {@link + * ServerProperties} and listens for HTTP with Armeria. + */ +public final class OpenSharingServer implements AutoCloseable { + + private final Server server; + private final OpenSharing openSharing; + + private OpenSharingServer(Server server, OpenSharing openSharing) { + this.server = server; + this.openSharing = openSharing; + } + + /** + * Starts the server with the settings in {@code server.properties}, overridden by JVM system + * properties and environment variables, and blocks until the JVM shuts down. It takes no program + * arguments. + */ + public static void main(String[] args) throws InterruptedException { + if (args.length > 0) { + throw new IllegalArgumentException( + "program arguments are not supported; set server.properties keys as -D JVM options or" + + " environment variables"); + } + OpenSharingServer server = start(new ServerProperties()); + server.server.closeOnJvmShutdown(); + server.server.blockUntilShutdown(); + } + + /** + * Builds the server from {@code properties} and returns once it is listening. The catalog is + * loaded first, so an invalid catalog fails startup before the port is bound. + */ + public static OpenSharingServer start(ServerProperties properties) { + OpenSharing openSharing = OpenSharing.builder().catalog(catalog(properties)).build(); + Server server = + Server.builder() + .http(properties.getPort()) + // Armeria refuses to start without a service; routes nothing serves are not found. + .serviceUnder("/", (ctx, req) -> HttpResponse.of(HttpStatus.NOT_FOUND)) + .build(); + try { + server.start().join(); + } catch (RuntimeException e) { + server.close(); + throw e; + } + return new OpenSharingServer(server, openSharing); + } + + /** The OpenSharing instance built from the server's settings. */ + public OpenSharing openSharing() { + return openSharing; + } + + /** Stops the server and blocks until it has stopped. */ + @Override + public void close() { + server.stop().join(); + } + + /** + * The catalog named by {@code catalog.type}, matched ignoring case and surrounding spaces. Only + * the local catalog ships with this build; it reads {@code catalog.local.file}. + */ + static CatalogConnector catalog(ServerProperties properties) { + String type = properties.get(Property.CATALOG_TYPE); + return switch (type.trim().toLowerCase(Locale.ROOT)) { + case LocalCatalogConnector.NAME -> + new LocalCatalogConnector( + LocalCatalogLoader.load(properties.get(Property.LOCAL_CATALOG_FILE))); + default -> + throw new IllegalStateException( + "unknown catalog.type '" + type + "'; this build ships '" + + LocalCatalogConnector.NAME + "'"); + }; + } +} diff --git a/server/opensharing-server/src/main/java/io/opensharing/catalog/local/LocalCatalogLoader.java b/server/opensharing-server/src/main/java/io/opensharing/catalog/local/LocalCatalogLoader.java index 352a5a3d2..d3a62827a 100644 --- a/server/opensharing-server/src/main/java/io/opensharing/catalog/local/LocalCatalogLoader.java +++ b/server/opensharing-server/src/main/java/io/opensharing/catalog/local/LocalCatalogLoader.java @@ -6,10 +6,16 @@ import io.opensharing.exception.CatalogException; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; -/** Reads a {@link LocalCatalogFile} from YAML or JSON. */ +/** Reads a {@link LocalCatalogFile} from YAML or JSON, matching keys and enums ignoring case. */ public final class LocalCatalogLoader { + private static final String CLASSPATH = "classpath:"; + private static final String FILE = "file:"; + private static final ObjectMapper MAPPER = YAMLMapper.builder() .enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES) @@ -18,9 +24,23 @@ public final class LocalCatalogLoader { private LocalCatalogLoader() {} + /** + * Reads {@code location}: a {@code classpath:} resource, or a file path with or without {@code + * file:}. + */ + public static LocalCatalogFile load(String location) { + try (InputStream in = open(location)) { + return load(in, location); + } catch (IOException e) { + throw new CatalogException("failed to read local catalog file " + location, e); + } + } + + /** Parses {@code in}. {@code origin} only names the source in error messages. */ public static LocalCatalogFile load(InputStream in, String origin) { try { LocalCatalogFile file = MAPPER.readValue(in, LocalCatalogFile.class); + // A null YAML document, such as "~" or a lone "---", is an empty catalog. return file == null ? new LocalCatalogFile(null, null) : file; } catch (IOException e) { throw new CatalogException("failed to parse local catalog file " + origin, e); @@ -28,4 +48,25 @@ public static LocalCatalogFile load(InputStream in, String origin) { throw new CatalogException("invalid local catalog file " + origin + ": " + e.getMessage(), e); } } + + private static InputStream open(String location) throws IOException { + InputStream in; + if (location.startsWith(CLASSPATH)) { + // ClassLoader resources never start with "/", so "classpath:/x.yml" means "classpath:x.yml". + String resource = location.substring(CLASSPATH.length()).replaceFirst("^/", ""); + in = LocalCatalogLoader.class.getClassLoader().getResourceAsStream(resource); + } else { + // "file:" is stripped, not parsed as a URI, so relative paths and %-escapes stay as written. + Path path = Path.of(location.startsWith(FILE) ? location.substring(FILE.length()) : location); + try { + in = Files.newInputStream(path); + } catch (NoSuchFileException e) { + in = null; + } + } + if (in == null) { + throw new IllegalStateException("local catalog file '" + location + "' does not exist"); + } + return in; + } } diff --git a/server/opensharing-server/src/main/java/io/opensharing/config/ServerProperties.java b/server/opensharing-server/src/main/java/io/opensharing/config/ServerProperties.java new file mode 100644 index 000000000..c374fbe99 --- /dev/null +++ b/server/opensharing-server/src/main/java/io/opensharing/config/ServerProperties.java @@ -0,0 +1,96 @@ +package io.opensharing.config; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; + +/** + * The standalone server's settings, read from {@code server.properties} on the classpath. A JVM + * system property with the same name as a key overrides it, then an environment variable with that + * name or its upper snake case form, such as {@code SERVER_PORT} for {@code server.port}. A blank + * value anywhere means unset. + */ +public class ServerProperties { + + private static final String RESOURCE = "server.properties"; + + private final Properties properties = new Properties(defaults()); + private final Map env; + + public enum Property { + PORT("server.port", "8080"), + CATALOG_TYPE("catalog.type", "local"), + LOCAL_CATALOG_FILE("catalog.local.file", "classpath:local-catalog.yml"); + + private final String key; + private final String defaultValue; + + Property(String key, String defaultValue) { + this.key = key; + this.defaultValue = defaultValue; + } + } + + public ServerProperties() { + try (InputStream in = ServerProperties.class.getClassLoader().getResourceAsStream(RESOURCE)) { + if (in != null) { + properties.load(in); + } + } catch (IOException e) { + throw new UncheckedIOException("failed to read " + RESOURCE, e); + } + dropBlankValues(); + env = System.getenv(); + } + + /** Settings read from {@code values} instead of {@code server.properties}. */ + public ServerProperties(Properties values) { + this(values, System.getenv()); + } + + ServerProperties(Properties values, Map env) { + properties.putAll(values); + dropBlankValues(); + this.env = env; + } + + public String get(Property property) { + String value = System.getProperty(property.key); + if (isBlank(value)) { + value = env.get(property.key); + } + if (isBlank(value)) { + value = env.get(property.key.toUpperCase(Locale.ROOT).replaceAll("[.-]", "_")); + } + return isBlank(value) ? properties.getProperty(property.key) : value; + } + + public int getPort() { + String port = get(Property.PORT); + try { + return Integer.parseInt(port.trim()); + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + "invalid value '" + port + "' for " + Property.PORT.key + "; expected a port number"); + } + } + + private static Properties defaults() { + Properties defaults = new Properties(); + for (Property property : Property.values()) { + defaults.setProperty(property.key, property.defaultValue); + } + return defaults; + } + + private static boolean isBlank(String value) { + return value == null || value.isBlank(); + } + + private void dropBlankValues() { + properties.entrySet().removeIf(entry -> entry.getValue().toString().isBlank()); + } +} diff --git a/server/opensharing-server/src/main/resources/application.yml b/server/opensharing-server/src/main/resources/application.yml deleted file mode 100644 index 61d9f670f..000000000 --- a/server/opensharing-server/src/main/resources/application.yml +++ /dev/null @@ -1,69 +0,0 @@ -# Defaults for a local run. Override any key without editing this file, either as an argument: -# java -jar opensharing-server.jar --opensharing.delta.url-ttl=15m -# or as an environment variable: -# OPENSHARING_DELTA_URLTTL=15m java -jar opensharing-server.jar -spring: - application: - name: opensharing-server - datasource: - # The driver is derived from the URL, so pointing this at Postgres or MySQL is the only change - # needed to move the metadata store there. - url: jdbc:h2:file:./data/opensharing;AUTO_SERVER=TRUE - username: sa - password: "" - jpa: - hibernate: - # Creates missing tables and columns to match the entities on startup. - # Production should use 'validate' and manage the schema with a migration tool. - ddl-auto: update - # Closes the database session when each transaction ends, so nothing loads lazily outside the store. - open-in-view: false - jackson: - # Leaves null fields out of JSON responses, as the protocol expects for absent optional fields. - default-property-inclusion: non_null - mapper: - # Accepts enum values in requests in any case, such as "table" for TABLE. - accept-case-insensitive-enums: true - serialization: - # Writes dates and times as ISO-8601 strings, not epoch numbers. - write-dates-as-timestamps: false - -server: - # HTTP port the server listens on. - port: 8080 - error: - whitelabel: - # Errors use the protocol's JSON error body, not Spring's HTML error page. - enabled: false - -opensharing: - # Prefix the protocol-serving endpoints are mounted under; this is what goes in a profile file. - protocol-prefix: /api/1.0/opensharing - asset-credentials: - # Lifetime of the cloud credentials the catalog vends for reading an asset's storage. - ttl: 1h - pagination: - # Page size for list endpoints when the client sends no maxResults. - default-max-results: 500 - # Largest page size a client may request; larger values are capped to this. - max-max-results: 1000 - storage: - # Service-account key used to sign GCS URLs. - gcs-service-account-key-file: "" - delta: - # Whether Query Table may return pre-signed file URLs to recipients. - url-access-enabled: true - # Lifetime of each pre-signed file URL. - url-ttl: 1h - catalog: - # Which CatalogConnector the standalone server uses; local reads assets from a YAML file. - type: local - local: - # The YAML file the local catalog reads, as a Spring resource path. - file: classpath:local-catalog.yml - -logging: - level: - io.opensharing: INFO - # Delta Kernel narrates every log segment it loads, which is a line per request. - io.delta: WARN diff --git a/server/opensharing-server/src/main/resources/local-catalog.yml b/server/opensharing-server/src/main/resources/local-catalog.yml index 1d615f7a9..920e65480 100644 --- a/server/opensharing-server/src/main/resources/local-catalog.yml +++ b/server/opensharing-server/src/main/resources/local-catalog.yml @@ -1,4 +1,4 @@ -# Sample catalog for opensharing.catalog.type=local. +# Sample catalog for catalog.type=local. # FAKE credentials are placeholders. Use STATIC with credentials.values for real keys. credentials: provider: AWS diff --git a/server/opensharing-server/src/main/resources/logback.xml b/server/opensharing-server/src/main/resources/logback.xml new file mode 100644 index 000000000..7efeaf452 --- /dev/null +++ b/server/opensharing-server/src/main/resources/logback.xml @@ -0,0 +1,15 @@ + + + + %d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX} %-5level [%thread] %logger{36} : %msg%n + + + + + + + + + + + diff --git a/server/opensharing-server/src/main/resources/server.properties b/server/opensharing-server/src/main/resources/server.properties new file mode 100644 index 000000000..8882e4c57 --- /dev/null +++ b/server/opensharing-server/src/main/resources/server.properties @@ -0,0 +1,12 @@ +## OpenSharing standalone server settings, read from the classpath. +## A JVM system property (-Dserver.port=9090) or an environment variable with the same name as a +## key, or its upper snake case form (SERVER_PORT=9090), overrides it. A blank value keeps the +## default. + +# HTTP port the server listens on; 0 picks a free one. +server.port=8080 + +# Which CatalogConnector the server uses; local reads assets from a YAML file. +catalog.type=local +# The YAML file the local catalog reads: a classpath: resource or a file path. +catalog.local.file=classpath:local-catalog.yml diff --git a/server/opensharing-server/src/test/java/io/opensharing/OpenSharingServerTest.java b/server/opensharing-server/src/test/java/io/opensharing/OpenSharingServerTest.java new file mode 100644 index 000000000..9a839d20c --- /dev/null +++ b/server/opensharing-server/src/test/java/io/opensharing/OpenSharingServerTest.java @@ -0,0 +1,78 @@ +package io.opensharing; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import io.opensharing.auth.AuthContext; +import io.opensharing.auth.UserContext; +import io.opensharing.catalog.Asset; +import io.opensharing.catalog.AssetType; +import io.opensharing.catalog.CatalogConnector; +import io.opensharing.catalog.DataSourceFormat; +import io.opensharing.catalog.TableProperties; +import io.opensharing.config.ServerProperties; +import java.util.Properties; +import org.junit.jupiter.api.Test; + +class OpenSharingServerTest { + + @Test + void loadsTheConfiguredLocalCatalog() { + CatalogConnector catalog = OpenSharingServer.catalog(properties()); + UserContext user = UserContext.fromUserIdAndName("alice@example.com", "alice@example.com"); + + assertEquals("local", catalog.name()); + TableProperties table = + (TableProperties) + catalog + .resolveAsset(new Asset(AssetType.TABLE, "main.sales.table1"), AuthContext.of(user)) + .additionalProperties(); + assertEquals(DataSourceFormat.DELTA, table.dataSourceFormat()); + } + + @Test + void rejectsAMissingLocalCatalogFile() { + ServerProperties properties = properties("catalog.local.file", "classpath:missing.yml"); + + IllegalStateException missing = + assertThrows(IllegalStateException.class, () -> OpenSharingServer.catalog(properties)); + assertEquals("local catalog file 'classpath:missing.yml' does not exist", missing.getMessage()); + } + + @Test + void rejectsAnUnknownCatalogType() { + ServerProperties properties = properties("catalog.type", "glue"); + + IllegalStateException unknown = + assertThrows(IllegalStateException.class, () -> OpenSharingServer.catalog(properties)); + assertEquals("unknown catalog.type 'glue'; this build ships 'local'", unknown.getMessage()); + } + + @Test + void buildsOpenSharingWithTheConfiguredCatalog() { + try (OpenSharingServer server = OpenSharingServer.start(properties())) { + assertEquals("local", server.openSharing().catalog().name()); + } + } + + @Test + void rejectsProgramArguments() { + IllegalArgumentException rejected = + assertThrows( + IllegalArgumentException.class, + () -> OpenSharingServer.main(new String[] {"--server.port=9090"})); + assertEquals( + "program arguments are not supported; set server.properties keys as -D JVM options or" + + " environment variables", + rejected.getMessage()); + } + + private static ServerProperties properties(String... overrides) { + Properties props = new Properties(); + props.setProperty("server.port", "0"); + for (int i = 0; i < overrides.length; i += 2) { + props.setProperty(overrides[i], overrides[i + 1]); + } + return new ServerProperties(props); + } +} diff --git a/server/opensharing-server/src/test/java/io/opensharing/config/ServerPropertiesTest.java b/server/opensharing-server/src/test/java/io/opensharing/config/ServerPropertiesTest.java new file mode 100644 index 000000000..657a6c913 --- /dev/null +++ b/server/opensharing-server/src/test/java/io/opensharing/config/ServerPropertiesTest.java @@ -0,0 +1,86 @@ +package io.opensharing.config; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import io.opensharing.config.ServerProperties.Property; +import java.util.Map; +import java.util.Properties; +import org.junit.jupiter.api.Test; + +class ServerPropertiesTest { + + @Test + void readsServerPropertiesFromTheClasspath() { + ServerProperties properties = new ServerProperties(); + + assertEquals(8080, properties.getPort()); + assertEquals("local", properties.get(Property.CATALOG_TYPE)); + assertEquals("classpath:local-catalog.yml", properties.get(Property.LOCAL_CATALOG_FILE)); + } + + @Test + void keepsTheDefaultOfBlankKeys() { + Properties props = props("server.port", "9090"); + props.setProperty("catalog.local.file", " "); + + ServerProperties properties = new ServerProperties(props); + + assertEquals(9090, properties.getPort()); + assertEquals("classpath:local-catalog.yml", properties.get(Property.LOCAL_CATALOG_FILE)); + } + + @Test + void letsSystemPropertiesOverrideTheFile() { + ServerProperties properties = new ServerProperties(props("server.port", "9090")); + System.setProperty("server.port", "7070"); + try { + assertEquals(7070, properties.getPort()); + } finally { + System.clearProperty("server.port"); + } + } + + @Test + void ignoresBlankOverrides() { + ServerProperties properties = + new ServerProperties(props("server.port", "9090"), Map.of("CATALOG_TYPE", " ")); + System.setProperty("server.port", ""); + try { + assertEquals(9090, properties.getPort()); + assertEquals("local", properties.get(Property.CATALOG_TYPE)); + } finally { + System.clearProperty("server.port"); + } + } + + @Test + void readsEnvironmentVariablesByKeyOrUpperSnakeCase() { + ServerProperties properties = + new ServerProperties( + new Properties(), + Map.of( + "SERVER_PORT", "7070", + "catalog.type", "dotted", + "CATALOG_TYPE", "snake")); + + assertEquals(7070, properties.getPort()); + assertEquals("dotted", properties.get(Property.CATALOG_TYPE)); + } + + @Test + void rejectsAPortThatIsNotANumber() { + ServerProperties properties = new ServerProperties(props("server.port", "eighty")); + + IllegalArgumentException invalid = + assertThrows(IllegalArgumentException.class, properties::getPort); + assertEquals( + "invalid value 'eighty' for server.port; expected a port number", invalid.getMessage()); + } + + private static Properties props(String key, String value) { + Properties props = new Properties(); + props.setProperty(key, value); + return props; + } +} diff --git a/server/pom.xml b/server/pom.xml index 1dca47e68..287199aa4 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -27,12 +27,18 @@ UTF-8 21 - 3.5.3 + 1.28.4 + + 4.1.126.Final + 2.19.1 + 3.1.0 + 2.0.17 + 1.5.18 + 5.12.2 4.0.0 3.4.3 2.35.4 hadoop3-2.2.30 - 2.9.0 @@ -43,12 +49,48 @@ ${project.version} - org.springframework.boot - spring-boot-dependencies - ${spring-boot.version} + io.netty + netty-bom + ${netty.version} pom import + + com.linecorp.armeria + armeria-bom + ${armeria.version} + pom + import + + + com.fasterxml.jackson + jackson-bom + ${jackson.version} + pom + import + + + org.junit + junit-bom + ${junit.version} + pom + import + + + jakarta.persistence + jakarta.persistence-api + ${jakarta-persistence.version} + + + org.slf4j + slf4j-api + ${slf4j.version} + + + ch.qos.logback + logback-classic + ${logback.version} + @@ -71,9 +113,9 @@ 3.2.5 - org.springframework.boot - spring-boot-maven-plugin - ${spring-boot.version} + org.apache.maven.plugins + maven-shade-plugin + 3.6.0