diff --git a/ b/ new file mode 100644 index 0000000..85a548e --- /dev/null +++ b/ @@ -0,0 +1,36 @@ + + +# Type-surface conformance status + +Generated by `dev/build-status-matrix.py` from the type-surface runners +(`table-spec/types/`). It reports; it does not block: + +- **PASS** - matches the spec-derived fixture. +- **FAIL** - a genuine divergence from the spec-derived fixture. +- **UNSUPPORTED** - a type the implementation has not built yet (a skip-list candidate). + + +| Implementation | Pass | Fail | Unsupported | Skip | Status | +|---|---|---|---|---|---| + +## Divergences and gaps + +None reported. + diff --git a/.github/workflows/conformance-java.yml b/.github/workflows/conformance-java.yml new file mode 100644 index 0000000..3f0c030 --- /dev/null +++ b/.github/workflows/conformance-java.yml @@ -0,0 +1,113 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: "Conformance (Java)" + +on: + push: + branches: + - main + pull_request: + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +jobs: + build: + name: build ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + max-parallel: 15 + fail-fast: false + matrix: + os: + - 'ubuntu-latest' + - 'macos-latest' + steps: + - name: Checkout fixtures + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Check runner deps are release-clean + run: bash dev/check-runner-deps.sh + - name: Install Java + uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0 + with: + distribution: zulu + java-version: '17' + - name: Set up Gradle + uses: gradle/actions/setup-gradle@9c971963bec38e04b3d30dcc455b5382be2fdbfb # v6.3.0 + - name: Format check and grader self-test + working-directory: runners/java + run: ./gradlew spotlessCheck test + + conformance-java: + name: iceberg-java type conformance ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - 'ubuntu-latest' + - 'macos-latest' + # Advisory: never blocks this repo's CI (tracks a moving upstream SNAPSHOT). + continue-on-error: true + steps: + - name: Checkout fixtures + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Install Java + uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0 + with: + distribution: zulu + java-version: '17' + - name: Set up Gradle + uses: gradle/actions/setup-gradle@9c971963bec38e04b3d30dcc455b5382be2fdbfb # v6.3.0 + - name: Run type conformance + id: run + working-directory: runners/java + run: | + set +e + meta="https://repository.apache.org/content/repositories/snapshots/org/apache/iceberg/iceberg-api/maven-metadata.xml" + version="$(curl -fsSL "$meta" | grep -o '[^<]*' | sed -e 's/<[^>]*>//g' | head -1)" + if [ -z "$version" ]; then + version="1.12.0-SNAPSHOT" + echo "WARN: could not resolve latest SNAPSHOT from $meta; using fallback $version" >&2 + fi + echo "resolved=iceberg-api:$version" >> "$GITHUB_OUTPUT" + if ./gradlew installDist --quiet -PicebergVersion="$version"; then + build/install/conformance-java/bin/conformance-java --surface types 2>&1 | tee "$GITHUB_WORKSPACE/runner-output.txt" + code=${PIPESTATUS[0]} + else + code=2 + fi + echo "code=$code" >> "$GITHUB_OUTPUT" + - name: Report results (step summary + annotations) + if: always() + env: + RUN_CODE: ${{ steps.run.outputs.code }} + RUN_RESOLVED: ${{ steps.run.outputs.resolved }} + run: > + bash "$GITHUB_WORKSPACE/dev/ci-report.sh" + iceberg-java "$GITHUB_WORKSPACE/runner-output.txt" + snapshot "$RUN_CODE" + "$RUN_RESOLVED" diff --git a/README.md b/README.md index ec2d472..ad5d0a4 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,11 @@ repository to a commit, runs the surfaces it opts into, and bumps that pin deliberately. There is no central conformance gate and no pass/fail matrix across implementations. +## Runners + +Reference runners that check each implementation against the fixtures live in +[`runners/`](runners/README.md), with a per-language support matrix and CI status. + ## Contributing TODO: `CONTRIBUTING.md`, covering repository layout, fixture format, and how to diff --git a/dev/.rat-excludes b/dev/.rat-excludes index bc614e9..f5ca92e 100644 --- a/dev/.rat-excludes +++ b/dev/.rat-excludes @@ -24,3 +24,7 @@ lib # Apache header; they are covered by the repository LICENSE and NOTICE at the root. **/*.json **/*.jsonl + +# Vendored Gradle wrapper (upstream Gradle files, no ASF header). +**/gradle/wrapper/gradle-wrapper.properties +**/gradle/wrapper/gradle-wrapper.jar diff --git a/dev/check-runner-deps.sh b/dev/check-runner-deps.sh new file mode 100755 index 0000000..b3f04d8 --- /dev/null +++ b/dev/check-runner-deps.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Fail if a committed runner dependency file carries a local source override. +set -uo pipefail +root="$(cd "$(dirname "$0")/.." && pwd)" +fail=0 + +check() { # + hits="$(grep -nE "$2" "$root/$1" 2>/dev/null)" + if [ -n "$hits" ]; then + echo "FAIL: $1 - $3" + printf '%s\n' "$hits" | sed 's/^/ /' + fail=1 + fi +} + +check runners/java/build.gradle 'mavenLocal|includeBuild' "committed local build reference - leaked" +check runners/java/settings.gradle 'includeBuild' "committed composite-build reference - leaked" + +if [ "$fail" -eq 0 ]; then + echo "runner deps are release-clean" +else + exit 1 +fi diff --git a/dev/ci-report.sh b/dev/ci-report.sh new file mode 100755 index 0000000..0a3c2ca --- /dev/null +++ b/dev/ci-report.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Render a runner's exit code and output to the CI step summary and annotations. +# Usage: ci-report.sh [] +set -uo pipefail + +impl="${1:?usage: ci-report.sh []}" +out="${2:?usage: ci-report.sh []}" +lane="${3:-release}" +# Missing code = the run step was skipped; default to ERROR, never PASS. +code="${4:-2}" +resolved="${5:-unknown}" + +totals="$(grep -m1 '^TOTALS:' "$out" 2>/dev/null || echo 'TOTALS: (runner produced no summary - see the step log)')" +fails="$(grep -m1 '^FAIL ids:' "$out" 2>/dev/null || true)" +advisory="$(grep -m1 '^ADVISORY_FAIL ids:' "$out" 2>/dev/null || true)" +skip="$(grep -m1 '^SKIP ids:' "$out" 2>/dev/null || true)" + +case "$code" in + 0) verdict="PASS" ;; + 1) verdict="FAIL" ;; + *) verdict="ERROR" ;; +esac + +if [ "$verdict" = "FAIL" ]; then level="error"; else level="warning"; fi + +# Step summary (no-op when not in CI). +if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + { + echo "### ${impl} - types conformance (${lane} lane)" + echo + echo "- result: **${verdict}**" + echo "- tested against: \`${resolved}\`" + echo + echo '```' + echo "${totals}" + [ -n "${fails}" ] && echo "${fails}" + [ -n "${advisory}" ] && echo "${advisory}" + [ -n "${skip}" ] && echo "${skip}" + echo '```' + echo + } >> "${GITHUB_STEP_SUMMARY}" +fi + +# Inline annotations. +case "$verdict" in + FAIL) echo "::${level} title=${impl} ${lane} divergence::${fails:-conformance FAIL} (tested ${resolved})" ;; + ERROR) echo "::warning title=${impl} ${lane} runner error::runner exited ${code} before a verdict - see the step log" ;; +esac +[ -n "${advisory}" ] && echo "::notice title=${impl} advisory_fail (unmet SHOULD, non-blocking)::${advisory}" +[ -n "${skip}" ] && echo "::notice title=${impl} skip (skip-list candidate)::${skip}" + +echo "RESULT: ${impl} [${lane}] ${verdict} (${resolved}) - ${totals}" diff --git a/runners/README.md b/runners/README.md new file mode 100644 index 0000000..d83f76d --- /dev/null +++ b/runners/README.md @@ -0,0 +1,73 @@ + + +# Runners + +A runner parses each fixture `input` with one implementation's own type parser and +grades the result. Each lives under `runners//` with its own workflow, so +one runner's break can't redden another's. + +## Support + +| Runner | Surfaces | CI | +| --- | --- | --- | +| [Java](java/) | `types` | [![Java](https://github.com/nssalian/iceberg-verification/actions/workflows/conformance-java.yml/badge.svg)](https://github.com/nssalian/iceberg-verification/actions/workflows/conformance-java.yml) | +| Go | _planned_ | - | +| Rust | _planned_ | - | +| Python | _planned_ | - | + +Surfaces are the fixture sets a runner opts into (`table-spec/manifest.json`); `schema` +has fixtures but no runner yet. The CI column is this repo's own run status, not a +conformance ranking of the implementations. + +Each case grades into four states (full contract in `table-spec/types/README.md`): + +- **pass** - matches the expectation. +- **fail** - a `must` violation; the only state that exits non-zero. +- **advisory_fail** - a failed `normative_level: should` case; reported, never blocks. +- **skip** - not run here (surface not subscribed, or the type isn't modeled). + +`--surface ` (or `CONFORMANCE_SURFACES`) limits the run to a surface in +`table-spec/manifest.json`; the default runs every surface the runner implements. A +build or setup failure exits 2 (ERROR), distinct from a conformance `fail`. Output is +one line per case plus a summary: + +``` +TOTALS: N cases | pass=.. fail=.. advisory_fail=.. skip=.. +FAIL ids: [...] +ADVISORY_FAIL ids: [...] +SKIP ids: [...] +``` + +The Java runner implements the `types` surface against the latest `iceberg-api` +SNAPSHOT (no release carries the v3 types yet). Go, Rust, Python, and a nightly lane +are not part of this change. + +## Consuming the fixtures + +Vendor this repo as a pinned git submodule and run the fixtures in your own CI: + +``` +git submodule add https://github.com/apache/iceberg-verification conformance +``` + +Then reuse a runner here (pointed at your build via `--include-build`, scoped with +`--surface`) or write your own check over `conformance/table-spec//**/cases.json`: +parse `input`, compare to `decoded` (and `canonical` if present), report the four +states. Only `fail` blocks. diff --git a/runners/java/.gitignore b/runners/java/.gitignore new file mode 100644 index 0000000..4fd961c --- /dev/null +++ b/runners/java/.gitignore @@ -0,0 +1,5 @@ +# Local-only: an iceberg clone for `--include-build` (CI resolves the Maven SNAPSHOT). +/iceberg-src/ + +/.gradle/ +/build/ diff --git a/runners/java/README.md b/runners/java/README.md new file mode 100644 index 0000000..8c20784 --- /dev/null +++ b/runners/java/README.md @@ -0,0 +1,37 @@ + + +# Java conformance runner + +[![Conformance (Java)](https://github.com/nssalian/iceberg-verification/actions/workflows/conformance-java.yml/badge.svg)](https://github.com/nssalian/iceberg-verification/actions/workflows/conformance-java.yml) + +Checks [Apache Iceberg (Java)](https://github.com/apache/iceberg) against the +`table-spec/types` fixtures. See [`runners/README.md`](../README.md) for the shared +four-state contract. + +## Running it + +```sh +# from runners/java (bundled wrapper, JDK 17) +./gradlew run # against the build.gradle floor +./gradlew run -PicebergVersion=-SNAPSHOT # against a dev SNAPSHOT (CI uses this) +``` + +Depends on `iceberg-api` + `iceberg-core` (the type parsers) and `jackson-databind`, +from Maven Central. Prints one line per case plus totals; exits non-zero on any fail. diff --git a/runners/java/build.gradle b/runners/java/build.gradle new file mode 100644 index 0000000..0359e11 --- /dev/null +++ b/runners/java/build.gradle @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { + id 'application' + id 'com.diffplug.spotless' version '8.9.0' +} + +def icebergVersion = (project.findProperty('icebergVersion') ?: '1.11.0').toString() + +repositories { + mavenCentral() + if (icebergVersion.endsWith('SNAPSHOT')) { + maven { url = uri('https://repository.apache.org/content/repositories/snapshots') } + } +} + +java { + toolchain { + languageVersion = JavaLanguageVersion.of(17) + } +} + +dependencies { + implementation "org.apache.iceberg:iceberg-api:${icebergVersion}" + implementation "org.apache.iceberg:iceberg-core:${icebergVersion}" + implementation 'com.fasterxml.jackson.core:jackson-databind:2.18.2' + testImplementation 'org.junit.jupiter:junit-jupiter:5.10.2' + testRuntimeOnly 'org.junit.platform:junit-platform-launcher' +} + +test { + useJUnitPlatform() +} + +application { + mainClass = 'org.apache.iceberg.conformance.VerifyTypes' +} + +spotless { + java { + googleJavaFormat("1.22.0") + removeUnusedImports() + trimTrailingWhitespace() + endWithNewline() + } +} diff --git a/runners/java/gradle/wrapper/gradle-wrapper.jar b/runners/java/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000..1b33c55 Binary files /dev/null and b/runners/java/gradle/wrapper/gradle-wrapper.jar differ diff --git a/runners/java/gradle/wrapper/gradle-wrapper.properties b/runners/java/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..a4cf193 --- /dev/null +++ b/runners/java/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,8 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionSha256Sum=6f74b601422d6d6fc4e1f9a1ab6522f642c2fdcbc15ae33ebd30ba3d7198e854 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.5-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/runners/java/gradlew b/runners/java/gradlew new file mode 100755 index 0000000..30403b3 --- /dev/null +++ b/runners/java/gradlew @@ -0,0 +1,253 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit + +if [ ! -e $APP_HOME/gradle/wrapper/gradle-wrapper.jar ]; then + curl -o $APP_HOME/gradle/wrapper/gradle-wrapper.jar https://raw.githubusercontent.com/gradle/gradle/v8.14.5/gradle/wrapper/gradle-wrapper.jar +fi + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/runners/java/gradlew.bat b/runners/java/gradlew.bat new file mode 100644 index 0000000..07742d4 --- /dev/null +++ b/runners/java/gradlew.bat @@ -0,0 +1,95 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=.\ + +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/runners/java/settings.gradle b/runners/java/settings.gradle new file mode 100644 index 0000000..8a899a2 --- /dev/null +++ b/runners/java/settings.gradle @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +rootProject.name = 'conformance-java' diff --git a/runners/java/src/main/java/org/apache/iceberg/conformance/VerifyTypes.java b/runners/java/src/main/java/org/apache/iceberg/conformance/VerifyTypes.java new file mode 100644 index 0000000..f1a7fa8 --- /dev/null +++ b/runners/java/src/main/java/org/apache/iceberg/conformance/VerifyTypes.java @@ -0,0 +1,448 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.conformance; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.iceberg.SchemaParser; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; + +/** + * Checks Apache Iceberg (Java) against the type-surface fixtures. Parses each case {@code input} + * with {@link Types#fromTypeName(String)} (string types) or {@link SchemaParser#fromJson(String)} + * (nested types) and grades it per the shared contract in {@code runners/README.md}. Exits 0 when + * no case fails, 1 on any fail, 2 on a setup error. + */ +public final class VerifyTypes { + + private static final String[] SURFACE_ROOTS = {"table-spec/types"}; + + private VerifyTypes() {} + + public static void main(String[] args) { + Path start = Paths.get(System.getProperty("user.dir")); + List surfaces = new ArrayList<>(); + for (int i = 0; i < args.length; i++) { + String a = args[i]; + if (a.equals("--surface")) { + if (i + 1 >= args.length) { + System.err.println("Setup error: --surface requires a value"); + System.exit(2); + } + surfaces.add(args[++i]); + } else if (a.startsWith("--surface=")) { + surfaces.add(a.substring("--surface=".length())); + } else { + start = Paths.get(a); + } + } + String env = System.getenv("CONFORMANCE_SURFACES"); + if (env != null) { + for (String s : env.split("[,\\s]+")) { + if (!s.isEmpty()) { + surfaces.add(s); + } + } + } + Path root = findRepoRoot(start.toAbsolutePath()); + if (root == null) { + System.err.println("Could not locate repo root (no table-spec/) above " + start); + System.exit(2); + } + + List cases; + try { + cases = loadCases(root, surfaces); + } catch (IOException e) { + // A fixture-read failure is a setup error (exit 2), not a conformance FAIL. + System.err.println("Setup error reading fixtures: " + e.getMessage()); + System.exit(2); + return; + } + if (cases.isEmpty()) { + System.err.println("No cases found under " + root); + System.exit(2); + } + + int pass = 0; + int fail = 0; + int advisoryFail = 0; + int skip = 0; + List failIds = new ArrayList<>(); + List advisoryIds = new ArrayList<>(); + List skipIds = new ArrayList<>(); + + for (JsonNode node : cases) { + String id = node.get("id").asText(); + String[] graded = grade(node); + String status = graded[0]; + String detail = graded[1]; + + switch (status) { + case "pass": + pass++; + break; + case "advisory_fail": + advisoryFail++; + advisoryIds.add(id); + break; + case "skip": + skip++; + skipIds.add(id); + break; + default: + fail++; + failIds.add(id); + break; + } + if (detail.isEmpty()) { + System.out.printf(Locale.ROOT, "%-32s %s%n", id, status); + } else { + System.out.printf(Locale.ROOT, "%-32s %s (%s)%n", id, status, detail); + } + } + + System.out.printf( + Locale.ROOT, + "%nTOTALS: %d cases | pass=%d fail=%d advisory_fail=%d skip=%d%n", + cases.size(), + pass, + fail, + advisoryFail, + skip); + if (!failIds.isEmpty()) { + System.out.println("FAIL ids: " + failIds); + } + if (!advisoryIds.isEmpty()) { + System.out.println("ADVISORY_FAIL ids: " + advisoryIds); + } + if (!skipIds.isEmpty()) { + System.out.println("SKIP ids: " + skipIds); + } + if (fail > 0) { + System.exit(1); + } + } + + /** Grades one case, returning {@code {status, detail}}. */ + static String[] grade(JsonNode node) { + String id = node.get("id").asText(); + JsonNode input = node.get("input"); + boolean valid = node.get("valid").asBoolean(); + + Type parsed = null; + RuntimeException parseError = null; + try { + parsed = parseType(input); + } catch (RuntimeException e) { + parseError = e; + } + + String status; + String detail; + + if (!valid) { + if (parseError != null) { + status = "pass"; + detail = "rejected: " + parseError.getMessage(); + } else { + status = failState(node); + detail = "expected reject, parsed as \"" + parsed + "\""; + } + } else if (parseError != null) { + if (!isModeled(input)) { + status = "skip"; + detail = "not modeled: " + parseError.getMessage(); + } else { + status = failState(node); + detail = + "expected accept, parse threw " + + parseError.getClass().getSimpleName() + + ": " + + parseError.getMessage(); + } + } else { + JsonNode decoded = node.get("decoded"); + if (decoded == null) { + System.err.println("Setup error: valid case " + id + " has no decoded shape"); + System.exit(2); + } + String mismatch = decodedMismatch(parsed, decoded); + if (mismatch != null) { + status = failState(node); + detail = + "decoded mismatch: expected " + + decoded + + ", actual " + + describe(parsed) + + " [" + + mismatch + + "]"; + } else if (node.hasNonNull("canonical") + && !parsed.toString().equals(node.get("canonical").asText())) { + // The case pins a canonical spelling: the re-serialized type must reproduce it. + status = failState(node); + detail = + "canonical mismatch: expected \"" + + node.get("canonical").asText() + + "\", actual \"" + + parsed + + "\""; + } else { + status = "pass"; + detail = ""; + } + } + return new String[] {status, detail}; + } + + static String failState(JsonNode node) { + return node.hasNonNull("normative_level") + && "should".equals(node.get("normative_level").asText()) + ? "advisory_fail" + : "fail"; + } + + private static final Set EXACT_KEYWORDS = + Set.of( + "boolean", + "string", + "int", + "long", + "float", + "double", + "timestamp", + "timestamptz", + "timestamp_ns", + "timestamptz_ns", + "date", + "time", + "uuid", + "binary", + "unknown", + "variant"); + private static final String[] PREFIX_KEYWORDS = {"fixed", "decimal", "geometry", "geography"}; + + static boolean isModeled(JsonNode input) { + if (!input.isTextual()) { + return true; + } + String s = input.asText(); + if (EXACT_KEYWORDS.contains(s)) { + return true; + } + for (String p : PREFIX_KEYWORDS) { + if (s.startsWith(p)) { + return true; + } + } + return false; + } + + /** + * Parses a type from its Appendix-C JSON: a bare string for primitive and geospatial types (via + * {@link Types#fromTypeName(String)}), or an object for a nested type, wrapped as the single + * field of a schema and parsed via {@link SchemaParser#fromJson(String)}. The wrapper field uses + * a very high field id (2147483447) so it never collides with the ids inside a nested type under + * test, which Iceberg would otherwise reject as duplicate schema ids. + */ + static Type parseType(JsonNode input) { + if (input.isTextual()) { + return Types.fromTypeName(input.asText()); + } + String schemaJson = + "{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":2147483447,\"name\":\"f\"," + + "\"required\":true,\"type\":" + + input + + "}]}"; + return SchemaParser.fromJson(schemaJson).columns().get(0).type(); + } + + private static Path findRepoRoot(Path start) { + for (Path dir = start; dir != null; dir = dir.getParent()) { + if (Files.isDirectory(dir.resolve("table-spec"))) { + return dir; + } + } + return null; + } + + private static boolean surfaceMatch(String rel, List surfaces) { + if (surfaces.isEmpty()) { + return true; + } + for (String s : surfaces) { + String prefix = "table-spec/" + s.replaceAll("/+$", ""); + if (rel.equals(prefix) || rel.startsWith(prefix + "/")) { + return true; + } + } + return false; + } + + private static List loadCases(Path root, List surfaces) throws IOException { + ObjectMapper mapper = new ObjectMapper(); + List cases = new ArrayList<>(); + for (String surface : SURFACE_ROOTS) { + Path base = root.resolve(surface); + if (!Files.isDirectory(base)) { + continue; + } + List files; + try (Stream walk = Files.walk(base)) { + files = + walk.filter(Files::isRegularFile) + .filter(p -> p.getFileName().toString().equals("cases.json")) + .collect(Collectors.toList()); + } + for (Path file : files) { + String rel = root.relativize(file).toString().replace(java.io.File.separatorChar, '/'); + if (!surfaceMatch(rel, surfaces)) { + continue; + } + JsonNode document = mapper.readTree(file.toFile()); + JsonNode array = document.get("cases"); + if (array == null || !array.isArray()) { + throw new IOException("Missing \"cases\" array in " + file); + } + for (JsonNode node : array) { + cases.add(node); + } + } + } + cases.sort(Comparator.comparing(node -> node.get("id").asText())); + return cases; + } + + /** + * Compares a parsed type against the language-neutral {@code decoded} shape. Returns null on + * match, or a short reason on mismatch. + */ + static String decodedMismatch(Type type, JsonNode decoded) { + String kind = decoded.get("type").asText(); + switch (kind) { + case "decimal": + if (!(type instanceof Types.DecimalType)) { + return "Not a decimal"; + } + Types.DecimalType d = (Types.DecimalType) type; + return d.precision() == decoded.get("precision").asInt() + && d.scale() == decoded.get("scale").asInt() + ? null + : "Precision/scale"; + case "fixed": + if (!(type instanceof Types.FixedType)) { + return "Not a fixed"; + } + return ((Types.FixedType) type).length() == decoded.get("length").asInt() ? null : "Length"; + case "geometry": + if (!(type instanceof Types.GeometryType)) { + return "Not a geometry"; + } + return ((Types.GeometryType) type).crs().equals(decoded.get("crs").asText()) ? null : "Crs"; + case "geography": + if (!(type instanceof Types.GeographyType)) { + return "Not a geography"; + } + Types.GeographyType g = (Types.GeographyType) type; + return g.crs().equals(decoded.get("crs").asText()) + && g.algorithm().toString().equals(decoded.get("algorithm").asText()) + ? null + : "Crs/algorithm"; + case "struct": + if (!(type instanceof Types.StructType)) { + return "Not a struct"; + } + List fields = ((Types.StructType) type).fields(); + JsonNode fieldsNode = decoded.get("fields"); + if (fields.size() != fieldsNode.size()) { + return "Field count"; + } + for (int i = 0; i < fields.size(); i++) { + Types.NestedField f = fields.get(i); + JsonNode fn = fieldsNode.get(i); + if (f.fieldId() != fn.get("id").asInt()) { + return "Field id"; + } + if (!f.name().equals(fn.get("name").asText())) { + return "Field name"; + } + if (f.isRequired() != fn.get("required").asBoolean()) { + return "Field required"; + } + String sub = decodedMismatch(f.type(), fn.get("type")); + if (sub != null) { + return "field[" + i + "]: " + sub; + } + } + return null; + case "list": + if (!(type instanceof Types.ListType)) { + return "Not a list"; + } + Types.ListType lt = (Types.ListType) type; + if (lt.elementId() != decoded.get("element-id").asInt()) { + return "element-id"; + } + if (lt.isElementRequired() != decoded.get("element-required").asBoolean()) { + return "element-required"; + } + return decodedMismatch(lt.elementType(), decoded.get("element")); + case "map": + if (!(type instanceof Types.MapType)) { + return "Not a map"; + } + Types.MapType mt = (Types.MapType) type; + if (mt.keyId() != decoded.get("key-id").asInt()) { + return "key-id"; + } + if (mt.valueId() != decoded.get("value-id").asInt()) { + return "value-id"; + } + if (mt.isValueRequired() != decoded.get("value-required").asBoolean()) { + return "value-required"; + } + String keyMismatch = decodedMismatch(mt.keyType(), decoded.get("key")); + if (keyMismatch != null) { + return "key: " + keyMismatch; + } + return decodedMismatch(mt.valueType(), decoded.get("value")); + default: + // Primitive: the language-neutral name equals the canonical type string. + return type.toString().equals(kind) ? null : "Primitive name"; + } + } + + private static String describe(Type type) { + return type.getClass().getSimpleName() + "(\"" + type + "\")"; + } +} diff --git a/runners/java/src/test/java/org/apache/iceberg/conformance/VerifyTypesTest.java b/runners/java/src/test/java/org/apache/iceberg/conformance/VerifyTypesTest.java new file mode 100644 index 0000000..0719053 --- /dev/null +++ b/runners/java/src/test/java/org/apache/iceberg/conformance/VerifyTypesTest.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.conformance; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.Test; + +/** Guards the grader - grade() dispatch and the comparison verbs - against regressions. */ +class VerifyTypesTest { + + private final ObjectMapper mapper = new ObjectMapper(); + + private JsonNode json(String raw) throws Exception { + return mapper.readTree(raw); + } + + @Test + void decodedMismatchReturnsNullOnMatchAndReasonOnMismatch() throws Exception { + assertNull(VerifyTypes.decodedMismatch(Types.IntegerType.get(), json("{\"type\":\"int\"}"))); + assertNotNull(VerifyTypes.decodedMismatch(Types.StringType.get(), json("{\"type\":\"int\"}"))); + assertNull( + VerifyTypes.decodedMismatch( + Types.DecimalType.of(9, 2), + json("{\"type\":\"decimal\",\"precision\":9,\"scale\":2}"))); + assertNotNull( + VerifyTypes.decodedMismatch( + Types.DecimalType.of(9, 2), + json("{\"type\":\"decimal\",\"precision\":10,\"scale\":2}"))); + } + + @Test + void failStateIsAdvisoryForShouldAndFailOtherwise() throws Exception { + assertEquals("advisory_fail", VerifyTypes.failState(json("{\"normative_level\":\"should\"}"))); + assertEquals("fail", VerifyTypes.failState(json("{}"))); + assertEquals("fail", VerifyTypes.failState(json("{\"normative_level\":\"must\"}"))); + } + + @Test + void isModeledTracksTheAllowList() throws Exception { + assertTrue(VerifyTypes.isModeled(json("\"variant\""))); + assertTrue(VerifyTypes.isModeled(json("\"decimal(9, 2)\""))); + assertFalse(VerifyTypes.isModeled(json("\"madeuptype\""))); + assertTrue(VerifyTypes.isModeled(json("{\"type\":\"struct\"}"))); + } + + @Test + void parseTypeReadsPrimitiveTypeNames() throws Exception { + assertEquals("long", VerifyTypes.parseType(json("\"long\"")).toString()); + } + + @Test + void gradeCoversEachDispatchState() throws Exception { + assertEquals( + "skip", VerifyTypes.grade(json("{\"id\":\"a\",\"input\":\"json\",\"valid\":true}"))[0]); + assertEquals( + "fail", + VerifyTypes.grade( + json( + "{\"id\":\"b\",\"input\":\"int\",\"valid\":true,\"decoded\":{\"type\":\"long\"}}"))[ + 0]); + assertEquals( + "fail", VerifyTypes.grade(json("{\"id\":\"c\",\"input\":\"int\",\"valid\":false}"))[0]); + assertEquals( + "pass", + VerifyTypes.grade( + json("{\"id\":\"d\",\"input\":\"int\",\"valid\":true,\"decoded\":{\"type\":\"int\"}}"))[ + 0]); + assertEquals( + "advisory_fail", + VerifyTypes.grade( + json( + "{\"id\":\"e\",\"input\":\"int\",\"valid\":true,\"normative_level\":\"should\"," + + "\"decoded\":{\"type\":\"long\"}}"))[0]); + assertEquals( + "fail", + VerifyTypes.grade( + json( + "{\"id\":\"f\",\"input\":\"decimal(9,2)\",\"valid\":true," + + "\"decoded\":{\"type\":\"decimal\",\"precision\":9,\"scale\":2}," + + "\"canonical\":\"WRONG\"}"))[0]); + assertEquals( + "fail", + VerifyTypes.grade( + json( + "{\"id\":\"g\",\"input\":\"decimal(39,0)\",\"valid\":true," + + "\"decoded\":{\"type\":\"decimal\",\"precision\":39,\"scale\":0}}"))[0]); + assertEquals( + "pass", + VerifyTypes.grade(json("{\"id\":\"h\",\"input\":\"notatype\",\"valid\":false}"))[0]); + } + + // Nested arms (struct/list/map) and fixed; the geo arms are covered by the live fixtures. + @Test + void decodedMismatchCoversNestedArms() throws Exception { + assertNull( + VerifyTypes.decodedMismatch( + Types.FixedType.ofLength(16), json("{\"type\":\"fixed\",\"length\":16}"))); + assertNotNull( + VerifyTypes.decodedMismatch( + Types.FixedType.ofLength(8), json("{\"type\":\"fixed\",\"length\":16}"))); + assertNull( + VerifyTypes.decodedMismatch( + Types.StructType.of(Types.NestedField.required(1, "a", Types.IntegerType.get())), + json( + "{\"type\":\"struct\",\"fields\":[{\"id\":1,\"name\":\"a\",\"required\":true," + + "\"type\":{\"type\":\"int\"}}]}"))); + assertNotNull( + VerifyTypes.decodedMismatch( + Types.StructType.of(Types.NestedField.required(2, "a", Types.IntegerType.get())), + json( + "{\"type\":\"struct\",\"fields\":[{\"id\":1,\"name\":\"a\",\"required\":true," + + "\"type\":{\"type\":\"int\"}}]}"))); + assertNull( + VerifyTypes.decodedMismatch( + Types.ListType.ofRequired(3, Types.StringType.get()), + json( + "{\"type\":\"list\",\"element-id\":3,\"element-required\":true," + + "\"element\":{\"type\":\"string\"}}"))); + assertNotNull( + VerifyTypes.decodedMismatch( + Types.ListType.ofRequired(9, Types.StringType.get()), + json( + "{\"type\":\"list\",\"element-id\":3,\"element-required\":true," + + "\"element\":{\"type\":\"string\"}}"))); + assertNull( + VerifyTypes.decodedMismatch( + Types.MapType.ofRequired(4, 5, Types.StringType.get(), Types.LongType.get()), + json( + "{\"type\":\"map\",\"key-id\":4,\"value-id\":5,\"value-required\":true," + + "\"key\":{\"type\":\"string\"},\"value\":{\"type\":\"long\"}}"))); + assertNotNull( + VerifyTypes.decodedMismatch( + Types.MapType.ofRequired(4, 9, Types.StringType.get(), Types.LongType.get()), + json( + "{\"type\":\"map\",\"key-id\":4,\"value-id\":5,\"value-required\":true," + + "\"key\":{\"type\":\"string\"},\"value\":{\"type\":\"long\"}}"))); + } +} diff --git a/table-spec/manifest.json b/table-spec/manifest.json new file mode 100644 index 0000000..90ad50b --- /dev/null +++ b/table-spec/manifest.json @@ -0,0 +1,16 @@ +{ + "surfaces": [ + { + "name": "types", + "path": "table-spec/types", + "description": "Type decoding: each spec type parsed in isolation.", + "subsurfaces": ["primitive", "variant", "nested", "geospatial"] + }, + { + "name": "schema", + "path": "table-spec/schema", + "description": "Schema decoding: the schema document, its id spaces and field tree.", + "subsurfaces": ["core", "unknown"] + } + ] +} diff --git a/table-spec/types/README.md b/table-spec/types/README.md index 7b4cf6e..a297558 100644 --- a/table-spec/types/README.md +++ b/table-spec/types/README.md @@ -34,7 +34,7 @@ its own type object to `decoded`, so the comparison does not depend on one language's representation. - `valid: true` - the parser succeeds and the decoded type equals `decoded`. A - type an implementation does not model is UNSUPPORTED, not a failure. If the case + type an implementation does not model is skip, not a failure. If the case also carries `canonical`, re-serializing the parsed type must equal it byte for byte (the write direction). - `valid: false` - the parser must reject `input`. A rejection passes; a