diff --git a/Makefile b/Makefile index 7c2ce35..fb5507e 100644 --- a/Makefile +++ b/Makefile @@ -84,7 +84,44 @@ selftest: $(VENV_STAMP) ## Check the validator accepts a good fixture and reject assert_reject "decoded decimal typo (precison)" '{"cases":[{"id":"a","valid":true,"input":"decimal(9,2)","decoded":{"type":"decimal","precison":9,"scale":2},"clause":"c","spec_ref":"s"}]}'; \ assert_reject "decoded fixed carrying len not length" '{"cases":[{"id":"a","valid":true,"input":"fixed[16]","decoded":{"type":"fixed","len":16},"clause":"c","spec_ref":"s"}]}'; \ assert_reject "decoded unknown type name" '{"cases":[{"id":"a","valid":true,"input":"x","decoded":{"type":"notatype"},"clause":"c","spec_ref":"s"}]}'; \ - assert_reject "decoded simple type with extra key" '{"cases":[{"id":"a","valid":true,"input":"int","decoded":{"type":"int","precision":9},"clause":"c","spec_ref":"s"}]}' + assert_reject "decoded simple type with extra key" '{"cases":[{"id":"a","valid":true,"input":"int","decoded":{"type":"int","precision":9},"clause":"c","spec_ref":"s"}]}'; \ + run_partition_transform_validator() { \ + d="$$(mktemp -d)"; \ + mkdir -p "$$d/table-spec/partition-transforms/x"; \ + printf '%s' "$$1" > "$$d/table-spec/partition-transforms/x/cases.json"; \ + rc=0; out="$$($(VENV_PYTHON) dev/validate-fixtures.py "$$d" 2>&1)" || rc=$$?; \ + rm -rf "$$d"; \ + }; \ + assert_partition_transform_accept() { \ + run_partition_transform_validator "$$2"; \ + if [ "$$rc" -ne 0 ]; then \ + echo "FAILED: validator rejected a well-formed fixture ($$1)"; echo "$$out"; exit 1; \ + fi; \ + echo "ok: accepted ($$1)"; \ + }; \ + assert_partition_transform_reject() { \ + run_partition_transform_validator "$$2"; \ + if [ "$$rc" -ne 1 ] || ! grep -q '^FAILED: ' <<< "$$out"; then \ + echo "FAILED: validator did not reject a malformed fixture ($$1), exit code $$rc"; echo "$$out"; exit 1; \ + fi; \ + if [ -n "$$3" ] && ! grep -Fq "$$3" <<< "$$out"; then \ + echo "FAILED: validator report did not contain '$$3' ($$1)"; echo "$$out"; exit 1; \ + fi; \ + echo "ok: rejected ($$1)"; \ + }; \ + assert_partition_transform_accept "well-formed partition transform" '{"cases":[{"id":"a","valid":true,"input":{"transform":"bucket[100]","source_type":"int","values":[34,null]},"decoded":{"result_types":["int"],"values":[79,null]},"clause":"c","spec_ref":"s"}]}'; \ + assert_partition_transform_reject "unknown partition transform" '{"cases":[{"id":"a","valid":true,"input":{"transform":"mystery","source_type":"int","values":[1,null]},"decoded":{"result_types":["int"],"values":[1,null]},"clause":"c","spec_ref":"s"}]}'; \ + assert_partition_transform_reject "unsupported transform source pair" '{"cases":[{"id":"a","valid":true,"input":{"transform":"bucket[10]","source_type":"boolean","values":[true,null]},"decoded":{"result_types":["int"],"values":[1,null]},"clause":"c","spec_ref":"s"}]}'; \ + assert_partition_transform_reject "unequal partition transform value lengths" '{"cases":[{"id":"a","valid":true,"input":{"transform":"identity","source_type":"int","values":[1,null]},"decoded":{"result_types":["int"],"values":[1]},"clause":"c","spec_ref":"s"}]}'; \ + assert_partition_transform_reject "partition transform without null input" '{"cases":[{"id":"a","valid":true,"input":{"transform":"identity","source_type":"int","values":[1]},"decoded":{"result_types":["int"],"values":[1]},"clause":"c","spec_ref":"s"}]}'; \ + assert_partition_transform_reject "partition transform null propagation mismatch" '{"cases":[{"id":"a","valid":true,"input":{"transform":"bucket[10]","source_type":"int","values":[null]},"decoded":{"result_types":["int"],"values":[0]},"clause":"c","spec_ref":"s"}]}'; \ + assert_partition_transform_reject "wrong bucket result type" '{"cases":[{"id":"a","valid":true,"input":{"transform":"bucket[10]","source_type":"long","values":["1",null]},"decoded":{"result_types":["long"],"values":[1,null]},"clause":"c","spec_ref":"s"}]}'; \ + assert_partition_transform_reject "wrong day result type" '{"cases":[{"id":"a","valid":true,"input":{"transform":"day","source_type":"date","values":["1970-01-01",null]},"decoded":{"result_types":["int"],"values":["1970-01-01",null]},"clause":"c","spec_ref":"s"}]}'; \ + assert_partition_transform_reject "identity value mismatch" '{"cases":[{"id":"a","valid":true,"input":{"transform":"identity","source_type":"string","values":["a",null]},"decoded":{"result_types":["string"],"values":["b",null]},"clause":"c","spec_ref":"s"}]}'; \ + assert_partition_transform_reject "void missing an allowed result type" '{"cases":[{"id":"a","valid":true,"input":{"transform":"void","source_type":"string","values":["a",null]},"decoded":{"result_types":["int"],"values":[null,null]},"clause":"c","spec_ref":"s"}]}'; \ + assert_partition_transform_reject "object in partition transform result types" '{"cases":[{"id":"a","valid":true,"input":{"transform":"bucket[100]","source_type":"int","values":[34,null]},"decoded":{"result_types":["int",{}],"values":[79,null]},"clause":"c","spec_ref":"s"}]}' 'cases/0/decoded/result_types/1'; \ + assert_partition_transform_reject "nested list in partition transform result types" '{"cases":[{"id":"a","valid":true,"input":{"transform":"bucket[100]","source_type":"int","values":[34,null]},"decoded":{"result_types":[[]],"values":[79,null]},"clause":"c","spec_ref":"s"}]}' 'cases/0/decoded/result_types/0'; \ + assert_partition_transform_reject "partition transform misspelled field" '{"cases":[{"id":"a","valid":true,"input":{"transform":"identity","source-type":"int","values":[1,null]},"decoded":{"result_types":["int"],"values":[1,null]},"clause":"c","spec_ref":"s"}]}' license: ## Check ASF license headers with Apache RAT dev/check-license diff --git a/dev/schema/cases.partition-transforms.schema.json b/dev/schema/cases.partition-transforms.schema.json new file mode 100644 index 0000000..f223aca --- /dev/null +++ b/dev/schema/cases.partition-transforms.schema.json @@ -0,0 +1,99 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://iceberg.apache.org/verification/cases.partition-transforms.schema.json", + "title": "Conformance cases (partition-transforms surface extension)", + "description": "Extra structural constraints for table-spec/partition-transforms. Applied on top of cases.base.schema.json by dev/validate-fixtures.py.", + "type": "object", + "required": ["cases"], + "additionalProperties": false, + "properties": { + "cases": { + "type": "array", + "items": { "$ref": "#/$defs/case" } + } + }, + "$defs": { + "case": { + "type": "object", + "required": ["id", "valid", "input", "decoded", "clause", "spec_ref"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "valid": { "const": true }, + "input": { "$ref": "#/$defs/input" }, + "decoded": { "$ref": "#/$defs/decoded" }, + "clause": { "type": "string", "minLength": 1 }, + "spec_ref": { "type": "string", "minLength": 1 }, + "normative_level": { "enum": ["must", "should"] } + } + }, + "input": { + "type": "object", + "required": ["transform", "source_type", "values"], + "additionalProperties": false, + "properties": { + "transform": { + "type": "string", + "pattern": "^(identity|bucket\\[[0-9]+\\]|truncate\\[[0-9]+\\]|year|month|day|hour|void)$" + }, + "source_type": { "$ref": "#/$defs/typeName" }, + "values": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/value" } + } + } + }, + "decoded": { + "type": "object", + "required": ["result_types", "values"], + "additionalProperties": false, + "properties": { + "result_types": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "$ref": "#/$defs/typeName" } + }, + "values": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/value" } + } + } + }, + "typeName": { + "type": "string", + "anyOf": [ + { + "enum": [ + "unknown", + "boolean", + "int", + "long", + "float", + "double", + "date", + "time", + "timestamp", + "timestamptz", + "timestamp_ns", + "timestamptz_ns", + "string", + "uuid", + "binary", + "geometry", + "geography" + ] + }, + { "pattern": "^decimal\\([1-9][0-9]*, [0-9]+\\)$" }, + { "pattern": "^fixed\\[[1-9][0-9]*\\]$" }, + { "pattern": "^geometry\\([^()]+\\)$" }, + { "pattern": "^geography\\([^()]+, [^()]+\\)$" } + ] + }, + "value": { + "type": ["null", "boolean", "number", "string"] + } + } +} diff --git a/dev/validate-fixtures.py b/dev/validate-fixtures.py index 97ebb40..29bc24f 100755 --- a/dev/validate-fixtures.py +++ b/dev/validate-fixtures.py @@ -16,10 +16,10 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""Validate the conformance type fixtures against the JSON Schemas in dev/schema/. +"""Validate conformance fixtures against the JSON Schemas in dev/schema/. -Each cases.json is validated against the base structural schema and the types schema -(which requires clause and spec_ref); case ids must be unique within a surface. +Each cases.json is validated against the base structural schema and its surface +schema, when present; case ids must be unique within a surface. """ import glob @@ -47,10 +47,188 @@ def _is_types_surface(path): return "/table-spec/types/" in p or p.startswith("table-spec/types/") +def _is_partition_transforms_surface(path): + p = _rel(path) + return "/table-spec/partition-transforms/" in p or p.startswith( + "table-spec/partition-transforms/" + ) + + +IDENTITY_TYPES = { + "unknown", + "boolean", + "int", + "long", + "float", + "double", + "decimal", + "date", + "time", + "timestamp", + "timestamptz", + "timestamp_ns", + "timestamptz_ns", + "string", + "uuid", + "fixed", + "binary", +} +BUCKET_TYPES = { + "int", + "long", + "decimal", + "date", + "time", + "timestamp", + "timestamptz", + "timestamp_ns", + "timestamptz_ns", + "string", + "uuid", + "fixed", + "binary", +} +TRUNCATE_TYPES = {"int", "long", "decimal", "string", "binary"} +DATE_TRANSFORM_TYPES = { + "date", + "timestamp", + "timestamptz", + "timestamp_ns", + "timestamptz_ns", +} +HOUR_TYPES = {"timestamp", "timestamptz", "timestamp_ns", "timestamptz_ns"} +VOID_TYPES = IDENTITY_TYPES | {"geometry", "geography"} + + +def _type_family(type_name): + if not isinstance(type_name, str): + return None + if type_name.startswith("decimal("): + return "decimal" + if type_name.startswith("fixed["): + return "fixed" + if type_name.startswith("geometry("): + return "geometry" + if type_name.startswith("geography("): + return "geography" + return type_name + + +def _transform_family(transform): + if not isinstance(transform, str): + return None + if transform.startswith("bucket["): + return "bucket" + if transform.startswith("truncate["): + return "truncate" + return transform + + +def _json_scalar_equal(left, right): + return type(left) is type(right) and left == right + + +def _partition_transform_errors(path, doc): + errors = [] + cases = doc.get("cases") if isinstance(doc, dict) else None + if not isinstance(cases, list): + return errors + + allowed_sources = { + "identity": IDENTITY_TYPES, + "bucket": BUCKET_TYPES, + "truncate": TRUNCATE_TYPES, + "year": DATE_TRANSFORM_TYPES, + "month": DATE_TRANSFORM_TYPES, + "day": DATE_TRANSFORM_TYPES, + "hour": HOUR_TYPES, + "void": VOID_TYPES, + } + + for index, case in enumerate(cases): + if not isinstance(case, dict): + continue + input_value = case.get("input") + decoded = case.get("decoded") + if not isinstance(input_value, dict) or not isinstance(decoded, dict): + continue + + transform = input_value.get("transform") + source_type = input_value.get("source_type") + values = input_value.get("values") + result_types = decoded.get("result_types") + expected = decoded.get("values") + transform_family = _transform_family(transform) + source_family = _type_family(source_type) + label = f"{path}: cases/{index}" + + permitted = allowed_sources.get(transform_family) + if permitted is None: + errors.append(f"{label}: unknown transform '{transform}'") + elif source_family not in permitted: + errors.append( + f"{label}: transform '{transform}' does not support source type " + f"'{source_type}'" + ) + + if isinstance(values, list) and isinstance(expected, list): + if len(values) != len(expected): + errors.append( + f"{label}: input.values and decoded.values must have equal lengths" + ) + if None not in values: + errors.append(f"{label}: input.values must include null") + for value_index, (value, output) in enumerate(zip(values, expected)): + if value is None and output is not None: + errors.append( + f"{label}: decoded.values/{value_index} must be null for a " + "null input" + ) + if transform_family == "void" and output is not None: + errors.append( + f"{label}: decoded.values/{value_index} must be null for void" + ) + if transform_family == "identity" and not _json_scalar_equal( + value, output + ): + errors.append( + f"{label}: decoded.values/{value_index} must equal its " + "identity input" + ) + + required_result_types = None + if transform_family in {"bucket", "year", "month", "hour"}: + required_result_types = {"int"} + elif transform_family == "day": + required_result_types = {"date"} + elif transform_family in {"identity", "truncate"} and isinstance( + source_type, str + ): + required_result_types = {source_type} + elif transform_family == "void" and isinstance(source_type, str): + required_result_types = {source_type, "int"} + + if ( + required_result_types is not None + and isinstance(result_types, list) + and all(isinstance(result_type, str) for result_type in result_types) + ): + if set(result_types) != required_result_types: + errors.append( + f"{label}: decoded.result_types must be exactly " + f"{sorted(required_result_types)}" + ) + + return errors + + def main(): root = sys.argv[1] if len(sys.argv) > 1 else "." base_validator = jsonschema.Draft202012Validator(_load_schema("cases.base.schema.json")) types_validator = jsonschema.Draft202012Validator(_load_schema("cases.types.schema.json")) + partition_transforms_validator = jsonschema.Draft202012Validator( + _load_schema("cases.partition-transforms.schema.json") + ) files = sorted(glob.glob(f"{root}/table-spec/**/cases.json", recursive=True)) if not files: @@ -71,11 +249,16 @@ def main(): validators = [base_validator] if _is_types_surface(path): validators.append(types_validator) + if _is_partition_transforms_surface(path): + validators.append(partition_transforms_validator) for validator in validators: for err in sorted(validator.iter_errors(doc), key=str): where = "/".join(str(p) for p in err.absolute_path) or "(root)" errors.append(f"{path}: {where}: {err.message}") + if _is_partition_transforms_surface(path): + errors.extend(_partition_transform_errors(path, doc)) + cases = doc.get("cases") if isinstance(doc, dict) else None if isinstance(cases, list): # ids are unique per surface (the top-level dir under table-spec), so a diff --git a/table-spec/partition-transforms/README.md b/table-spec/partition-transforms/README.md new file mode 100644 index 0000000..b1b4749 --- /dev/null +++ b/table-spec/partition-transforms/README.md @@ -0,0 +1,165 @@ + + +# Partition transforms + +Binding a partition transform to a source type and applying it to values +produces the same logical values and result type in every implementation. The +Iceberg specification is the source of truth for this surface. + +## Assertion + +``` +apply(bind(transform, source_type), ordered_inputs) == ordered_expected_values +result_type(bind(transform, source_type)) is one of allowed_result_types +``` + +Bytes are not compared. Each implementation decodes the language-neutral input +values, binds the transform, applies it in order, and maps its results back to +the language-neutral representation below. + +## Scope + +This surface covers the value transform and its result type. It binds every +standard transform to each source type permitted by the Partition Transforms +table, subject to the deliberate omissions below. The cases include the spec's +examples, boundary values, and an explicit `null` input for every +transform/source-type path. As required by the spec, "All transforms must return +`null` for a `null` input value." + +Consumers subscribe by subdirectory: + +- `identity/` - "Source value, unmodified." +- `bucket/` - Murmur3 x86 32-bit, seed 0, with the sign bit discarded before + taking modulo `N`. +- `truncate/` - integer, long, decimal, string, and binary truncation. +- `year/`, `month/`, `day/`, and `hour/` - temporal ordinals from the Unix + epoch. `day` expected values are logical dates. +- `void/` - "Always produces `null`." + +The subdirectories are independent subscription boundaries. For example, an +implementation without nanosecond timestamp types may skip cases for those +source types while running the other cases in the same directory. + +## Case format + +Each `cases.json` holds a `cases` array. A case has this shape: + +```json +{ + "id": "bucket-int-100", + "valid": true, + "input": { + "transform": "bucket[100]", + "source_type": "int", + "values": [34, null] + }, + "decoded": { + "result_types": ["int"], + "values": [79, null] + }, + "clause": "Bucket Transform Details and Appendix B: int hash reference vector", + "spec_ref": "format/spec.md#appendix-b-32-bit-hash-requirements" +} +``` + +| field | meaning | +| --- | --- | +| `id` | case id, unique within this surface | +| `valid` | always `true`; invalid binding and transform-string parsing are outside this surface | +| `input.transform` | the transform to construct and bind | +| `input.source_type` | the source type to bind | +| `input.values` | ordered logical source values | +| `decoded.result_types` | all result types the specification permits | +| `decoded.values` | expected logical results corresponding by index to `input.values` | +| `clause` | the specification rule or example the case pins | +| `spec_ref` | anchor into `format/spec.md` | + +The result type list normally has one item. The `void` transform permits either +the source type or `int`, so those cases list both choices (deduplicated when +the source type is already `int`). + +## Language-neutral values + +- `boolean`, `int`, and temporal ordinal results are JSON primitives. +- `long` values are decimal strings so JSON number handling cannot lose 64-bit + precision. +- Decimal values are scale-preserving strings. For example, `"14.20"` is not + rewritten as `"14.2"`. +- `fixed` and `binary` values are uppercase hexadecimal strings with two digits + per byte and no separators. +- Dates use ISO `YYYY-MM-DD`. Times and timestamps without a zone use ISO local + forms and retain the precision shown by the string. +- Timestamps with a zone use ISO forms ending in `Z` or a numeric offset. They + are decoded as instants; offset-equivalent strings compare as the same instant. +- Nanosecond timestamp values use nine fractional digits where the additional + precision is material. +- UUID values use canonical lowercase hyphenated text. +- Strings are Unicode scalar values. String truncation counts code points, not + UTF-8 bytes. Binary truncation counts bytes. +- Every asserted null is written explicitly as JSON `null`. + +`day` results are normalized to the logical date strings used here before +comparison. A reader that exposes the spec-equivalent integer day ordinal must +convert that ordinal to a date relative to `1970-01-01`. + +## Procedure and comparison + +For each case in a subscribed directory: + +1. Construct `input.source_type` and `input.transform`, then bind the transform + to the source type. +2. Assert that the bound transform's result type is a member of + `decoded.result_types`. +3. Decode each member of `input.values`, apply the bound transform in order, + and normalize only the transport encodings documented above. +4. Compare each result exactly with the value at the same index in + `decoded.values`. + +Report every difference. A result-type failure is labeled +`/ result_type`; a value failure is labeled +`/ values[]`. A runner reports `pass`, `fail`, or +`skip`; `skip` means the subdirectory or source type is unsupported and was not +run. + +## Provenance + +The Partition Transforms table, Bucket Transform Details, Truncate Transform +Details, and Appendix B of `format/spec.md` define the expected values. The +bucket vectors are copied from Appendix B and converted to `bucket[100]` using +the specified sign-bit rule. The values were independently cross-checked +against Apache Iceberg Java transform tests and PyIceberg transform tests. Those +implementations are cross-checks, not the source of truth. + +## Left out on purpose + +- Partition-spec JSON, parsing invalid transform strings, and invalid + transform/source bindings. +- Partition evolution, predicate projection and residuals, sorting, + human-readable paths, table metadata, manifests, and physical encodings. +- Minimum or maximum values for `N` and `W`, and integer-overflow behavior. The + specification does not settle those choices for this surface. +- `identity` for `geometry` and `geography`, which the Partition Transforms + table explicitly excludes. +- `void` for nested values and `variant`. The cases cover the primitive types + shared with the type surface. `geometry` and `geography` are null-only because + this surface does not choose a language-neutral geospatial value encoding. +- Exact bytes except where a transform definition operates on bytes. Even for + `fixed` and `binary`, the assertion is the decoded logical byte sequence, not + a file or metadata encoding. diff --git a/table-spec/partition-transforms/bucket/cases.json b/table-spec/partition-transforms/bucket/cases.json new file mode 100644 index 0000000..22bcc56 --- /dev/null +++ b/table-spec/partition-transforms/bucket/cases.json @@ -0,0 +1,148 @@ +{ + "cases": [ + { + "id": "bucket-int-100", + "valid": true, + "input": { "transform": "bucket[100]", "source_type": "int", "values": [null, 34] }, + "decoded": { "result_types": ["int"], "values": [null, 79] }, + "clause": "Bucket Transform Details and Appendix B: int hash reference vector", + "spec_ref": "format/spec.md#appendix-b-32-bit-hash-requirements" + }, + { + "id": "bucket-long-100", + "valid": true, + "input": { "transform": "bucket[100]", "source_type": "long", "values": [null, "34"] }, + "decoded": { "result_types": ["int"], "values": [null, 79] }, + "clause": "Appendix B: integer and long hash results must be identical for all integer values", + "spec_ref": "format/spec.md#appendix-b-32-bit-hash-requirements" + }, + { + "id": "bucket-decimal-9-2-100", + "valid": true, + "input": { "transform": "bucket[100]", "source_type": "decimal(9, 2)", "values": [null, "14.20"] }, + "decoded": { "result_types": ["int"], "values": [null, 59] }, + "clause": "Appendix B: decimal 14.20 hashes from its minimum two's-complement big-endian unscaled value; bucket discards the sign bit", + "spec_ref": "format/spec.md#appendix-b-32-bit-hash-requirements" + }, + { + "id": "bucket-decimal-10-3-100", + "valid": true, + "input": { "transform": "bucket[100]", "source_type": "decimal(10, 3)", "values": [null, "1.420"] }, + "decoded": { "result_types": ["int"], "values": [null, 59] }, + "clause": "Appendix B: decimal hashes use the unscaled value and hash results are not dependent on decimal scale", + "spec_ref": "format/spec.md#appendix-b-32-bit-hash-requirements" + }, + { + "id": "bucket-date-100", + "valid": true, + "input": { "transform": "bucket[100]", "source_type": "date", "values": [null, "2017-11-16"] }, + "decoded": { "result_types": ["int"], "values": [null, 26] }, + "clause": "Appendix B: date hash reference vector; bucket discards the sign bit", + "spec_ref": "format/spec.md#appendix-b-32-bit-hash-requirements" + }, + { + "id": "bucket-time-100", + "valid": true, + "input": { "transform": "bucket[100]", "source_type": "time", "values": [null, "22:31:08"] }, + "decoded": { "result_types": ["int"], "values": [null, 59] }, + "clause": "Appendix B: time hash reference vector; bucket discards the sign bit", + "spec_ref": "format/spec.md#appendix-b-32-bit-hash-requirements" + }, + { + "id": "bucket-timestamp-100", + "valid": true, + "input": { + "transform": "bucket[100]", + "source_type": "timestamp", + "values": [null, "2017-11-16T22:31:08", "2017-11-16T22:31:08.000001"] + }, + "decoded": { "result_types": ["int"], "values": [null, 7, 38] }, + "clause": "Appendix B: timestamp hash reference vectors at whole-second and microsecond precision", + "spec_ref": "format/spec.md#appendix-b-32-bit-hash-requirements" + }, + { + "id": "bucket-timestamptz-100", + "valid": true, + "input": { + "transform": "bucket[100]", + "source_type": "timestamptz", + "values": [ + null, + "2017-11-16T14:31:08-08:00", + "2017-11-16T22:31:08Z", + "2017-11-16T14:31:08.000001-08:00", + "2017-11-16T22:31:08.000001Z" + ] + }, + "decoded": { "result_types": ["int"], "values": [null, 7, 7, 38, 38] }, + "clause": "Appendix B: timestamptz hashes microseconds from the Unix epoch, so equivalent instants hash identically", + "spec_ref": "format/spec.md#appendix-b-32-bit-hash-requirements" + }, + { + "id": "bucket-timestamp-ns-100", + "valid": true, + "input": { + "transform": "bucket[100]", + "source_type": "timestamp_ns", + "values": [null, "2017-11-16T22:31:08", "2017-11-16T22:31:08.000001001", "2017-11-16T22:31:08.000001999"] + }, + "decoded": { "result_types": ["int"], "values": [null, 7, 38, 38] }, + "clause": "Appendix B: nanosecond timestamps must be converted to microsecond precision before hashing", + "spec_ref": "format/spec.md#appendix-b-32-bit-hash-requirements" + }, + { + "id": "bucket-timestamptz-ns-100", + "valid": true, + "input": { + "transform": "bucket[100]", + "source_type": "timestamptz_ns", + "values": [ + null, + "2017-11-16T14:31:08-08:00", + "2017-11-16T22:31:08Z", + "2017-11-16T14:31:08.000001001-08:00", + "2017-11-16T14:31:08.000001999-08:00" + ] + }, + "decoded": { "result_types": ["int"], "values": [null, 7, 7, 38, 38] }, + "clause": "Appendix B: zoned nanosecond timestamps hash their instant after conversion to microsecond precision", + "spec_ref": "format/spec.md#appendix-b-32-bit-hash-requirements" + }, + { + "id": "bucket-string-100", + "valid": true, + "input": { "transform": "bucket[100]", "source_type": "string", "values": [null, "iceberg"] }, + "decoded": { "result_types": ["int"], "values": [null, 89] }, + "clause": "Appendix B: string UTF-8 hash reference vector", + "spec_ref": "format/spec.md#appendix-b-32-bit-hash-requirements" + }, + { + "id": "bucket-uuid-100", + "valid": true, + "input": { + "transform": "bucket[100]", + "source_type": "uuid", + "values": [null, "f79c3e09-677c-4bbd-a479-3f349cb785e7"] + }, + "decoded": { "result_types": ["int"], "values": [null, 40] }, + "clause": "Appendix B: UUID big-endian byte hash reference vector", + "spec_ref": "format/spec.md#appendix-b-32-bit-hash-requirements" + }, + { + "id": "bucket-fixed-4-100", + "valid": true, + "input": { "transform": "bucket[100]", "source_type": "fixed[4]", "values": [null, "00010203"] }, + "decoded": { "result_types": ["int"], "values": [null, 41] }, + "clause": "Appendix B: fixed bytes hash reference vector; bucket discards the sign bit", + "spec_ref": "format/spec.md#appendix-b-32-bit-hash-requirements" + }, + { + "id": "bucket-binary-100", + "valid": true, + "input": { "transform": "bucket[100]", "source_type": "binary", "values": [null, "00010203"] }, + "decoded": { "result_types": ["int"], "values": [null, 41] }, + "clause": "Appendix B: binary bytes hash reference vector, identical to fixed for the same bytes", + "spec_ref": "format/spec.md#appendix-b-32-bit-hash-requirements" + } + ] +} diff --git a/table-spec/partition-transforms/day/cases.json b/table-spec/partition-transforms/day/cases.json new file mode 100644 index 0000000..191fec8 --- /dev/null +++ b/table-spec/partition-transforms/day/cases.json @@ -0,0 +1,129 @@ +{ + "cases": [ + { + "id": "day-date", + "valid": true, + "input": { + "transform": "day", + "source_type": "date", + "values": [null, "1969-12-31", "1970-01-01", "2000-02-29", "2000-03-01", "2017-12-01"] + }, + "decoded": { + "result_types": ["date"], + "values": [null, "1969-12-31", "1970-01-01", "2000-02-29", "2000-03-01", "2017-12-01"] + }, + "clause": "Partition Transforms: extract a date day, as days from 1970-01-01; expected values are logical dates", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "day-timestamp", + "valid": true, + "input": { + "transform": "day", + "source_type": "timestamp", + "values": [ + null, + "1969-12-31T23:59:59.999999", + "1970-01-01T00:00:00", + "2000-02-29T23:59:59.999999", + "2000-03-01T00:00:00", + "2017-12-01T10:12:55.038194" + ] + }, + "decoded": { + "result_types": ["date"], + "values": [null, "1969-12-31", "1970-01-01", "2000-02-29", "2000-03-01", "2017-12-01"] + }, + "clause": "Partition Transforms: extract a timestamp day; expected values are logical dates", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "day-timestamptz", + "valid": true, + "input": { + "transform": "day", + "source_type": "timestamptz", + "values": [ + null, + "1969-12-31T23:59:59.999999Z", + "1970-01-01T00:00:00Z", + "2000-02-29T23:59:59.999999Z", + "2000-03-01T00:00:00Z", + "2017-12-01T10:12:55.038194Z", + "2017-12-01T12:12:55.038194+02:00", + "2017-11-30T23:30:00-10:00" + ] + }, + "decoded": { + "result_types": ["date"], + "values": [ + null, + "1969-12-31", + "1970-01-01", + "2000-02-29", + "2000-03-01", + "2017-12-01", + "2017-12-01", + "2017-12-01" + ] + }, + "clause": "Partition Transforms: timestamp-with-zone day uses the instant, including offset-equivalent values that cross a local date", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "day-timestamp-ns", + "valid": true, + "input": { + "transform": "day", + "source_type": "timestamp_ns", + "values": [ + null, + "1969-12-31T23:59:59.999999999", + "1970-01-01T00:00:00.000000000", + "2000-02-29T23:59:59.999999999", + "2000-03-01T00:00:00.000000000", + "2017-12-01T10:12:55.038194789" + ] + }, + "decoded": { + "result_types": ["date"], + "values": [null, "1969-12-31", "1970-01-01", "2000-02-29", "2000-03-01", "2017-12-01"] + }, + "clause": "Partition Transforms: extract a nanosecond timestamp day; expected values are logical dates", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "day-timestamptz-ns", + "valid": true, + "input": { + "transform": "day", + "source_type": "timestamptz_ns", + "values": [ + null, + "1969-12-31T23:59:59.999999999Z", + "1970-01-01T00:00:00.000000000Z", + "2000-02-29T23:59:59.999999999Z", + "2000-03-01T00:00:00.000000000Z", + "2017-12-01T10:12:55.038194789Z", + "2017-12-01T12:12:55.038194789+02:00", + "2017-11-30T23:30:00.000000001-10:00" + ] + }, + "decoded": { + "result_types": ["date"], + "values": [ + null, + "1969-12-31", + "1970-01-01", + "2000-02-29", + "2000-03-01", + "2017-12-01", + "2017-12-01", + "2017-12-01" + ] + }, + "clause": "Partition Transforms: zoned nanosecond timestamp day uses the instant across offsets", + "spec_ref": "format/spec.md#partition-transforms" + } + ] +} diff --git a/table-spec/partition-transforms/hour/cases.json b/table-spec/partition-transforms/hour/cases.json new file mode 100644 index 0000000..f75fb44 --- /dev/null +++ b/table-spec/partition-transforms/hour/cases.json @@ -0,0 +1,84 @@ +{ + "cases": [ + { + "id": "hour-timestamp", + "valid": true, + "input": { + "transform": "hour", + "source_type": "timestamp", + "values": [ + null, + "1969-12-31T23:59:59.999999", + "1970-01-01T00:00:00", + "2000-02-29T23:59:59.999999", + "2000-03-01T00:00:00", + "2017-12-01T10:12:55.038194" + ] + }, + "decoded": { "result_types": ["int"], "values": [null, -1, 0, 264407, 264408, 420034] }, + "clause": "Partition Transforms: extract a timestamp hour, as hours from 1970-01-01 00:00:00", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "hour-timestamptz", + "valid": true, + "input": { + "transform": "hour", + "source_type": "timestamptz", + "values": [ + null, + "1969-12-31T23:59:59.999999Z", + "1970-01-01T00:00:00Z", + "2000-02-29T23:59:59.999999Z", + "2000-03-01T00:00:00Z", + "2017-12-01T10:12:55.038194Z", + "2017-12-01T12:12:55.038194+02:00", + "2017-11-30T23:30:00-10:00" + ] + }, + "decoded": { "result_types": ["int"], "values": [null, -1, 0, 264407, 264408, 420034, 420034, 420033] }, + "clause": "Partition Transforms: timestamp-with-zone hour uses the instant across equivalent offsets and a local-date crossing", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "hour-timestamp-ns", + "valid": true, + "input": { + "transform": "hour", + "source_type": "timestamp_ns", + "values": [ + null, + "1969-12-31T23:59:59.999999999", + "1970-01-01T00:00:00.000000000", + "2000-02-29T23:59:59.999999999", + "2000-03-01T00:00:00.000000000", + "2017-12-01T10:12:55.038194789" + ] + }, + "decoded": { "result_types": ["int"], "values": [null, -1, 0, 264407, 264408, 420034] }, + "clause": "Partition Transforms: extract a nanosecond timestamp hour, as hours from the Unix epoch", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "hour-timestamptz-ns", + "valid": true, + "input": { + "transform": "hour", + "source_type": "timestamptz_ns", + "values": [ + null, + "1969-12-31T23:59:59.999999999Z", + "1970-01-01T00:00:00.000000000Z", + "2000-02-29T23:59:59.999999999Z", + "2000-03-01T00:00:00.000000000Z", + "2017-12-01T10:12:55.038194789Z", + "2017-12-01T12:12:55.038194789+02:00", + "2017-11-30T23:30:00.000000001-10:00" + ] + }, + "decoded": { "result_types": ["int"], "values": [null, -1, 0, 264407, 264408, 420034, 420034, 420033] }, + "clause": "Partition Transforms: zoned nanosecond timestamp hour uses the instant across offsets", + "spec_ref": "format/spec.md#partition-transforms" + } + ] +} diff --git a/table-spec/partition-transforms/identity/cases.json b/table-spec/partition-transforms/identity/cases.json new file mode 100644 index 0000000..875b93f --- /dev/null +++ b/table-spec/partition-transforms/identity/cases.json @@ -0,0 +1,207 @@ +{ + "cases": [ + { + "id": "identity-unknown", + "valid": true, + "input": { "transform": "identity", "source_type": "unknown", "values": [null] }, + "decoded": { "result_types": ["unknown"], "values": [null] }, + "clause": "Partition Transforms: identity is the source value, unmodified; null propagation", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "identity-boolean", + "valid": true, + "input": { "transform": "identity", "source_type": "boolean", "values": [null, false, true] }, + "decoded": { "result_types": ["boolean"], "values": [null, false, true] }, + "clause": "Partition Transforms: identity is the source value, unmodified", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "identity-int", + "valid": true, + "input": { "transform": "identity", "source_type": "int", "values": [null, -34, 0, 34] }, + "decoded": { "result_types": ["int"], "values": [null, -34, 0, 34] }, + "clause": "Partition Transforms: identity is the source value, unmodified", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "identity-long", + "valid": true, + "input": { + "transform": "identity", + "source_type": "long", + "values": [null, "-9223372036854775808", "0", "9223372036854775807"] + }, + "decoded": { "result_types": ["long"], "values": [null, "-9223372036854775808", "0", "9223372036854775807"] }, + "clause": "Partition Transforms: identity is the source value, unmodified", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "identity-float", + "valid": true, + "input": { "transform": "identity", "source_type": "float", "values": [null, -1.5, 0, 1.5] }, + "decoded": { "result_types": ["float"], "values": [null, -1.5, 0, 1.5] }, + "clause": "Partition Transforms: identity is the source value, unmodified", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "identity-double", + "valid": true, + "input": { "transform": "identity", "source_type": "double", "values": [null, -1.25, 0, 1.25] }, + "decoded": { "result_types": ["double"], "values": [null, -1.25, 0, 1.25] }, + "clause": "Partition Transforms: identity is the source value, unmodified", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "identity-decimal-9-2", + "valid": true, + "input": { "transform": "identity", "source_type": "decimal(9, 2)", "values": [null, "-14.20", "0.00", "14.20"] }, + "decoded": { "result_types": ["decimal(9, 2)"], "values": [null, "-14.20", "0.00", "14.20"] }, + "clause": "Partition Transforms: identity preserves the decimal value, scale, and source type", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "identity-date", + "valid": true, + "input": { + "transform": "identity", + "source_type": "date", + "values": [null, "1969-12-31", "1970-01-01", "2000-02-29"] + }, + "decoded": { "result_types": ["date"], "values": [null, "1969-12-31", "1970-01-01", "2000-02-29"] }, + "clause": "Partition Transforms: identity is the source value, unmodified", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "identity-time", + "valid": true, + "input": { + "transform": "identity", + "source_type": "time", + "values": [null, "00:00:00", "22:31:08", "22:31:08.000001"] + }, + "decoded": { "result_types": ["time"], "values": [null, "00:00:00", "22:31:08", "22:31:08.000001"] }, + "clause": "Partition Transforms: identity is the source value, unmodified", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "identity-timestamp", + "valid": true, + "input": { + "transform": "identity", + "source_type": "timestamp", + "values": [null, "1969-12-31T23:59:59.999999", "1970-01-01T00:00:00", "2017-11-16T22:31:08.000001"] + }, + "decoded": { + "result_types": ["timestamp"], + "values": [null, "1969-12-31T23:59:59.999999", "1970-01-01T00:00:00", "2017-11-16T22:31:08.000001"] + }, + "clause": "Partition Transforms: identity is the source value, unmodified", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "identity-timestamptz", + "valid": true, + "input": { + "transform": "identity", + "source_type": "timestamptz", + "values": [null, "1970-01-01T00:00:00Z", "2017-11-16T14:31:08.000001-08:00", "2017-11-16T22:31:08.000001Z"] + }, + "decoded": { + "result_types": ["timestamptz"], + "values": [null, "1970-01-01T00:00:00Z", "2017-11-16T14:31:08.000001-08:00", "2017-11-16T22:31:08.000001Z"] + }, + "clause": "Partition Transforms: identity preserves timestamp-with-zone instants and source type", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "identity-timestamp-ns", + "valid": true, + "input": { + "transform": "identity", + "source_type": "timestamp_ns", + "values": [ + null, + "1969-12-31T23:59:59.999999999", + "1970-01-01T00:00:00.000000000", + "2017-11-16T22:31:08.000001001" + ] + }, + "decoded": { + "result_types": ["timestamp_ns"], + "values": [ + null, + "1969-12-31T23:59:59.999999999", + "1970-01-01T00:00:00.000000000", + "2017-11-16T22:31:08.000001001" + ] + }, + "clause": "Partition Transforms: identity preserves nanosecond precision and source type", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "identity-timestamptz-ns", + "valid": true, + "input": { + "transform": "identity", + "source_type": "timestamptz_ns", + "values": [ + null, + "1970-01-01T00:00:00.000000000Z", + "2017-11-16T14:31:08.000001001-08:00", + "2017-11-16T22:31:08.000001001Z" + ] + }, + "decoded": { + "result_types": ["timestamptz_ns"], + "values": [ + null, + "1970-01-01T00:00:00.000000000Z", + "2017-11-16T14:31:08.000001001-08:00", + "2017-11-16T22:31:08.000001001Z" + ] + }, + "clause": "Partition Transforms: identity preserves nanosecond timestamp-with-zone instants and source type", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "identity-string", + "valid": true, + "input": { "transform": "identity", "source_type": "string", "values": [null, "", "iceberg", "a😀雪"] }, + "decoded": { "result_types": ["string"], "values": [null, "", "iceberg", "a😀雪"] }, + "clause": "Partition Transforms: identity is the source value, unmodified", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "identity-uuid", + "valid": true, + "input": { + "transform": "identity", + "source_type": "uuid", + "values": [null, "00000000-0000-0000-0000-000000000000", "f79c3e09-677c-4bbd-a479-3f349cb785e7"] + }, + "decoded": { + "result_types": ["uuid"], + "values": [null, "00000000-0000-0000-0000-000000000000", "f79c3e09-677c-4bbd-a479-3f349cb785e7"] + }, + "clause": "Partition Transforms: identity is the source value, unmodified", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "identity-fixed-4", + "valid": true, + "input": { "transform": "identity", "source_type": "fixed[4]", "values": [null, "00010203", "FFFFFFFF"] }, + "decoded": { "result_types": ["fixed[4]"], "values": [null, "00010203", "FFFFFFFF"] }, + "clause": "Partition Transforms: identity preserves fixed bytes and source type", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "identity-binary", + "valid": true, + "input": { "transform": "identity", "source_type": "binary", "values": [null, "", "00010203", "FFFFFFFF"] }, + "decoded": { "result_types": ["binary"], "values": [null, "", "00010203", "FFFFFFFF"] }, + "clause": "Partition Transforms: identity preserves binary bytes and source type", + "spec_ref": "format/spec.md#partition-transforms" + } + ] +} diff --git a/table-spec/partition-transforms/month/cases.json b/table-spec/partition-transforms/month/cases.json new file mode 100644 index 0000000..30fa378 --- /dev/null +++ b/table-spec/partition-transforms/month/cases.json @@ -0,0 +1,96 @@ +{ + "cases": [ + { + "id": "month-date", + "valid": true, + "input": { + "transform": "month", + "source_type": "date", + "values": [null, "1969-12-31", "1970-01-01", "2000-02-29", "2000-03-01", "2017-12-01"] + }, + "decoded": { "result_types": ["int"], "values": [null, -1, 0, 361, 362, 575] }, + "clause": "Partition Transforms: extract a date month, as months from 1970-01-01", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "month-timestamp", + "valid": true, + "input": { + "transform": "month", + "source_type": "timestamp", + "values": [ + null, + "1969-12-31T23:59:59.999999", + "1970-01-01T00:00:00", + "2000-02-29T23:59:59.999999", + "2000-03-01T00:00:00", + "2017-12-01T10:12:55.038194" + ] + }, + "decoded": { "result_types": ["int"], "values": [null, -1, 0, 361, 362, 575] }, + "clause": "Partition Transforms: extract a timestamp month, as months from 1970-01-01", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "month-timestamptz", + "valid": true, + "input": { + "transform": "month", + "source_type": "timestamptz", + "values": [ + null, + "1969-12-31T23:59:59.999999Z", + "1970-01-01T00:00:00Z", + "2000-02-29T23:59:59.999999Z", + "2000-03-01T00:00:00Z", + "2017-12-01T10:12:55.038194Z", + "2017-12-01T12:12:55.038194+02:00", + "2017-11-30T23:30:00-10:00" + ] + }, + "decoded": { "result_types": ["int"], "values": [null, -1, 0, 361, 362, 575, 575, 575] }, + "clause": "Partition Transforms: timestamp-with-zone month uses the instant across offset-equivalent values and date boundaries", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "month-timestamp-ns", + "valid": true, + "input": { + "transform": "month", + "source_type": "timestamp_ns", + "values": [ + null, + "1969-12-31T23:59:59.999999999", + "1970-01-01T00:00:00.000000000", + "2000-02-29T23:59:59.999999999", + "2000-03-01T00:00:00.000000000", + "2017-12-01T10:12:55.038194789" + ] + }, + "decoded": { "result_types": ["int"], "values": [null, -1, 0, 361, 362, 575] }, + "clause": "Partition Transforms: extract a nanosecond timestamp month, as months from 1970-01-01", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "month-timestamptz-ns", + "valid": true, + "input": { + "transform": "month", + "source_type": "timestamptz_ns", + "values": [ + null, + "1969-12-31T23:59:59.999999999Z", + "1970-01-01T00:00:00.000000000Z", + "2000-02-29T23:59:59.999999999Z", + "2000-03-01T00:00:00.000000000Z", + "2017-12-01T10:12:55.038194789Z", + "2017-12-01T12:12:55.038194789+02:00", + "2017-11-30T23:30:00.000000001-10:00" + ] + }, + "decoded": { "result_types": ["int"], "values": [null, -1, 0, 361, 362, 575, 575, 575] }, + "clause": "Partition Transforms: zoned nanosecond timestamp month uses the instant across offsets", + "spec_ref": "format/spec.md#partition-transforms" + } + ] +} diff --git a/table-spec/partition-transforms/truncate/cases.json b/table-spec/partition-transforms/truncate/cases.json new file mode 100644 index 0000000..dda10f4 --- /dev/null +++ b/table-spec/partition-transforms/truncate/cases.json @@ -0,0 +1,75 @@ +{ + "cases": [ + { + "id": "truncate-int-width-10", + "valid": true, + "input": { + "transform": "truncate[10]", + "source_type": "int", + "values": [null, 0, 1, 5, 9, 10, 11, -1, -5, -10, -11] + }, + "decoded": { "result_types": ["int"], "values": [null, 0, 0, 0, 0, 10, 10, -10, -10, -10, -20] }, + "clause": "Truncate Transform Details: int v - (v % W), with positive remainders for negative values", + "spec_ref": "format/spec.md#truncate-transform-details" + }, + { + "id": "truncate-long-width-10", + "valid": true, + "input": { + "transform": "truncate[10]", + "source_type": "long", + "values": [null, "0", "1", "5", "9", "10", "11", "-1", "-5", "-10", "-11"] + }, + "decoded": { + "result_types": ["long"], + "values": [null, "0", "0", "0", "0", "10", "10", "-10", "-10", "-10", "-20"] + }, + "clause": "Truncate Transform Details: long v - (v % W), with positive remainders for negative values", + "spec_ref": "format/spec.md#truncate-transform-details" + }, + { + "id": "truncate-decimal-9-2-width-10", + "valid": true, + "input": { + "transform": "truncate[10]", + "source_type": "decimal(9, 2)", + "values": [null, "12.34", "12.30", "12.29", "0.05", "-0.05"] + }, + "decoded": { "result_types": ["decimal(9, 2)"], "values": [null, "12.30", "12.30", "12.20", "0.00", "-0.10"] }, + "clause": "Truncate Transform Details: decimal width is applied using the scale of the decimal column and remainders are positive", + "spec_ref": "format/spec.md#truncate-transform-details" + }, + { + "id": "truncate-decimal-9-2-width-50", + "valid": true, + "input": { "transform": "truncate[50]", "source_type": "decimal(9, 2)", "values": [null, "10.65"] }, + "decoded": { "result_types": ["decimal(9, 2)"], "values": [null, "10.50"] }, + "clause": "Truncate Transform Details example: W=50, scale=2 truncates 10.65 to 10.50", + "spec_ref": "format/spec.md#truncate-transform-details" + }, + { + "id": "truncate-string-length-3", + "valid": true, + "input": { + "transform": "truncate[3]", + "source_type": "string", + "values": [null, "", "ice", "iceberg", "a😀bcd", "😀😀😀😀"] + }, + "decoded": { "result_types": ["string"], "values": [null, "", "ice", "ice", "a😀b", "😀😀😀"] }, + "clause": "Truncate Transform Details: strings are truncated to no more than L Unicode code points", + "spec_ref": "format/spec.md#truncate-transform-details" + }, + { + "id": "truncate-binary-length-3", + "valid": true, + "input": { + "transform": "truncate[3]", + "source_type": "binary", + "values": [null, "", "0001", "000102", "0001020304", "F09F9880"] + }, + "decoded": { "result_types": ["binary"], "values": [null, "", "0001", "000102", "000102", "F09F98"] }, + "clause": "Truncate Transform Details: binary values have no assumed encoding and are truncated to L bytes", + "spec_ref": "format/spec.md#truncate-transform-details" + } + ] +} diff --git a/table-spec/partition-transforms/void/cases.json b/table-spec/partition-transforms/void/cases.json new file mode 100644 index 0000000..54bd3b3 --- /dev/null +++ b/table-spec/partition-transforms/void/cases.json @@ -0,0 +1,168 @@ +{ + "cases": [ + { + "id": "void-unknown", + "valid": true, + "input": { "transform": "void", "source_type": "unknown", "values": [null] }, + "decoded": { "result_types": ["unknown", "int"], "values": [null] }, + "clause": "Partition Transforms: void always produces null; result type is source type or int", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "void-boolean", + "valid": true, + "input": { "transform": "void", "source_type": "boolean", "values": [null, true] }, + "decoded": { "result_types": ["boolean", "int"], "values": [null, null] }, + "clause": "Partition Transforms: void always produces null; result type is source type or int", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "void-int", + "valid": true, + "input": { "transform": "void", "source_type": "int", "values": [null, 34] }, + "decoded": { "result_types": ["int"], "values": [null, null] }, + "clause": "Partition Transforms: void always produces null; source type and int are the same allowed result type", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "void-long", + "valid": true, + "input": { "transform": "void", "source_type": "long", "values": [null, "34"] }, + "decoded": { "result_types": ["long", "int"], "values": [null, null] }, + "clause": "Partition Transforms: void always produces null; result type is source type or int", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "void-float", + "valid": true, + "input": { "transform": "void", "source_type": "float", "values": [null, 1.5] }, + "decoded": { "result_types": ["float", "int"], "values": [null, null] }, + "clause": "Partition Transforms: void always produces null; result type is source type or int", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "void-double", + "valid": true, + "input": { "transform": "void", "source_type": "double", "values": [null, 1.25] }, + "decoded": { "result_types": ["double", "int"], "values": [null, null] }, + "clause": "Partition Transforms: void always produces null; result type is source type or int", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "void-decimal-9-2", + "valid": true, + "input": { "transform": "void", "source_type": "decimal(9, 2)", "values": [null, "14.20"] }, + "decoded": { "result_types": ["decimal(9, 2)", "int"], "values": [null, null] }, + "clause": "Partition Transforms: void always produces null; result type is source type or int", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "void-date", + "valid": true, + "input": { "transform": "void", "source_type": "date", "values": [null, "2017-11-16"] }, + "decoded": { "result_types": ["date", "int"], "values": [null, null] }, + "clause": "Partition Transforms: void always produces null; result type is source type or int", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "void-time", + "valid": true, + "input": { "transform": "void", "source_type": "time", "values": [null, "22:31:08"] }, + "decoded": { "result_types": ["time", "int"], "values": [null, null] }, + "clause": "Partition Transforms: void always produces null; result type is source type or int", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "void-timestamp", + "valid": true, + "input": { "transform": "void", "source_type": "timestamp", "values": [null, "2017-11-16T22:31:08.000001"] }, + "decoded": { "result_types": ["timestamp", "int"], "values": [null, null] }, + "clause": "Partition Transforms: void always produces null; result type is source type or int", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "void-timestamptz", + "valid": true, + "input": { + "transform": "void", + "source_type": "timestamptz", + "values": [null, "2017-11-16T14:31:08.000001-08:00"] + }, + "decoded": { "result_types": ["timestamptz", "int"], "values": [null, null] }, + "clause": "Partition Transforms: void always produces null; result type is source type or int", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "void-timestamp-ns", + "valid": true, + "input": { + "transform": "void", + "source_type": "timestamp_ns", + "values": [null, "2017-11-16T22:31:08.000001001"] + }, + "decoded": { "result_types": ["timestamp_ns", "int"], "values": [null, null] }, + "clause": "Partition Transforms: void always produces null; result type is source type or int", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "void-timestamptz-ns", + "valid": true, + "input": { + "transform": "void", + "source_type": "timestamptz_ns", + "values": [null, "2017-11-16T14:31:08.000001001-08:00"] + }, + "decoded": { "result_types": ["timestamptz_ns", "int"], "values": [null, null] }, + "clause": "Partition Transforms: void always produces null; result type is source type or int", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "void-string", + "valid": true, + "input": { "transform": "void", "source_type": "string", "values": [null, "iceberg"] }, + "decoded": { "result_types": ["string", "int"], "values": [null, null] }, + "clause": "Partition Transforms: void always produces null; result type is source type or int", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "void-uuid", + "valid": true, + "input": { "transform": "void", "source_type": "uuid", "values": [null, "f79c3e09-677c-4bbd-a479-3f349cb785e7"] }, + "decoded": { "result_types": ["uuid", "int"], "values": [null, null] }, + "clause": "Partition Transforms: void always produces null; result type is source type or int", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "void-fixed-4", + "valid": true, + "input": { "transform": "void", "source_type": "fixed[4]", "values": [null, "00010203"] }, + "decoded": { "result_types": ["fixed[4]", "int"], "values": [null, null] }, + "clause": "Partition Transforms: void always produces null; result type is source type or int", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "void-binary", + "valid": true, + "input": { "transform": "void", "source_type": "binary", "values": [null, "00010203"] }, + "decoded": { "result_types": ["binary", "int"], "values": [null, null] }, + "clause": "Partition Transforms: void always produces null; result type is source type or int", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "void-geometry", + "valid": true, + "input": { "transform": "void", "source_type": "geometry(OGC:CRS84)", "values": [null] }, + "decoded": { "result_types": ["geometry(OGC:CRS84)", "int"], "values": [null] }, + "clause": "Partition Transforms: void accepts any source type and always produces null", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "void-geography", + "valid": true, + "input": { "transform": "void", "source_type": "geography(OGC:CRS84, spherical)", "values": [null] }, + "decoded": { "result_types": ["geography(OGC:CRS84, spherical)", "int"], "values": [null] }, + "clause": "Partition Transforms: void accepts any source type and always produces null", + "spec_ref": "format/spec.md#partition-transforms" + } + ] +} diff --git a/table-spec/partition-transforms/year/cases.json b/table-spec/partition-transforms/year/cases.json new file mode 100644 index 0000000..26c4327 --- /dev/null +++ b/table-spec/partition-transforms/year/cases.json @@ -0,0 +1,96 @@ +{ + "cases": [ + { + "id": "year-date", + "valid": true, + "input": { + "transform": "year", + "source_type": "date", + "values": [null, "1969-12-31", "1970-01-01", "2000-02-29", "2000-03-01", "2017-12-01"] + }, + "decoded": { "result_types": ["int"], "values": [null, -1, 0, 30, 30, 47] }, + "clause": "Partition Transforms: extract a date year, as years from 1970", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "year-timestamp", + "valid": true, + "input": { + "transform": "year", + "source_type": "timestamp", + "values": [ + null, + "1969-12-31T23:59:59.999999", + "1970-01-01T00:00:00", + "2000-02-29T23:59:59.999999", + "2000-03-01T00:00:00", + "2017-12-01T10:12:55.038194" + ] + }, + "decoded": { "result_types": ["int"], "values": [null, -1, 0, 30, 30, 47] }, + "clause": "Partition Transforms: extract a timestamp year, as years from 1970", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "year-timestamptz", + "valid": true, + "input": { + "transform": "year", + "source_type": "timestamptz", + "values": [ + null, + "1969-12-31T23:59:59.999999Z", + "1970-01-01T00:00:00Z", + "2000-02-29T23:59:59.999999Z", + "2000-03-01T00:00:00Z", + "2017-12-01T10:12:55.038194Z", + "2017-12-01T12:12:55.038194+02:00", + "2017-11-30T23:30:00-10:00" + ] + }, + "decoded": { "result_types": ["int"], "values": [null, -1, 0, 30, 30, 47, 47, 47] }, + "clause": "Partition Transforms: timestamp-with-zone year uses the instant; equivalent instants and an offset date crossing are included", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "year-timestamp-ns", + "valid": true, + "input": { + "transform": "year", + "source_type": "timestamp_ns", + "values": [ + null, + "1969-12-31T23:59:59.999999999", + "1970-01-01T00:00:00.000000000", + "2000-02-29T23:59:59.999999999", + "2000-03-01T00:00:00.000000000", + "2017-12-01T10:12:55.038194789" + ] + }, + "decoded": { "result_types": ["int"], "values": [null, -1, 0, 30, 30, 47] }, + "clause": "Partition Transforms: extract a nanosecond timestamp year, as years from 1970", + "spec_ref": "format/spec.md#partition-transforms" + }, + { + "id": "year-timestamptz-ns", + "valid": true, + "input": { + "transform": "year", + "source_type": "timestamptz_ns", + "values": [ + null, + "1969-12-31T23:59:59.999999999Z", + "1970-01-01T00:00:00.000000000Z", + "2000-02-29T23:59:59.999999999Z", + "2000-03-01T00:00:00.000000000Z", + "2017-12-01T10:12:55.038194789Z", + "2017-12-01T12:12:55.038194789+02:00", + "2017-11-30T23:30:00.000000001-10:00" + ] + }, + "decoded": { "result_types": ["int"], "values": [null, -1, 0, 30, 30, 47, 47, 47] }, + "clause": "Partition Transforms: zoned nanosecond timestamp year uses the instant across offsets", + "spec_ref": "format/spec.md#partition-transforms" + } + ] +}