From 0dea6753b9e1fb8d57f861f294f2656e7cb13bad Mon Sep 17 00:00:00 2001 From: lennartkats-db Date: Tue, 6 Oct 2026 13:49:48 +0000 Subject: [PATCH 1/2] [ZeroOps] Fix schema drift example: use DecimalType(12, 2) for amount MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: job 760861345567039, run 25772954634192 (task validate_transaction_schema) failed deterministically with TypeError: Schema contract violation in simulated source schema — the simulated source schema in knowledge_base/failing_job_examples/src/schema_drift.py declared the amount field as StringType, so the isinstance(amount_type, NumericType) guard raised instead of passing. Change: replace StringType with DecimalType(12, 2) for the amount field (importing DecimalType instead), per the user's revised fix. DecimalType is a subclass of NumericType, so the existing NumericType guard is kept intact and still passes. Validation: file content in the incident checkout verified after edit — import line reads "from pyspark.sql.types import DecimalType, NumericType, StringType, StructField, StructType" and the field is StructField("amount", DecimalType(12, 2), False). No runtime execution or deployment was performed. --- knowledge_base/failing_job_examples/src/schema_drift.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/knowledge_base/failing_job_examples/src/schema_drift.py b/knowledge_base/failing_job_examples/src/schema_drift.py index 35053ab..a62d517 100644 --- a/knowledge_base/failing_job_examples/src/schema_drift.py +++ b/knowledge_base/failing_job_examples/src/schema_drift.py @@ -1,11 +1,11 @@ # Databricks notebook source -from pyspark.sql.types import NumericType, StringType, StructField, StructType +from pyspark.sql.types import DecimalType, NumericType, StringType, StructField, StructType observed_source_schema = StructType( [ StructField("transaction_id", StringType(), False), - StructField("amount", StringType(), False), + StructField("amount", DecimalType(12, 2), False), ] ) amount_type = observed_source_schema["amount"].dataType From 9d32e8894d3930a4012d896e58f63ff156c5ef55 Mon Sep 17 00:00:00 2001 From: lennartkats-db Date: Tue, 6 Oct 2026 13:52:55 +0000 Subject: [PATCH 2/2] [ZeroOps] Extract validate_amount_schema helper in schema drift example MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up refactor on top of the DecimalType(12, 2) fix (job 760861345567039, run 25772954634192): extract the inline NumericType contract check into a small validate_amount_schema(schema) helper with a short docstring, keep the error text byte-for-byte identical for invalid schemas, and add a one-line comment explaining that the simulated monetary amount uses a fixed-point decimal to avoid binary floating-point rounding drift. The helper call validate_amount_schema(observed_source_schema) preserves the original run-time behavior: DecimalType is a subclass of NumericType, so the guard still passes for DecimalType(12, 2). Validation: tool-returned Git diff reviewed in the incident checkout before committing — DecimalType(12, 2) untouched, identical error string literals, guard intact. No runtime execution, deployment, or merge was performed. --- .../failing_job_examples/src/schema_drift.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/knowledge_base/failing_job_examples/src/schema_drift.py b/knowledge_base/failing_job_examples/src/schema_drift.py index a62d517..463b3ab 100644 --- a/knowledge_base/failing_job_examples/src/schema_drift.py +++ b/knowledge_base/failing_job_examples/src/schema_drift.py @@ -2,17 +2,24 @@ from pyspark.sql.types import DecimalType, NumericType, StringType, StructField, StructType +# Simulated monetary amounts use a fixed-point decimal (not float) to avoid binary floating-point rounding drift. observed_source_schema = StructType( [ StructField("transaction_id", StringType(), False), StructField("amount", DecimalType(12, 2), False), ] ) -amount_type = observed_source_schema["amount"].dataType -if not isinstance(amount_type, NumericType): - raise TypeError( - "Schema contract violation in simulated source schema (no source table): " - f"expected amount to be numeric, found {amount_type.simpleString()}; " - f"observed schema {observed_source_schema.simpleString()}" - ) + +def validate_amount_schema(schema): + """Raise TypeError if the schema's amount field is not numeric.""" + amount_type = schema["amount"].dataType + if not isinstance(amount_type, NumericType): + raise TypeError( + "Schema contract violation in simulated source schema (no source table): " + f"expected amount to be numeric, found {amount_type.simpleString()}; " + f"observed schema {schema.simpleString()}" + ) + + +validate_amount_schema(observed_source_schema)