Skip to content
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 16 additions & 9 deletions knowledge_base/failing_job_examples/src/schema_drift.py
Original file line number Diff line number Diff line change
@@ -1,18 +1,25 @@
# Databricks notebook source
from pyspark.sql.types import NumericType, StringType, StructField, StructType
from pyspark.sql.types import DecimalType, NumericType, StringType, StructField, StructType


# Simulated monetary amounts use a fixed-point decimal (not float) to avoid binary floating-point rounding drift.
observed_source_schema = StructType(
[
StructField("transaction_id", StringType(), False),
StructField("amount", StringType(), False),
StructField("amount", DecimalType(12, 2), False),
]
)
amount_type = observed_source_schema["amount"].dataType

if not isinstance(amount_type, NumericType):
raise TypeError(
"Schema contract violation in simulated source schema (no source table): "
f"expected amount to be numeric, found {amount_type.simpleString()}; "
f"observed schema {observed_source_schema.simpleString()}"
)

def validate_amount_schema(schema):
"""Raise TypeError if the schema's amount field is not numeric."""
amount_type = schema["amount"].dataType
if not isinstance(amount_type, NumericType):
raise TypeError(
"Schema contract violation in simulated source schema (no source table): "
f"expected amount to be numeric, found {amount_type.simpleString()}; "
f"observed schema {schema.simpleString()}"
)


validate_amount_schema(observed_source_schema)
Loading