build: upgrade pyspark

pyspark from 3.4 -> 3.5.2 and delta spark 2.4 -> 3.2.0

Author: georgeRobertson2

poetry.lock

@@ -33,7 +33,7 @@ packages = [
 python = ">=3.10,<3.13"
 boto3 = ">=1.34.162,<1.36"  # breaking change beyond 1.36
 botocore = ">=1.34.162,<1.36"  # breaking change beyond 1.36
-delta-spark = "2.4.*"
+delta-spark = "3.2.0"
 duckdb = "1.1.*"  # breaking changes beyond 1.1
 Jinja2 = "3.1.*"
 lxml = "^4.9.1"
@@ -43,7 +43,7 @@ pandas = "^2.2.2"
 polars = "0.20.*"
 pyarrow = "^17.0.0"
 pydantic = "1.10.16"
-pyspark = "3.4.*"
+pyspark = "3.5.2"
 typing_extensions = "^4.6.2"
 
 [tool.poetry.group.dev]

pyproject.toml

@@ -89,7 +89,7 @@ def get_spark_session() -> SparkSession:
     os.environ["PYSPARK_SUBMIT_ARGS"] = " ".join(
         [
             "--packages",
-            "com.databricks:spark-xml_2.12:0.16.0,io.delta:delta-core_2.12:2.4.0",
+            "com.databricks:spark-xml_2.12:0.16.0,io.delta:delta-spark_2.12:3.2.0",
             "pyspark-shell",
         ]
     )