Fix #27079: [BUG] S3 Ingestion Failure: Pydantic ValidationError 'ext… (#27180)

* Fix #27079: [BUG] S3 Ingestion Failure: Pydantic ValidationError 'extra_forbidden' for BucketArn in S3BucketResponse

* Address comments

Author: harshach

ingestion/src/metadata/ingestion/source/mlmodel/sagemaker/metadata.py

@@ -13,7 +13,7 @@
 import traceback
 from typing import Iterable, List, Optional
 
-from pydantic import BaseModel, ConfigDict, Field, ValidationError
+from pydantic import BaseModel, Field, ValidationError
 
 from metadata.generated.schema.api.data.createMlModel import CreateMlModelRequest
 from metadata.generated.schema.entity.data.mlmodel import (
@@ -53,10 +53,6 @@
 
 
 class SageMakerModel(BaseModel):
-    model_config = ConfigDict(
-        extra="forbid",
-    )
-
     name: str = Field(..., description="Model name", title="Model Name")
     arn: str = Field(..., description="Model ARN in AWS account", title="Model ARN")
     creation_timestamp: str = Field(

ingestion/src/metadata/ingestion/source/storage/gcs/models.py

@@ -14,7 +14,7 @@
 from datetime import datetime
 from typing import List, Optional
 
-from pydantic import BaseModel, Extra, Field
+from pydantic import BaseModel, ConfigDict, Field
 
 from metadata.generated.schema.entity.data.container import (
     ContainerDataModel,
@@ -29,9 +29,6 @@ class GCSBucketResponse(BaseModel):
     Class modelling a response received from gcs_client.list_buckets operation
     """
 
-    class Config:
-        extra = Extra.forbid
-
     name: str = Field(..., description="Bucket name")
     project_id: str = Field(..., description="Project ID")
     creation_date: Optional[datetime] = Field(
@@ -45,8 +42,9 @@ class GCSContainerDetails(BaseModel):
     Class mapping container details used to create the container requests
     """
 
-    class Config:
-        extra = Extra.forbid
+    model_config = ConfigDict(
+        extra="forbid",
+    )
 
     name: str = Field(..., description="Bucket name")
     prefix: str = Field(..., description="Prefix for the container")

ingestion/src/metadata/ingestion/source/storage/s3/models.py

@@ -29,10 +29,6 @@ class S3BucketResponse(BaseModel):
     Class modelling a response received from s3_client.list_buckets operation
     """
 
-    model_config = ConfigDict(
-        extra="forbid",
-    )
-
     name: str = Field(..., description="Bucket name", alias="Name")
     creation_date: Optional[datetime] = Field(
         None,

ingestion/tests/unit/topology/storage/test_s3_storage.py

@@ -83,9 +83,24 @@
 }
 MOCK_S3_BUCKETS_RESPONSE = {
     "Buckets": [
-        {"Name": "test_transactions", "CreationDate": datetime.datetime(2000, 1, 1)},
-        {"Name": "test_sales", "CreationDate": datetime.datetime(2000, 2, 2)},
-        {"Name": "events", "CreationDate": datetime.datetime(2000, 3, 3)},
+        {
+            "Name": "test_transactions",
+            "CreationDate": datetime.datetime(2000, 1, 1),
+            "BucketArn": "arn:aws:s3:::test_transactions",
+            "BucketRegion": "us-east-1",
+        },
+        {
+            "Name": "test_sales",
+            "CreationDate": datetime.datetime(2000, 2, 2),
+            "BucketArn": "arn:aws:s3:::test_sales",
+            "BucketRegion": "us-east-1",
+        },
+        {
+            "Name": "events",
+            "CreationDate": datetime.datetime(2000, 3, 3),
+            "BucketArn": "arn:aws:s3:::events",
+            "BucketRegion": "us-east-1",
+        },
     ]
 }
 MOCK_S3_METADATA_FILE_RESPONSE = {