adding azure blob data loader

Author: Chenglong-MS

py-src/data_formulator/data_loader/__init__.py

@@ -2,11 +2,13 @@
 from data_formulator.data_loader.mysql_data_loader import MySQLDataLoader
 from data_formulator.data_loader.kusto_data_loader import KustoDataLoader
 from data_formulator.data_loader.s3_data_loader import S3DataLoader
+from data_formulator.data_loader.azure_blob_data_loader import AzureBlobDataLoader
 
 DATA_LOADERS = {
     "mysql": MySQLDataLoader,
     "kusto": KustoDataLoader,
     "s3": S3DataLoader,
+    "azure_blob": AzureBlobDataLoader,
 }
 
-__all__ = ["ExternalDataLoader", "MySQLDataLoader", "KustoDataLoader", "S3DataLoader", "DATA_LOADERS"]
+__all__ = ["ExternalDataLoader", "MySQLDataLoader", "KustoDataLoader", "S3DataLoader", "AzureBlobDataLoader", "DATA_LOADERS"]

py-src/data_formulator/data_loader/azure_blob_data_loader.py

@@ -67,6 +67,11 @@ def ingest_df_to_duckdb(self, df: pd.DataFrame, table_name: str):
     def list_params() -> List[Dict[str, Any]]:
         pass
 
+    @staticmethod
+    @abstractmethod
+    def auth_instructions() -> str:
+        pass
+
     @abstractmethod
     def __init__(self, params: Dict[str, Any], duck_db_conn: duckdb.DuckDBPyConnection):
         pass

py-src/data_formulator/data_loader/external_data_loader.py

@@ -23,6 +23,41 @@ def list_params() -> bool:
             {"name": "tenant_id", "type": "string", "required": False, "description": "only necessary for AppKey auth"}
         ]
         return params_list
+    
+    @staticmethod
+    def auth_instructions() -> str:
+        return """
+Azure Kusto Authentication Instructions:
+
+This data loader supports two authentication methods:
+
+**Method 1: Azure CLI Authentication (Recommended for development)**
+1. Install Azure CLI: https://docs.microsoft.com/en-us/cli/azure/install-azure-cli
+2. Run `az login` in your terminal to authenticate
+3. Ensure you have access to the specified Kusto cluster and database
+4. Leave client_id, client_secret, and tenant_id parameters empty
+
+**Method 2: Application Key Authentication (Recommended for production)**
+1. Register an Azure AD application in your tenant
+2. Generate a client secret for the application
+3. Grant the application appropriate permissions to your Kusto cluster:
+   - Go to your Kusto cluster in Azure Portal
+   - Navigate to Permissions > Add
+   - Add your application as a user with appropriate role (e.g., "AllDatabasesViewer" for read access)
+4. Provide the following parameters:
+   - client_id: Application (client) ID from your Azure AD app registration
+   - client_secret: Client secret value you generated
+   - tenant_id: Directory (tenant) ID from your Azure AD
+
+**Required Parameters:**
+- kusto_cluster: Your Kusto cluster URI (e.g., "https://mycluster.region.kusto.windows.net")
+- kusto_database: Name of the database you want to access
+
+**Troubleshooting:**
+- If authentication fails, ensure you have the correct permissions on the Kusto cluster
+- For CLI auth, make sure you're logged in with `az account show`
+- For app key auth, verify your client_id, client_secret, and tenant_id are correct
+        """
 
     def __init__(self, params: Dict[str, Any], duck_db_conn: duckdb.DuckDBPyConnection):
 

py-src/data_formulator/data_loader/kusto_data_loader.py

@@ -18,6 +18,38 @@ def list_params() -> bool:
         ]
         return params_list
 
+    @staticmethod
+    def auth_instructions() -> str:
+        return """
+MySQL Connection Instructions:
+
+1. **Local MySQL Setup:**
+   - Ensure MySQL server is running on your machine
+   - Default connection: host='localhost', user='root'
+   - If you haven't set a root password, leave password field empty
+
+2. **Remote MySQL Connection:**
+   - Obtain host address, username, and password from your database administrator
+   - Ensure the MySQL server allows remote connections
+   - Check that your IP is whitelisted in MySQL's user permissions
+
+3. **Common Connection Parameters:**
+   - user: Your MySQL username (default: 'root')
+   - password: Your MySQL password (leave empty if no password set)
+   - host: MySQL server address (default: 'localhost')
+   - database: Target database name to connect to
+
+4. **Troubleshooting:**
+   - Verify MySQL service is running: `brew services list` (macOS) or `sudo systemctl status mysql` (Linux)
+   - Test connection: `mysql -u [username] -p -h [host] [database]`
+   - Common issues: Wrong credentials, server not running, firewall blocking connection
+
+5. **Security Notes:**
+   - Use dedicated database users with limited privileges for applications
+   - Avoid using root user for application connections
+   - Consider using SSL connections for remote databases
+        """
+
     def __init__(self, params: Dict[str, Any], duck_db_conn: duckdb.DuckDBPyConnection):
         self.params = params
         self.duck_db_conn = duck_db_conn

py-src/data_formulator/data_loader/mysql_data_loader.py

@@ -19,6 +19,70 @@ def list_params() -> List[Dict[str, Any]]:
         ]
         return params_list
 
+    @staticmethod
+    def auth_instructions() -> str:
+        return """
+To connect to Amazon S3, you'll need the following AWS credentials and configuration:
+
+**Required Parameters:**
+- **AWS Access Key ID**: Your AWS access key identifier
+- **AWS Secret Access Key**: Your AWS secret access key  
+- **Region Name**: The AWS region where your S3 bucket is located (e.g., 'us-east-1', 'us-west-2')
+- **Bucket**: The name of your S3 bucket
+
+**Optional Parameters:**
+- **AWS Session Token**: Required only if using temporary credentials (e.g., from AWS STS or IAM roles)
+
+**How to Get AWS Credentials:**
+
+1. **AWS IAM User (Recommended for programmatic access):**
+   - Go to AWS Console → IAM → Users
+   - Create a new user or select existing user
+   - Go to "Security credentials" tab
+   - Click "Create access key"
+   - Choose "Application running outside AWS"
+   - Save both the Access Key ID and Secret Access Key securely
+
+2. **Required S3 Permissions:**
+   Your IAM user/role needs these permissions for the target bucket:
+   ```json
+   {
+     "Version": "2012-10-17",
+     "Statement": [
+       {
+         "Effect": "Allow",
+         "Action": [
+           "s3:GetObject",
+           "s3:ListBucket"
+         ],
+         "Resource": [
+           "arn:aws:s3:::your-bucket-name",
+           "arn:aws:s3:::your-bucket-name/*"
+         ]
+       }
+     ]
+   }
+   ```
+
+3. **Finding Your Region:**
+   - Go to S3 Console → Select your bucket → Properties
+   - Look for "AWS Region" in the bucket overview
+
+**Security Best Practices:**
+- Never share your secret access key
+- Use IAM roles when possible instead of long-term access keys
+- Consider using temporary credentials with session tokens for enhanced security
+- Regularly rotate your access keys
+- Use the principle of least privilege for S3 permissions
+
+**Supported File Formats:**
+- CSV files (.csv)
+- Parquet files (.parquet) 
+- JSON files (.json, .jsonl)
+
+The connector will automatically detect file types and load them appropriately using DuckDB's S3 integration.
+        """
+
     def __init__(self, params: Dict[str, Any], duck_db_conn: duckdb.DuckDBPyConnection):
         self.params = params
         self.duck_db_conn = duck_db_conn
@@ -120,32 +184,9 @@ def _estimate_row_count(self, s3_url: str) -> int:
                 count = self.duck_db_conn.execute(f"SELECT COUNT(*) FROM read_parquet('{s3_url}')").fetchone()[0]
                 return count
 
-            # For CSV files, we'll sample the file to estimate size
-            sample_size = 1000
-            sample_df = self.duck_db_conn.execute(f"SELECT * FROM read_csv_auto('{s3_url}') LIMIT {sample_size}").df()
-            
-            # Get file size from S3
-            import boto3
-            s3_client = boto3.client(
-                's3',
-                aws_access_key_id=self.aws_access_key_id,
-                aws_secret_access_key=self.aws_secret_access_key,
-                aws_session_token=self.aws_session_token if self.aws_session_token else None,
-                region_name=self.region_name
-            )
-            
-            key = s3_url.replace(f"s3://{self.bucket}/", "")
-            response = s3_client.head_object(Bucket=self.bucket, Key=key)
-            file_size = response['ContentLength']
-            
-            # Estimate based on sample size and file size
-            if len(sample_df) > 0:
-                # Calculate average row size in bytes
-                avg_row_size = file_size / len(sample_df)
-                estimated_rows = int(file_size / avg_row_size)
-                return min(estimated_rows, 1000000)  # Cap at 1 million for UI performance
-            
-            return 0
+            # For CSV, JSON, and JSONL files, we'll skip row count
+            if s3_url.lower().endswith('.csv') or s3_url.lower().endswith('.json') or s3_url.lower().endswith('.jsonl'):
+                return 0
         except Exception as e:
             print(f"Error estimating row count for {s3_url}: {e}")
             return 0

py-src/data_formulator/data_loader/s3_data_loader.py

@@ -653,7 +653,7 @@ export const DBTableSelectionDialog: React.FC<{ buttonElement: any }> = function
                     sx={{px: 0.5}}
                 >
                     <Typography variant="caption" sx={{color: "text.secondary", fontWeight: "bold", px: 1}}>connect external data</Typography>
-                    {["file upload", "mysql", "kusto","s3"].map((dataLoaderType, i) => (
+                    {["file upload", ...Object.keys(dataLoaderParamDefs ?? {})].map((dataLoaderType, i) => (
                         <Tab 
                             key={`dataLoader:${dataLoaderType}`} 
                             wrapped 
@@ -911,7 +911,7 @@ export const DataLoaderForm: React.FC<{
                     return [
                     <TableRow
                         key={tableName}
-                        sx={{ '&:last-child td, &:last-child th': { border: 0 }, '& .MuiTableCell-root': { padding: 0.25 }}}
+                        sx={{ '&:last-child td, &:last-child th': { border: 0 }, '& .MuiTableCell-root': { padding: 0.25, wordWrap: 'break-word', whiteSpace: 'normal' }}}
                     >
                         <TableCell sx={{borderBottom: displaySamples[tableName] ? 'none' : '1px solid rgba(0, 0, 0, 0.1)'}}>
                             <IconButton size="small" onClick={() => toggleDisplaySamples(tableName)}>

src/views/DBTableManager.tsx

@@ -363,8 +363,6 @@ export const ChartEditorFC: FC<{  cachedCandidates: DictTable[],
     const [errorMessage, setErrorMessage] = useState<{content: string, severity: "error" | "warning" | "info" | "success"}>({content: "", severity: "error"});
     const [showError, setShowError] = useState<boolean>(false);
 
-    
-
     let createVisTableRowsLocal = (rows: any[]) => {
         if (visFields.length == 0) {
             return rows;