[pull] main from activepieces:main

.github/workflows/release-pieces.yml

@@ -42,6 +42,8 @@ jobs:
 
       - name: Install dependencies
         run: bun install
+        env:
+          REDISMS_DISABLE_POSTINSTALL: "1"
 
       - name: copy project .npmrc to user level
         run: cp .npmrc $HOME/.npmrc

.github/workflows/validate-publishable-packages.yml

@@ -36,6 +36,8 @@ jobs:
 
       - name: Install dependencies
         run: bun install
+        env:
+          REDISMS_DISABLE_POSTINSTALL: "1"
 
       - name: validate publishable packages
         run: npx ts-node -r tsconfig-paths/register -P packages/server/engine/tsconfig.lib.json tools/scripts/validate-publishable-packages.ts

packages/pieces/community/duckdb/package.json

@@ -1,22 +1,25 @@
 {
   "name": "@activepieces/piece-duckdb",
-  "version": "0.0.8",
+  "version": "0.0.9",
   "type": "commonjs",
   "main": "./dist/src/index.js",
   "types": "./dist/src/index.d.ts",
   "dependencies": {
     "@activepieces/pieces-common": "workspace:*",
     "@activepieces/pieces-framework": "workspace:*",
     "@duckdb/node-api": "1.4.3-r.2",
+    "csv-parse": "5.6.0",
     "@activepieces/core-piece-types": "workspace:*",
     "@activepieces/core-utils": "workspace:*"
   },
   "devDependencies": {
-    "tslib": "^2.3.0"
+    "tslib": "^2.3.0",
+    "vitest": "3.2.6"
   },
   "scripts": {
     "build": "tsc -p tsconfig.lib.json && cp package.json dist/",
     "bundle": "node ../../../../dist/packages/cli/src/index.js pieces bundle",
-    "lint": "eslint 'src/**/*.ts'"
+    "lint": "eslint 'src/**/*.ts'",
+    "test": "vitest run"
   }
 }

packages/pieces/community/duckdb/src/lib/actions/create-and-query-db.ts

@@ -1,6 +1,8 @@
-import { createAction, Property } from '@activepieces/pieces-framework';
+import { ApFile, createAction, Property } from '@activepieces/pieces-framework';
 
 import { DuckDBInstance } from '@duckdb/node-api';
+import { quotedIdentifier } from '@duckdb/node-api/lib/sql';
+import { parse as parseCsv } from 'csv-parse/sync';
 
 export const createAndQueryDB = createAction({
   name: 'createAndQueryDB',
@@ -29,23 +31,38 @@ export const createAndQueryDB = createAction({
         data: Property.Json({
           displayName: 'JSON Data',
           description:
-            'Table data in the form of a JSON array of objects. Object keys would be taken as column names.',
-          required: true,
+            'Table data as a JSON array of objects. Object keys are taken as column names. Takes priority over the file below if set.',
+          required: false,
           defaultValue: [],
         }),
+        file: Property.File({
+          displayName: 'CSV/JSON File',
+          description:
+            'Upload a CSV or JSON file as the table source, used only if JSON Data above is empty. Format is detected from the file extension.',
+          required: false,
+        }),
         schema: Property.Json({
           displayName: 'Schema in JSON format',
           description: `
 Provide the schema as a JSON object. Note that only the specified columns would be loaded.
 Leave empty to autodetect schema, although it is recommended to specify the schema for consistency and avoid unexpected behaviour.
 
+A column type that doesn't match the actual shape of the row data (e.g. a STRUCT/array type for a column that's a plain string, which is always the case for CSV-sourced tables) silently loads as null instead of erroring - double check this against autodetected schema if a column is unexpectedly empty.
+
 More information on data types and accepted values:
 - https://duckdb.org/2023/03/03/json
 - https://duckdb.org/docs/stable/sql/data_types/overview
           `.trim(),
           required: false,
           defaultValue: {},
         }),
+        keepNestedFields: Property.Checkbox({
+          displayName: 'Keep Nested Fields',
+          description:
+            'When off, a nested object/struct field is split into individual top-level columns (e.g. "meta": {"a":1} becomes column "a"). Turn on to keep it as a single nested column instead, queryable with dot notation (e.g. "meta.a").',
+          required: false,
+          defaultValue: false,
+        }),
       },
     }),
     query: Property.ShortText({
@@ -72,10 +89,12 @@ More information on data types and accepted values:
 
     const dbTables: any[] = context.propsValue.tables ?? [];
     for (const dbTable of dbTables) {
-      const dbData = JSON.stringify(dbTable.data);
+      const tableName = quotedIdentifier(dbTable.name);
+      const dbData = JSON.stringify(resolveTableRows(dbTable));
       let dbSchema = null;
 
-      if (dbTable.schema) {
+      const hasSchema = dbTable.schema && Object.keys(dbTable.schema).length > 0;
+      if (hasSchema) {
         dbSchema = JSON.stringify([dbTable.schema]);
       } else {
         const schemaResult = await connection.run(
@@ -87,16 +106,21 @@ More information on data types and accepted values:
         dbSchema = detectedSchema[0][0];
       }
 
-      await connection.run(
-        `
-          CREATE TABLE ${dbTable.name} AS
+      const shouldFlatten = !dbTable.keepNestedFields;
+      const createTableQuery = shouldFlatten
+        ? `
+          CREATE TABLE ${tableName} AS
             SELECT UNNEST(JSON_TRANSFORM($sourceData, $sourceSchema), recursive := true);
-        `,
-        {
-          sourceData: dbData,
-          sourceSchema: dbSchema,
-        }
-      );
+        `
+        : `
+          CREATE TABLE ${tableName} AS
+            SELECT row.* FROM (SELECT UNNEST(JSON_TRANSFORM($sourceData, $sourceSchema)) AS row) t;
+        `;
+
+      await connection.run(createTableQuery, {
+        sourceData: dbData,
+        sourceSchema: dbSchema,
+      });
     }
 
     const queryArgs = context.propsValue.args ?? [];
@@ -118,3 +142,23 @@ More information on data types and accepted values:
     return results;
   },
 });
+
+function resolveTableRows(dbTable: { data?: unknown; file?: ApFile }): unknown {
+  if (Array.isArray(dbTable.data) ? dbTable.data.length > 0 : dbTable.data) {
+    return dbTable.data;
+  }
+
+  if (!dbTable.file) {
+    return dbTable.data;
+  }
+
+  const text = dbTable.file.data.toString('utf-8');
+  return dbTable.file.extension?.toLowerCase() === 'csv'
+    ? parseCsv(text, {
+        columns: true,
+        skip_empty_lines: true,
+        bom: true,
+        relax_column_count: true,
+      })
+    : JSON.parse(text.replace(/^/, ''));
+}