[deploy] 0.1.6.2 Merge pull request #121 from microsoft/dev

1. gracefully work with large data: when chart has too many categorical values on x/y/column/row axes, DF will not automatically cap the number of items displayed.

2. load data from Excel

3. enhanced model configuration.

Author: Chenglong-MS

package.json

@@ -45,7 +45,8 @@
         "vega": "^5.26.0",
         "vega-embed": "^6.21.0",
         "vega-lite": "^5.5.0",
-        "vm-browserify": "^1.1.2"
+        "vm-browserify": "^1.1.2",
+        "xlsx": "^0.18.5"
     },
     "scripts": {
         "lint": "eslint -c eslint.config.js src/**/*.{ts,tsx} --fix",

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "data_formulator"
-version = "0.1.6.1"
+version = "0.1.6.2"
 
 requires-python = ">=3.9"
 authors = [

src/app/dfSlice.tsx

@@ -675,8 +675,6 @@ export const dataFormulatorSlice = createSlice({
 
             console.log("load model complete");
             console.log("state.models", state.models);
-            console.log("state.selectedModelId", state.selectedModelId);
-            console.log("state.testedModels", state.testedModels);
         })
         .addCase(fetchCodeExpl.fulfilled, (state, action) => {
             let codeExpl = action.payload;

src/app/utils.tsx

@@ -98,7 +98,7 @@ export function runCodeOnInputListsInVM(
                     let target = undefined;
                     try {
                         // copy args to ensure correctness of mapping
-                        target = func(...JSON.parse(JSON.stringify(args)))
+                        target = func(...structuredClone(args))
                     } catch (err) {
                         console.warn(`execution err ${err}`)
                     }
@@ -179,7 +179,7 @@ export function baseTableToExtTable(table: any[], derivedFields: FieldItem[], al
                     let args = inputTuples;
                     if (func.length == baseCols.length * 2 + 1) {
                         // avoid side effect, use the copy of the column when calling the function
-                        args = [...inputTuples, rowIdx, ...JSON.parse(JSON.stringify(baseCols))]
+                        args = [...inputTuples, rowIdx, ...structuredClone(baseCols)]
                     }            
 
                     target = func(...args);
@@ -242,7 +242,13 @@ export function baseTableToExtTable(table: any[], derivedFields: FieldItem[], al
 }
 
 
-export const instantiateVegaTemplate = (chartType: string, encodingMap: { [key in Channel]: EncodingItem; }, allFields: FieldItem[], workingTable: any[]) => {
+export const assembleVegaChart = (
+    chartType: string, 
+    encodingMap: { [key in Channel]: EncodingItem; }, 
+    conceptShelfItems: FieldItem[], 
+    workingTable: any[],
+    maxNominalValues: number = 68
+) => {
 
     if (chartType == "Table") {
         return ["Table", undefined];
@@ -251,8 +257,7 @@ export const instantiateVegaTemplate = (chartType: string, encodingMap: { [key i
     let chartTemplate = getChartTemplate(chartType) as ChartTemplate;
     //console.log(chartTemplate);
 
-    let vgObj = JSON.parse(JSON.stringify(chartTemplate.template));
-    const baseTableSchemaObj: any = {};
+    let vgObj = structuredClone(chartTemplate.template);
 
     for (const [channel, encoding] of Object.entries(encodingMap)) {
 
@@ -262,30 +267,8 @@ export const instantiateVegaTemplate = (chartType: string, encodingMap: { [key i
             encodingObj["scale"] = {"type": "sqrt", "zero": true};
         }
 
-        const field = encoding.fieldID ? _.find(allFields, (f) => f.id === encoding.fieldID) : undefined;
+        const field = encoding.fieldID ? _.find(conceptShelfItems, (f) => f.id === encoding.fieldID) : undefined;
         if (field) {
-            //console.log(field)
-            // the synthesizer only need to see base table schema
-            let baseFields = (field.source == "derived" ? 
-                    (field.transform as ConceptTransformation).parentIDs.map((parentID) => allFields.find((f) => f.id == parentID) as FieldItem) 
-                  : [field]);
-
-            for (let baseField of baseFields) {
-                if (Object.keys(baseTableSchemaObj).includes(baseField.name)) {
-                    continue;
-                }
-                baseTableSchemaObj[baseField.name] = {
-                    channel, 
-                    dtype: getDType(baseField.type, workingTable.map(r => r[baseField.name])),
-                    name: baseField.name,
-                    original: baseField.source == "original",
-                    // domain: {
-                    //     values: [...new Set(baseField.domain.values)],
-                    //     is_complete: baseField.domain.isComplete
-                    // },
-                };
-            }
-
             // create the encoding
             encodingObj["field"] = field.name;
             encodingObj["type"] = getDType(field.type, workingTable.map(r => r[field.name]));
@@ -428,16 +411,8 @@ export const instantiateVegaTemplate = (chartType: string, encodingMap: { [key i
         vgObj = chartTemplate.postProcessor(vgObj, workingTable);
     }
 
-    // console.log(JSON.stringify(vgObj))
-
-    return [vgObj, baseTableSchemaObj];
-}
-
-export const assembleChart = (chart: Chart, conceptShelfItems: FieldItem[], dataValues: any[]) => {
-
-    let vgSpec: any = instantiateVegaTemplate(chart.chartType, chart.encodingMap, conceptShelfItems, dataValues)[0];
-
-    let values = JSON.parse(JSON.stringify(dataValues));
+    // this is the data that will be assembled into the vega chart
+    let values = structuredClone(workingTable);
     values = values.map((r: any) => { 
         let keys = Object.keys(r);
         let temporalKeys = keys.filter((k: string) => conceptShelfItems.some(concept => concept.name == k && (concept.type == "date" || concept.semanticType == "Year")));
@@ -446,9 +421,82 @@ export const assembleChart = (chart: Chart, conceptShelfItems: FieldItem[], data
         }
         return r;
     })
-    return {...vgSpec, data: {values: values}}
+
+    // Handle nominal axes with many entries
+    for (const channel of ['x', 'y', 'column', 'row', 'xOffset']) {
+        const encoding = vgObj.encoding?.[channel];
+        if (encoding?.type === 'nominal') {
+            const fieldName = encoding.field;
+            const uniqueValues = [...new Set(values.map((r: any) => r[fieldName]))];
+            
+            let valuesToKeep: any[];
+            if (uniqueValues.length > maxNominalValues) {
+
+                if (channel == 'x' || channel == 'y') {
+                    const oppositeChannel = channel === 'x' ? 'y' : 'x';
+                    const oppositeEncoding = vgObj.encoding?.[oppositeChannel];
+                    
+                    if (oppositeEncoding?.type === 'quantitative') {
+                        // Sort by the quantitative field and take top maxNominalValues
+                        const quantField = oppositeEncoding.field;
+                        valuesToKeep = uniqueValues
+                            .map(val => ({
+                                value: val,
+                                sum: workingTable
+                                    .filter(r => r[fieldName] === val)
+                                    .reduce((sum, r) => sum + (r[quantField] || 0), 0)
+                            }))
+                            .sort((a, b) => b.sum - a.sum)
+                            .slice(0, maxNominalValues)
+                            .map(v => v.value);
+                    } else {
+                        // If no quantitative axis, just take first maxNominalValues
+                        valuesToKeep = uniqueValues.slice(0, maxNominalValues);
+                    }
+                } else if (channel == 'row') {
+                    valuesToKeep = uniqueValues.slice(0, 20);
+                } else {
+                    valuesToKeep = uniqueValues.slice(0, maxNominalValues);
+                }
+
+                // Filter the working table
+                const omittedCount = uniqueValues.length - maxNominalValues;
+                const placeholder = `...${omittedCount} items omitted`;
+                values = values.filter((row: any) => valuesToKeep.includes(row[fieldName]));
+
+                // Add text formatting configuration
+                if (!encoding.axis) {
+                    encoding.axis = {};
+                }
+                encoding.axis.labelColor = {
+                    condition: {
+                        test: `datum.label == '${placeholder}'`,
+                        value: "#999999"
+                    },
+                    value: "#000000" // default color for other labels
+                };
+                encoding.axis.labelFont = {
+                    condition: {
+                        test: `datum.label == '${placeholder}'`,
+                        value: "italic"
+                    },
+                    value: "normal" // default font style for other labels
+                };
+
+                // Add placeholder to domain
+                if (!encoding.scale) {
+                    encoding.scale = {};
+                }
+                encoding.scale.domain = [...valuesToKeep, placeholder]
+            }
+        }
+    }
+
+    return {...vgObj, data: {values: values}}
 }
 
+
+
 export const adaptChart = (chart: Chart, targetTemplate: ChartTemplate) => {
 
     let discardedChannels = Object.entries(chart.encodingMap).filter(([ch, enc]) => {

src/data/utils.ts

@@ -3,12 +3,13 @@
 
 import * as d3 from 'd3';
 import Column from './column';
+import * as XLSX from 'xlsx';
 
 import { DictTable } from '../components/ComponentType';
 import { CoerceType, TestType, Type } from './types';
 import { ColumnTable } from './table';
 
-export const loadDataWrapper = (title: string, text: string, fileType: string): DictTable | undefined => {
+export const loadTextDataWrapper = (title: string, text: string, fileType: string): DictTable | undefined => {
 
     let tableName = title;
     //let tableName = title.replace(/\.[^/.]+$/ , "");
@@ -18,8 +19,7 @@ export const loadDataWrapper = (title: string, text: string, fileType: string):
         table = createTableFromText(tableName, text);
     } else if (fileType == "application/json") {
         table = createTableFromFromObjectArray(tableName, JSON.parse(text));
-    }
-
+    } 
     return table;
 };
 
@@ -43,7 +43,14 @@ export const createTableFromText = (title: string, text: string): DictTable | un
     // Should check the data file as well for the ending
     const isTabSeparated = tabNum / lineNum >= 1;
 
-    const values = isTabSeparated ? d3.tsvParse(text) : d3.csvParse(text);
+    // Use d3.dsvFormat to create a custom parser that properly handles quoted fields
+    // This ensures commas inside quoted fields won't be treated as delimiters
+    const values = isTabSeparated 
+        ? d3.tsvParse(text) 
+        : d3.dsvFormat(',').parse(text, row => {
+            // Process each row to ensure proper type handling
+            return row;
+          });
 
     return createTableFromFromObjectArray(title, values);
 };
@@ -145,20 +152,32 @@ export function tupleEqual(a: any[], b: any[]) {
     return true;
 }
 
-// export function arrayEqual(_arr1: any[], _arr2: any[]) {
-//     if (Array.isArray(_arr1) || !Array.isArray(_arr2) || _arr1.length !== _arr2.length) {
-//         return false;
-//     }
-    
-//     // .concat() to not mutate arguments
-//     const arr1 = _arr1.concat().sort();
-//     const arr2 = _arr2.concat().sort();
-    
-//     for (let i = 0; i < arr1.length; i++) {
-//         if (arr1[i] !== arr2[i]) {
-//             return false;
-//         }
-//     }
-    
-//     return true;
-// }
+export const loadBinaryDataWrapper = (title: string, arrayBuffer: ArrayBuffer): DictTable[] => {
+    try {
+        // Read the Excel file
+        const workbook = XLSX.read(arrayBuffer, { type: 'array' });
+        
+        // Get all sheet names
+        const sheetNames = workbook.SheetNames;
+        
+        // Create tables for each sheet
+        const tables: DictTable[] = [];
+        
+        for (const sheetName of sheetNames) {
+            // Get the worksheet
+            const worksheet = workbook.Sheets[sheetName];
+            
+            // Convert the worksheet to JSON
+            const jsonData = XLSX.utils.sheet_to_json(worksheet);
+            
+            // Create a table from the JSON data with sheet name included in the title
+            const sheetTable = createTableFromFromObjectArray(`${title}-${sheetName}`, jsonData);
+            tables.push(sheetTable);
+        }
+        
+        return tables;
+    } catch (error) {
+        console.error('Error processing Excel file:', error);
+        return [];
+    }
+};