live data support

Author: Chenglong-MS

py-src/data_formulator/demo_stream_routes.py

@@ -414,94 +414,6 @@
             }
         ]
     },
-    # ==========================================================================
-    # LIVE DATA EXAMPLES - Auto-refresh streaming datasets
-    # ==========================================================================
-    {
-        'source': 'demo',
-        'name': 'Live Sales Feed (Live)',
-        'description': 'Simulated e-commerce transactions with product details, regions, and channels. Updates every 1 second.',
-        'live': True,
-        'refreshIntervalSeconds': 1,
-        'tables': [
-            {
-                "format": 'csv',
-                "url": '/api/demo-stream/live-sales',
-                "sample": '''transaction_id,timestamp,product,category,quantity,unit_price,discount_pct,total,region,channel
-TX123456,2024-01-15T10:30:00Z,Wireless Headphones,Electronics,2,71.99,10,143.98,North America,Web
-TX123457,2024-01-15T10:29:45Z,Running Shoes,Sports,1,129.99,0,129.99,Europe,Mobile App
-TX123458,2024-01-15T10:29:30Z,Coffee Maker,Home,1,80.99,10,80.99,Asia Pacific,Web'''
-            }
-        ]
-    },
-    {
-        'source': 'open-notify.org',
-        'name': 'ISS Location (Live)',
-        'description': 'Real-time International Space Station position trajectory. Updates every 5 seconds.',
-        'live': True,
-        'refreshIntervalSeconds': 5,
-        'tables': [
-            {
-                "format": 'csv',
-                "url": '/api/demo-stream/iss',
-                "sample": '''timestamp,latitude,longitude,fetched_at
-2024-01-15T10:30:00Z,45.234,-122.456,2024-01-15T10:30:00Z
-2024-01-15T10:29:55Z,45.123,-122.345,2024-01-15T10:29:55Z
-2024-01-15T10:29:50Z,45.012,-122.234,2024-01-15T10:29:50Z'''
-            }
-        ]
-    },
-    {
-        'source': 'USGS',
-        'name': 'Earthquakes (Live)',
-        'description': 'Real-time earthquake data from USGS. New quakes appear over time. Updates every 60 seconds.',
-        'live': True,
-        'refreshIntervalSeconds': 60,
-        'tables': [
-            {
-                "format": 'csv',
-                "url": '/api/demo-stream/earthquakes?timeframe=hour',
-                "sample": '''id,time,latitude,longitude,depth_km,magnitude,place,type,status,felt,cdi,mmi,tsunami,sig,net,code,url,fetched_at
-us7000abc1,2024-01-15T10:25:00Z,36.234,-117.456,5.2,2.5,"10km N of Ridgecrest, CA",earthquake,reviewed,,,0,0,125,us,abc1,https://earthquake.usgs.gov/earthquakes/eventpage/us7000abc1,2024-01-15T10:30:00Z
-us7000abc2,2024-01-15T10:15:00Z,61.123,-150.345,45.8,3.1,"50km S of Anchorage, AK",earthquake,automatic,5,,0,0,180,us,abc2,https://earthquake.usgs.gov/earthquakes/eventpage/us7000abc2,2024-01-15T10:30:00Z'''
-            }
-        ]
-    },
-    {
-        'source': 'Open-Meteo',
-        'name': 'Weather (Live)',
-        'description': 'Current weather conditions for major US cities. Updates every 5 minutes.',
-        'live': True,
-        'refreshIntervalSeconds': 300,
-        'tables': [
-            {
-                "format": 'csv',
-                "url": '/api/demo-stream/weather',
-                "sample": '''city,latitude,longitude,temperature_c,humidity_percent,wind_speed_kmh,precipitation_mm,fetched_at
-Seattle,47.6062,-122.3321,8.5,72,12.5,0.2,2024-01-15T10:30:00Z
-New York,40.7128,-74.006,2.3,65,18.2,0.0,2024-01-15T10:30:00Z
-Los Angeles,34.0522,-118.2437,15.8,55,8.3,0.0,2024-01-15T10:30:00Z'''
-            }
-        ]
-    },
-    {
-        'source': 'yfinance',
-        'name': 'Stock Prices (Live)',
-        'description': 'Stock prices with 6 months of daily history + recent 15-minute intraday data. Updates every 5 minutes during market hours.',
-        'live': True,
-        'refreshIntervalSeconds': 300,
-        'tables': [
-            {
-                "format": 'csv',
-                "url": '/api/demo-stream/yfinance?symbols=AAPL,MSFT,GOOGL,NVDA',
-                "sample": '''symbol,timestamp,date,open,high,low,close,volume,data_type,fetched_at
-AAPL,2024-01-15 14:30:00,2024-01-15,185.50,186.20,185.30,185.95,1234567,intraday,2024-01-15T19:35:00Z
-MSFT,2024-01-15 14:30:00,2024-01-15,388.75,389.50,388.25,389.10,987654,intraday,2024-01-15T19:35:00Z
-GOOGL,2024-01-15 14:30:00,2024-01-15,141.20,141.80,141.00,141.55,654321,intraday,2024-01-15T19:35:00Z
-NVDA,2024-01-15 14:30:00,2024-01-15,545.20,548.80,544.50,547.35,2345678,intraday,2024-01-15T19:35:00Z'''
-            }
-        ]
-    }
 ]
 
 

py-src/data_formulator/example_datasets_config.py

@@ -182,6 +182,55 @@ let getUnrefedDerivedTableIds = (state: DataFormulatorState) => {
     let tableWithDescendants = state.tables.filter(table => state.tables.some(t => t.derive?.trigger.tableId == table.id)).map(t => t.id);
 
     return state.tables.filter(table => table.derive && !tableWithDescendants.includes(table.id) && !chartRefedTables.includes(table.id)).map(t => t.id);
+}
+
+// Helper function to auto-populate latitude/longitude encodings for map charts
+let autoPopulateMapEncodings = (chart: Chart, table: DictTable | undefined, conceptShelfItems: FieldItem[]) => {
+    if (!table) return;
+    
+    // Patterns to match latitude/longitude column names
+    const latPatterns = ['latitude', 'lat'];
+    const lonPatterns = ['longitude', 'lon', 'lng', 'long'];
+    
+    // Find latitude column (exact match first, then partial match)
+    let latColumn = table.names.find(name => 
+        latPatterns.some(p => name.toLowerCase() === p)
+    );
+    if (!latColumn) {
+        latColumn = table.names.find(name => 
+            latPatterns.some(p => name.toLowerCase().includes(p))
+        );
+    }
+    
+    // Find longitude column (exact match first, then partial match)
+    let lonColumn = table.names.find(name => 
+        lonPatterns.some(p => name.toLowerCase() === p)
+    );
+    if (!lonColumn) {
+        lonColumn = table.names.find(name => 
+            lonPatterns.some(p => name.toLowerCase().includes(p))
+        );
+    }
+    
+    // Auto-populate latitude encoding if found and not already set
+    if (latColumn && chart.encodingMap.latitude?.fieldID == undefined) {
+        const latField = conceptShelfItems.find(f => 
+            f.name === latColumn && table.names.includes(f.name)
+        );
+        if (latField) {
+            chart.encodingMap.latitude = { fieldID: latField.id };
+        }
+    }
+    
+    // Auto-populate longitude encoding if found and not already set
+    if (lonColumn && chart.encodingMap.longitude?.fieldID == undefined) {
+        const lonField = conceptShelfItems.find(f => 
+            f.name === lonColumn && table.names.includes(f.name)
+        );
+        if (lonField) {
+            chart.encodingMap.longitude = { fieldID: lonField.id };
+        }
+    }
 } 
 
 let deleteChartsRoutine = (state: DataFormulatorState, chartIds: string[]) => {
@@ -603,6 +652,13 @@ export const dataFormulatorSlice = createSlice({
             let chartType = action.payload.chartType;
             let tableId = action.payload.tableId || state.tables[0].id;
             let freshChart = generateFreshChart(tableId, chartType, "user") as Chart;
+            
+            // Auto-populate latitude/longitude for map charts
+            if (chartType.toLowerCase().includes('map')) {
+                let table = state.tables.find(t => t.id === tableId);
+                autoPopulateMapEncodings(freshChart, table, state.conceptShelfItems);
+            }
+            
             state.charts = [ freshChart , ...state.charts];
             state.focusedTableId = tableId;
             state.focusedChartId = freshChart.id;
@@ -652,6 +708,14 @@ export const dataFormulatorSlice = createSlice({
             let chart = dfSelectors.getAllCharts(state).find(c => c.id == chartId);
             if (chart) {
                 chart = adaptChart(chart, getChartTemplate(chartType) as ChartTemplate);
+                
+                // Auto-populate latitude/longitude for map charts
+                if (chartType.toLowerCase().includes('map')) {
+                    let allCharts = dfSelectors.getAllCharts(state);
+                    let table = getDataTable(chart, state.tables, allCharts, state.conceptShelfItems);
+                    autoPopulateMapEncodings(chart, table, state.conceptShelfItems);
+                }
+                
                 dfSelectors.replaceChart(state, chart);
             }
         },

src/app/dfSlice.tsx

@@ -688,7 +688,17 @@ export const assembleVegaChart = (
         if (temporalKeys.length > 0) {
             values = values.map((r: any) => { 
                 for (let temporalKey of temporalKeys) {
-                    r[temporalKey] = String(r[temporalKey]);
+                    const val = r[temporalKey];
+                    // Convert numeric timestamps to ISO date strings for Vega-Lite compatibility
+                    if (typeof val === 'number') {
+                        // Detect if timestamp is in seconds (10 digits) or milliseconds (13 digits)
+                        const timestamp = val < 1e12 ? val * 1000 : val;
+                        r[temporalKey] = new Date(timestamp).toISOString();
+                    } else if (val instanceof Date) {
+                        r[temporalKey] = val.toISOString();
+                    } else {
+                        r[temporalKey] = String(val);
+                    }
                 }
                 return r;
             })

src/app/utils.tsx

@@ -28,6 +28,7 @@ import chartIconCustomArea from '../assets/chart-icon-custom-area-min.png';
 import chartIconPie from '../assets/chart-icon-pie-min.png';
 import chartIconUSMap from '../assets/chart-icon-us-map-min.png';
 import chartIconPyramid from '../assets/chart-icon-pyramid-min.png';
+import chartIconWorldMap from '../assets/chart-icon-world-map-min.png';
 
 // Chart Icon Component using static imports
 const ChartIcon: React.FC<{ src: string; alt?: string }> = ({ src, alt = "" }) => {
@@ -385,7 +386,7 @@ const barCharts: ChartTemplate[] = [
 
 const mapCharts: ChartTemplate[] = [
     {
-        "chart": "US Map with Points",
+        "chart": "US Map",
         "icon": <ChartIcon src={chartIconUSMap} />,
         "template": {
             "width": 500,
@@ -429,6 +430,52 @@ const mapCharts: ChartTemplate[] = [
             "color": ["layer", 1, "encoding", "color"],
             "size": ["layer", 1, "encoding", "size"]
         }
+    },
+    {
+        "chart": "World Map",
+        "icon": <ChartIcon src={chartIconWorldMap} />,
+        "template": {
+            "width": 600,
+            "height": 350,
+            "layer": [
+                {
+                    "data": {
+                        "url": "https://vega.github.io/vega-lite/data/world-110m.json",
+                        "format": {
+                            "type": "topojson",
+                            "feature": "countries"
+                        }
+                    },
+                    "projection": {
+                        "type": "equalEarth"
+                    },
+                    "mark": {
+                        "type": "geoshape",
+                        "fill": "lightgray",
+                        "stroke": "white"
+                    }
+                },
+                {
+                    "projection": {
+                        "type": "equalEarth"
+                    },
+                    "mark": "circle",
+                    "encoding": {
+                        "longitude": { },
+                        "latitude": { },
+                        "size": {},
+                        "color": {}
+                    }
+                }
+            ]
+        },
+        "channels": ["longitude", "latitude", "color", "size"],
+        "paths": {
+            "longitude": ["layer", 1, "encoding", "longitude"],
+            "latitude": ["layer", 1, "encoding", "latitude"],
+            "color": ["layer", 1, "encoding", "color"],
+            "size": ["layer", 1, "encoding", "size"]
+        }
     }
 ]