add examples

Author: Chenglong-MS

public/df_stock_prices_live.json

@@ -67,7 +67,7 @@
             * the column can either be a column in the input data, or a new column that will be computed in the output data.
             * the mention don't have to be exact match, it can be semantically matching, e.g., if you mentioned "average score" in the text while the column to be computed is "Avg_Score", you should still highlight "**average score**" in the text.
     - determine "input_tables", the names of a subset of input tables from [CONTEXT] section that will be used to achieve the user's goal.
-        - Note that the first table is the table the user is currently viewing, it should take precedence if the user asks for visualization of the "current table".
+        - **IMPORTANT** Note that the Table 1 in [CONTEXT] section is the table the user is currently viewing, it should take precedence if the user refers to insights about the "current table".
         - At the same time, leverage table information to determine which tables are relevant to the user's goal and should be used.
     - "chart_type" must be one of "point", "bar", "line", "area", "heatmap", "group_bar", "boxplot"
     - "chart_encodings" should specify which fields should be used to create the visualization

public/screenshot-stock-price-live-thumbnail.webp

@@ -38,7 +38,7 @@
             - **IMPORTANT** Note that the Table 1 in [CONTEXT] section is the table the user is currently viewing, it should take precedence if the user refers to insights about the "current table".
             - At the same time, leverage table information to determine which tables are relevant to the user's goal and should be used.
         - determine "output_fields", the desired fields that the output data should have to achieve the user's goal, it's a good idea to include intermediate fields here.
-        - then decide "chart_encodings", which maps visualization channels (x, y, color, size, opacity, facet, etc.) to a subset of "output_fields" that will be visualized, 
+        - then decide "chart_encodings", which maps visualization channels (x, y, color, size, opacity, facet, latitude, longitude, etc.) to a subset of "output_fields" that will be visualized, 
             - the "chart_encodings" should be created to support the user's "chart_type".
             - first, determine whether the user has provided sufficient fields in "chart_encodings" that are needed to achieve their goal:
                 - if the user's "chart_encodings" are sufficient, simply copy it.
@@ -49,6 +49,7 @@
                 - if the user's "chart_encodings" is sufficient but can be optimized, you can reorder encodings to visualize the data more effectively.
             - sometimes, user may provide instruction to update visualizations fields they provided. You should leverage the user's goal to resolve the conflict and decide the final "chart_encodings"
                 - e.g., they may mention "use B metric instead" while A metric is in provided fields, in this case, you should update "chart_encodings" to update A metric with B metric.
+            - if the user provides latitude and longitude as visual channels, use "latitude" and "longitude" as visual channels in "chart_encodings" as opposed to "x" and "y".
         - guide on statistical analysis:
             - when the user asks for forecasting or regression analysis, you should consider the following:
                 - the output should be a long format table where actual x, y pairs and predicted x, y pairs are included in the X, Y columns, they are differentiated with a third column "is_predicted".

py-src/data_formulator/agents/agent_py_data_rec.py

@@ -27,6 +27,7 @@ export function useDataRefresh() {
     const timeoutRefs = useRef<Map<string, NodeJS.Timeout>>(new Map());
     const refreshInProgressRef = useRef<Map<string, boolean>>(new Map());
     const isActiveRef = useRef<Map<string, boolean>>(new Map());
+    const initializedTablesRef = useRef<Set<string>>(new Set()); // Track tables that have been initialized
     const lastMessageTimeRef = useRef<number>(0);
     const MESSAGE_THROTTLE_MS = 10000; // 10 seconds
 
@@ -316,6 +317,7 @@ export function useDataRefresh() {
     /**
      * Set up refresh intervals for tables with auto-refresh enabled.
      * Uses recursive setTimeout pattern to ensure updates wait until complete.
+     * Triggers immediate refresh for newly loaded tables.
      */
     useEffect(() => {
         // Clear all existing timeouts
@@ -334,22 +336,44 @@ export function useDataRefresh() {
 
             if (shouldAutoRefresh) {
                 const intervalMs = source.refreshIntervalSeconds! * 1000;
+                const isNewTable = !initializedTablesRef.current.has(table.id);
 
-                console.log(`[DataRefresh] Setting up auto-refresh for "${table.id}" every ${source.refreshIntervalSeconds}s (waiting for completion)`);
+                console.log(`[DataRefresh] Setting up auto-refresh for "${table.id}" every ${source.refreshIntervalSeconds}s (new=${isNewTable})`);
 
-                // Mark as active
+                // Mark as active and initialized
                 isActiveRef.current.set(table.id, true);
+                initializedTablesRef.current.add(table.id);
 
-                // Start the refresh cycle - first refresh happens after interval, then schedules next after completion
-                const initialTimeout = setTimeout(async () => {
-                    if (!isActiveRef.current.get(table.id)) {
-                        return;
-                    }
-                    await performRefresh(table);
-                    scheduleNextRefresh(table.id);
-                }, intervalMs);
+                if (isNewTable) {
+                    // For newly loaded tables, trigger immediate refresh then schedule next
+                    console.log(`[DataRefresh] Triggering immediate first refresh for newly loaded table "${table.id}"`);
+                    (async () => {
+                        if (!isActiveRef.current.get(table.id)) {
+                            return;
+                        }
+                        await performRefresh(table);
+                        scheduleNextRefresh(table.id);
+                    })();
+                } else {
+                    // For existing tables, continue with normal interval
+                    const initialTimeout = setTimeout(async () => {
+                        if (!isActiveRef.current.get(table.id)) {
+                            return;
+                        }
+                        await performRefresh(table);
+                        scheduleNextRefresh(table.id);
+                    }, intervalMs);
+
+                    timeoutRefs.current.set(table.id, initialTimeout);
+                }
+            }
+        });
 
-                timeoutRefs.current.set(table.id, initialTimeout);
+        // Clean up tables that no longer exist from the initialized set
+        const currentTableIds = new Set(tables.map(t => t.id));
+        initializedTablesRef.current.forEach(tableId => {
+            if (!currentTableIds.has(tableId)) {
+                initializedTablesRef.current.delete(tableId);
             }
         });
 

py-src/data_formulator/agents/agent_py_data_transform.py

@@ -55,9 +55,6 @@ export const DataFormulatorFC = ({ }) => {
     const theme = useTheme();
 
     const dispatch = useDispatch();
-
-    // Set up auto-refresh for streaming and database tables
-    const { manualRefresh, getRefreshInfo } = useDataRefresh();
 
     // Set up automatic refresh of derived tables when source data changes
     useDerivedTableRefresh();

src/app/useDataRefresh.tsx

@@ -11,6 +11,7 @@ import {
     alpha,
     useTheme,
 } from '@mui/material';
+import StreamIcon from '@mui/icons-material/Stream';
 
 // Example session data for pre-built sessions
 export interface ExampleSession {
@@ -19,36 +20,49 @@ export interface ExampleSession {
     description: string;
     previewImage: string;
     dataFile: string;
+    live: boolean;
 }
 
 export const exampleSessions: ExampleSession[] = [
+    {
+        id: 'stock-prices-live',
+        title: 'Stock Prices (Live)',
+        description: 'Live stock prices for different companies',
+        previewImage: '/screenshot-stock-price-live-thumbnail.webp',
+        dataFile: '/df_stock_prices_live.json',
+        live: true,
+    },
     {
         id: 'gas-prices',
         title: 'Gas Prices',
         description: 'Weekly gas prices across different grades and formulations',
         previewImage: '/gas_prices-thumbnail.webp',
         dataFile: '/df_gas_prices.json',
+        live: false,
     },
     {
         id: 'global-energy',
         title: 'Global Energy',
         description: 'Explore global energy consumption and CO2 emissions data',
         previewImage: '/global_energy-thumbnail.webp',
-        dataFile: '/df_global_energy.json',
+        dataFile: '/df_global_energy.json', 
+        live: false,
     },
     {
         id: 'movies',
         title: 'Movies',
         description: 'Analyze movie performance, budgets, and ratings data',
         previewImage: '/movies-thumbnail.webp',
         dataFile: '/df_movies.json',
+        live: false,
     },
     {
         id: 'unemployment',
         title: 'Unemployment',
         description: 'Unemployment rates across different industries over time',
         previewImage: '/unemployment-thumbnail.webp',
         dataFile: '/df_unemployment.json',
+        live: false,
     }
 ];
 
@@ -66,7 +80,7 @@ export const ExampleSessionCard: React.FC<{
                 cursor: disabled ? 'default' : 'pointer',
                 '&:hover': disabled ? {} : {
                     transform: 'translateY(-2px)',
-                    borderColor: alpha(theme.palette.primary.main, 0.4),
+                    borderColor: session.live ? alpha(theme.palette.secondary.main, 0.4) : alpha(theme.palette.primary.main, 0.4),
                 },
             }}
             onClick={disabled ? undefined : onClick}
@@ -111,7 +125,7 @@ export const ExampleSessionCard: React.FC<{
                             overflow: 'hidden'
                         }}
                     >
-                        <span style={{textDecoration: 'underline'}}>{session.title}:</span> {session.description}
+                        {session.live && <StreamIcon sx={{ fontSize: 10, color: 'success.main' }} />} <span style={{textDecoration: 'underline'}}>{session.title}:</span> {session.description}
                     </Typography>
                 </Box>
             </CardContent>