Major improvements and new features

.gitignore

@@ -33,3 +33,6 @@ yarn-error.log*
 # Temporary files
 *.tmp
 .cache/
+
+# Extracted GTFS files from ZIP
+tests/fixtures/*.txt

README.md

@@ -6,6 +6,9 @@ A TypeScript library for loading GTFS (General Transit Feed Specification) data
 
 **Théophile Helleboid / SysDevRun**
 
+- Email: contact@sys-dev-run.fr
+- Website: https://www.sys-dev-run.fr/
+
 ## Features
 
 - ✅ Load GTFS data from ZIP files (URL or local path)

package.json

@@ -41,10 +41,12 @@
   },
   "dependencies": {
     "jszip": "^3.10.1",
+    "papaparse": "^5.5.3",
     "sql.js": "^1.10.3"
   },
   "devDependencies": {
     "@types/node": "^20.11.5",
+    "@types/papaparse": "^5.5.0",
     "@types/sql.js": "^1.4.9",
     "@typescript-eslint/eslint-plugin": "^6.19.0",
     "@typescript-eslint/parser": "^6.19.0",

src/gtfs-sqljs.ts

@@ -8,7 +8,7 @@ import { loadGTFSZip } from './loaders/zip-loader';
 import { loadGTFSData } from './loaders/data-loader';
 
 // Query methods
-import { getStopById, getStopByCode, searchStopsByName, getAllStops } from './queries/stops';
+import { getStopById, getStopByCode, searchStopsByName, getAllStops, getStopsByTrip } from './queries/stops';
 import { getRouteById, getAllRoutes, getRoutesByAgency } from './queries/routes';
 import {
   getActiveServiceIds,
@@ -21,6 +21,7 @@ import {
   getTripsByRoute,
   getTripsByRouteAndService,
   getTripsByRouteServiceAndDirection,
+  getTripsByService,
 } from './queries/trips';
 import {
   getStopTimesByTrip,
@@ -286,6 +287,23 @@ export class GtfsSqlJs {
     return getTripsByRouteServiceAndDirection(this.db, routeId, serviceIds, directionId);
   }
 
+  /**
+   * Get all trips for a given date (YYYYMMDD format)
+   */
+  getTripsByDate(date: string): Trip[] {
+    if (!this.db) throw new Error('Database not initialized');
+    const serviceIds = getActiveServiceIds(this.db, date);
+    return getTripsByService(this.db, serviceIds);
+  }
+
+  /**
+   * Get stops for a given trip (ordered by stop_sequence)
+   */
+  getStopsByTrip(tripId: string): Stop[] {
+    if (!this.db) throw new Error('Database not initialized');
+    return getStopsByTrip(this.db, tripId);
+  }
+
   // ==================== Stop Time Methods ====================
 
   /**

src/loaders/csv-parser.ts

@@ -1,79 +1,35 @@
 /**
- * CSV Parser for GTFS files
+ * CSV Parser for GTFS files using papaparse
  */
 
+import Papa from 'papaparse';
+
 export interface ParsedCSV {
   headers: string[];
   rows: Record<string, string>[];
 }
 
 /**
- * Parse CSV text into structured data
+ * Parse CSV text into structured data using papaparse
  */
 export function parseCSV(text: string): ParsedCSV {
-  const lines = text.split(/\r?\n/).filter((line) => line.trim().length > 0);
-
-  if (lines.length === 0) {
-    return { headers: [], rows: [] };
+  const result = Papa.parse(text, {
+    header: true,
+    skipEmptyLines: true,
+    transformHeader: (header) => header.trim(),
+    transform: (value) => value.trim(),
+  });
+
+  if (result.errors.length > 0) {
+    console.warn('CSV parsing warnings:', result.errors);
   }
 
-  // Parse headers
-  const headers = parseCSVLine(lines[0]);
-
-  // Parse rows
-  const rows: Record<string, string>[] = [];
-  for (let i = 1; i < lines.length; i++) {
-    const values = parseCSVLine(lines[i]);
-    if (values.length === 0) continue;
-
-    const row: Record<string, string> = {};
-    for (let j = 0; j < headers.length; j++) {
-      const header = headers[j];
-      const value = values[j] || '';
-      row[header] = value;
-    }
-    rows.push(row);
-  }
+  const headers = result.meta.fields || [];
+  const rows = result.data as Record<string, string>[];
 
   return { headers, rows };
 }
 
-/**
- * Parse a single CSV line, handling quoted values
- */
-function parseCSVLine(line: string): string[] {
-  const result: string[] = [];
-  let current = '';
-  let inQuotes = false;
-
-  for (let i = 0; i < line.length; i++) {
-    const char = line[i];
-    const nextChar = line[i + 1];
-
-    if (char === '"') {
-      if (inQuotes && nextChar === '"') {
-        // Escaped quote
-        current += '"';
-        i++; // Skip next quote
-      } else {
-        // Toggle quote state
-        inQuotes = !inQuotes;
-      }
-    } else if (char === ',' && !inQuotes) {
-      // End of field
-      result.push(current.trim());
-      current = '';
-    } else {
-      current += char;
-    }
-  }
-
-  // Add last field
-  result.push(current.trim());
-
-  return result;
-}
-
 /**
  * Convert parsed row to typed object with proper type conversions
  */
@@ -83,14 +39,16 @@ export function convertRowTypes(
 ): Record<string, string | number | null> {
   const result: Record<string, string | number | null> = {};
 
-  for (const [key, value] of Object.entries(row)) {
-    // Empty string means null/undefined
-    if (value === '') {
+  // Iterate over all expected columns, not just the ones present in the row
+  for (const [key, type] of Object.entries(columnTypes)) {
+    const value = row[key];
+
+    // Handle missing or empty values
+    if (value === undefined || value === null || value === '') {
       result[key] = null;
       continue;
     }
 
-    const type = columnTypes[key];
     if (type === 'INTEGER') {
       const parsed = parseInt(value, 10);
       result[key] = isNaN(parsed) ? null : parsed;

src/loaders/data-loader.ts

@@ -58,7 +58,8 @@ async function loadTableData(db: Database, schema: TableSchema, csvContent: stri
     const typedRow = convertRowTypes(row, columnTypes);
     const values = columns.map((col) => {
       const value = typedRow[col];
-      return value === null ? null : value;
+      // Handle undefined, null, or empty values
+      return value === null || value === undefined ? null : value;
     });
 
     try {

src/loaders/zip-loader.ts

@@ -50,16 +50,11 @@ export async function loadGTFSZip(source: string | ArrayBuffer | Uint8Array): Pr
  * Fetch ZIP file from URL or file path
  */
 async function fetchZip(source: string): Promise<ArrayBuffer> {
-  // Try to detect environment and use appropriate method
-  if (typeof fetch !== 'undefined') {
-    // Browser or Node.js with fetch support
-    const response = await fetch(source);
-    if (!response.ok) {
-      throw new Error(`Failed to fetch GTFS ZIP: ${response.status} ${response.statusText}`);
-    }
-    return await response.arrayBuffer();
-  } else if (typeof require !== 'undefined') {
-    // Node.js environment without fetch
+  // Check if source is a URL or file path
+  const isUrl = source.startsWith('http://') || source.startsWith('https://');
+
+  // If it's a file path, try to use fs first
+  if (!isUrl) {
     try {
       // Dynamic import for Node.js fs module
       const fs = await import('fs');
@@ -68,9 +63,19 @@ async function fetchZip(source: string): Promise<ArrayBuffer> {
     } catch (error) {
       throw new Error(`Failed to read GTFS ZIP file: ${error}`);
     }
-  } else {
-    throw new Error('No method available to fetch ZIP file');
   }
+
+  // For URLs, use fetch
+  if (typeof fetch !== 'undefined') {
+    // Browser or Node.js with fetch support
+    const response = await fetch(source);
+    if (!response.ok) {
+      throw new Error(`Failed to fetch GTFS ZIP: ${response.status} ${response.statusText}`);
+    }
+    return await response.arrayBuffer();
+  }
+
+  throw new Error('No method available to fetch ZIP file from URL');
 }
 
 /**

src/queries/stops.ts

@@ -78,6 +78,28 @@ export function getAllStops(db: Database, limit?: number): Stop[] {
   return stops;
 }
 
+/**
+ * Get stops for a given trip (ordered by stop_sequence)
+ */
+export function getStopsByTrip(db: Database, tripId: string): Stop[] {
+  const stmt = db.prepare(`
+    SELECT s.* FROM stops s
+    INNER JOIN stop_times st ON s.stop_id = st.stop_id
+    WHERE st.trip_id = ?
+    ORDER BY st.stop_sequence
+  `);
+  stmt.bind([tripId]);
+
+  const stops: Stop[] = [];
+  while (stmt.step()) {
+    const row = stmt.getAsObject() as Record<string, unknown>;
+    stops.push(rowToStop(row));
+  }
+
+  stmt.free();
+  return stops;
+}
+
 /**
  * Convert database row to Stop object
  */

src/queries/trips.ts

@@ -96,6 +96,28 @@ export function getTripsByRouteServiceAndDirection(
   return trips;
 }
 
+/**
+ * Get all trips for a service (active on a given date)
+ */
+export function getTripsByService(db: Database, serviceIds: string[]): Trip[] {
+  if (serviceIds.length === 0) {
+    return [];
+  }
+
+  const placeholders = serviceIds.map(() => '?').join(', ');
+  const stmt = db.prepare(`SELECT * FROM trips WHERE service_id IN (${placeholders})`);
+  stmt.bind(serviceIds);
+
+  const trips: Trip[] = [];
+  while (stmt.step()) {
+    const row = stmt.getAsObject() as Record<string, unknown>;
+    trips.push(rowToTrip(row));
+  }
+
+  stmt.free();
+  return trips;
+}
+
 /**
  * Convert database row to Trip object
  */

tests/fixtures/README.md

@@ -0,0 +1,38 @@
+# GTFS Sample Feed
+
+## Source
+
+This sample GTFS feed comes from:
+https://developers.google.com/transit/gtfs/examples/gtfs-feed
+
+## License
+
+This sample feed is licensed under the **Apache 2.0 License**.
+
+Copyright Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+## Description
+
+This is a simple GTFS feed provided by Google for testing and demonstration purposes.
+It contains sample transit data including:
+
+- Agency information
+- Stops
+- Routes
+- Trips
+- Stop times
+- Calendar data
+
+The feed is used in our test suite to verify the gtfs-sqljs library works correctly with real GTFS data.