diff --git a/PERFORMANCE_RESULTS.md b/PERFORMANCE_RESULTS.md deleted file mode 100644 index b030418..0000000 --- a/PERFORMANCE_RESULTS.md +++ /dev/null @@ -1,267 +0,0 @@ -# GTFS Data Loading Performance Results - -## Test Dataset: Irigo (Angers, France) - -**Source:** https://chouette.enroute.mobi/api/v1/datas/Irigo/gtfs.zip -**Size:** 4.0 MB (compressed) -**Test Date:** 2025-11-07 - -### Dataset Contents: -- **1,782 stops** -- **103 routes** -- **9,154 trips** -- **295,056 stop times** (largest table) -- **206,377 shape points** - ---- - -## Performance Comparison - -### Before Optimizations (commit 4202461) -``` -โฑ๏ธ Load Time: 45.00 seconds -๐Ÿ“Š Rate: ~6,556 rows/sec -``` - -### After Optimizations (commit fff37d2) -``` -โฑ๏ธ Load Time: 5.76 seconds -๐Ÿ“Š Rate: ~51,231 rows/sec -๐Ÿš€ SPEEDUP: 7.8x faster -``` - ---- - -## Implemented Optimizations - -### 1. โœ… Transaction Wrapping -**Impact:** 100-1000x speedup (most critical) - -**Before:** -```javascript -for (const row of rows) { - stmt.run(values); // Each INSERT auto-commits! -} -``` - -**After:** -```javascript -db.run('BEGIN TRANSACTION'); -for (const row of rows) { - stmt.run(values); // Single commit at end -} -db.run('COMMIT'); -``` - -**Benefit:** Eliminates disk writes per row, reduces to single disk write per table. - ---- - -### 2. โœ… SQLite PRAGMA Optimizations -**Impact:** 2-5x speedup - -```javascript -db.run('PRAGMA synchronous = OFF'); // Skip fsync -db.run('PRAGMA journal_mode = MEMORY'); // Memory journal -db.run('PRAGMA temp_store = MEMORY'); // Temp tables in RAM -db.run('PRAGMA cache_size = -64000'); // 64MB cache -db.run('PRAGMA locking_mode = EXCLUSIVE'); // No lock overhead -``` - -**Benefit:** Optimizes SQLite for bulk loading, reduces I/O operations. - ---- - -### 3. โœ… Deferred Index Creation -**Impact:** 2-4x speedup - -**Before:** -```javascript -// Create tables WITH indexes -CREATE TABLE stops (...); -CREATE INDEX idx_stops_code ON stops(stop_code); -// Then insert data (slow - updates index on every row) -``` - -**After:** -```javascript -// Create tables WITHOUT indexes -CREATE TABLE stops (...); -// Insert ALL data first (fast) -// Create indexes AFTER data is loaded -CREATE INDEX idx_stops_code ON stops(stop_code); -``` - -**Benefit:** Indexes built once on complete data vs. updated on every INSERT. - ---- - -### 4. โœ… Batch Inserts (1000 rows per batch) -**Impact:** 1.5-3x speedup - -**Before:** -```sql -INSERT INTO stops VALUES (?); -- Execute 1,782 times -``` - -**After:** -```sql -INSERT INTO stops VALUES (?), (?), ..., (?); -- Execute 2 times (1000 + 782) -``` - -**Benefit:** Reduces SQL parsing overhead and statement execution overhead. - ---- - -### 5. โœ… Progress Callback -**Impact:** User experience improvement - -```javascript -const gtfs = await GtfsSqlJs.fromZip('gtfs.zip', { - onProgress: (progress) => { - console.log(`${progress.percentComplete}% - ${progress.message}`); - } -}); -``` - -**Features:** -- Real-time progress tracking (0-100%) -- Phase information (downloading, extracting, inserting, indexing, etc.) -- Row counts and file names -- Perfect for web workers and UI progress bars - ---- - -### 6. โœ… ANALYZE Command -**Impact:** 1.1-2x query performance improvement - -```javascript -db.run('ANALYZE'); -``` - -**Benefit:** Updates SQLite query planner statistics for optimal query execution. - ---- - -## Micro-Benchmark: stop_times.txt Only - -Testing with 10,000 rows from the largest table: - -| Method | Time | Rate | Speedup | -|--------|------|------|---------| -| Unoptimized | 0.661s | 15,128 rows/sec | 1.0x | -| Optimized | 0.064s | 156,250 rows/sec | **10.3x** | - -**Note:** The micro-benchmark shows higher speedup (10.3x) because it isolates just the insertion improvements without ZIP extraction and other overhead. - ---- - -## Performance Breakdown by Phase - -### Optimized Version (5.76s total): - -| Phase | Time | Percentage | -|-------|------|------------| -| Initialization | 0.1s | 2% | -| ZIP Extraction | 0.5s | 9% | -| Schema Creation | 0.1s | 2% | -| **Data Insertion** | **3.8s** | **66%** | -| Index Creation | 1.0s | 17% | -| ANALYZE | 0.26s | 4% | - -**Key Finding:** Data insertion is the dominant phase (66% of time), which is why transaction wrapping and batch inserts have such high impact. - ---- - -## Real-World Impact Estimates - -### Medium Feeds (500k rows): -- **Before:** 60-120 seconds -- **After:** 3-5 seconds -- **Speedup:** ~20-40x - -### Large Feeds (4M rows - NYC MTA scale): -- **Before:** 10-20 minutes -- **After:** 20-40 seconds -- **Speedup:** ~30-60x - -### Very Large Feeds (10M+ rows): -- **Before:** 30-60+ minutes -- **After:** 60-120 seconds -- **Speedup:** ~30-60x - ---- - -## Memory Usage - -**Before:** ~150-200 MB peak (parsing + insertion overhead) -**After:** ~150-200 MB peak (same) - -**Note:** Memory usage remains similar because optimizations focus on I/O and SQL execution, not memory allocation. - ---- - -## Backwards Compatibility - -โœ… **100% backwards compatible** - -All existing code continues to work without changes: -```javascript -// Old code - still works -const gtfs = await GtfsSqlJs.fromZip('gtfs.zip'); - -// New code - with progress callback -const gtfs = await GtfsSqlJs.fromZip('gtfs.zip', { - onProgress: (p) => console.log(p.message) -}); -``` - ---- - -## Browser vs Node.js Performance - -Both environments show similar speedups: -- **Browser:** 7-8x faster (limited by WASM performance) -- **Node.js:** 8-10x faster (native SQLite performance) - ---- - -## Conclusion - -The implemented optimizations provide **~8x faster** loading for real-world GTFS feeds, with the following improvements: - -1. **Transaction wrapping** - Most critical (100-1000x theoretical, 3-5x in practice) -2. **PRAGMA optimizations** - Significant I/O reduction (2-5x) -3. **Deferred indexes** - Major improvement for indexed tables (2-4x) -4. **Batch inserts** - Reduces parsing overhead (1.5-3x) -5. **ANALYZE** - Better query performance post-load -6. **Progress callback** - Enables better UX - -**Combined Effect:** 7.8x speedup on Irigo dataset (45s โ†’ 5.76s) - -The optimizations are most effective on larger datasets where the I/O and transaction overhead dominates. Smaller feeds (<10k rows) show less dramatic improvements but still benefit significantly. - ---- - -## Test Commands - -```bash -# Test optimized version -git checkout claude/analyze-gtfs-sqlite-insertion-011CUtePFhbyGj88vrGVPRFs -npm run build -node performance-test.js - -# Test old version -git checkout 4202461 -npm run build -node test-old-version.js - -# Run micro-benchmark -node benchmark-comparison.js -``` - ---- - -**Implementation by Claude Code** -**Branch:** `claude/analyze-gtfs-sqlite-insertion-011CUtePFhbyGj88vrGVPRFs` -**Commit:** `fff37d2` diff --git a/README.md b/README.md index 2385606..c91d4ee 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@
- gtfs-sqljs logo + gtfs-sqljs logo

gtfs-sqljs

[![npm version](https://img.shields.io/npm/v/gtfs-sqljs)](https://www.npmjs.com/package/gtfs-sqljs) diff --git a/benchmark-comparison.js b/benchmark-comparison.js deleted file mode 100644 index 403b7fa..0000000 --- a/benchmark-comparison.js +++ /dev/null @@ -1,160 +0,0 @@ -/** - * Benchmark Comparison - Test a subset to show relative performance - */ - -const initSqlJs = require('sql.js'); -const JSZip = require('jszip'); -const fs = require('fs'); - -const GTFS_PATH = '/tmp/irigo-gtfs.zip'; - -async function loadStopTimesCSV() { - const buffer = fs.readFileSync(GTFS_PATH); - const zip = await JSZip.loadAsync(buffer); - const file = zip.file('stop_times.txt'); - return await file.async('string'); -} - -// Test with STOP_TIMES only (largest table) -async function benchmarkOptimized(rowLimit = 10000) { - console.log(`\n๐Ÿš€ OPTIMIZED: Testing with ${rowLimit.toLocaleString()} rows\n`); - - const SQL = await initSqlJs(); - const db = new SQL.Database(); - - // Create table without indexes - db.run(`CREATE TABLE stop_times ( - trip_id TEXT, - arrival_time TEXT, - departure_time TEXT, - stop_id TEXT, - stop_sequence INTEGER - )`); - - // Load data - const content = await loadStopTimesCSV(); - const lines = content.split('\n').filter(l => l.trim()).slice(0, rowLimit + 1); - const headers = lines[0].split(',').map(h => h.trim().replace(/"/g, '')); - - const startTime = Date.now(); - - // OPTIMIZED: Transaction + Batch inserts - // Apply PRAGMAs - db.run('PRAGMA synchronous = OFF'); - db.run('PRAGMA journal_mode = MEMORY'); - db.run('BEGIN TRANSACTION'); - - const BATCH_SIZE = 1000; - for (let i = 1; i < lines.length; i += BATCH_SIZE) { - const batchEnd = Math.min(i + BATCH_SIZE, lines.length); - const batchLines = lines.slice(i, batchEnd); - - const placeholders = batchLines.map(() => '(?,?,?,?,?)').join(','); - const sql = `INSERT INTO stop_times VALUES ${placeholders}`; - - const values = []; - for (const line of batchLines) { - const cols = line.split(',').map(v => v.trim().replace(/^"/, '').replace(/"$/, '') || null); - values.push(cols[0], cols[1], cols[2], cols[3], cols[4]); - } - - const stmt = db.prepare(sql); - stmt.run(values); - stmt.free(); - } - - db.run('COMMIT'); - - // Create index after data load - db.run('CREATE INDEX idx_trip ON stop_times(trip_id)'); - - const endTime = Date.now(); - const duration = (endTime - startTime) / 1000; - - console.log(` โœ… Time: ${duration.toFixed(3)}s`); - console.log(` โœ… Rate: ${Math.floor((rowLimit / duration))} rows/sec`); - - db.close(); - return duration; -} - -async function benchmarkUnoptimized(rowLimit = 10000) { - console.log(`\n๐ŸŒ UNOPTIMIZED: Testing with ${rowLimit.toLocaleString()} rows\n`); - - const SQL = await initSqlJs(); - const db = new SQL.Database(); - - // Create table WITH index (old way) - db.run(`CREATE TABLE stop_times ( - trip_id TEXT, - arrival_time TEXT, - departure_time TEXT, - stop_id TEXT, - stop_sequence INTEGER - )`); - db.run('CREATE INDEX idx_trip ON stop_times(trip_id)'); - - // Load data - const content = await loadStopTimesCSV(); - const lines = content.split('\n').filter(l => l.trim()).slice(0, rowLimit + 1); - - const startTime = Date.now(); - - // UNOPTIMIZED: No transaction, no batching, single-row inserts - const stmt = db.prepare('INSERT INTO stop_times VALUES (?,?,?,?,?)'); - - for (let i = 1; i < lines.length; i++) { - const cols = lines[i].split(',').map(v => v.trim().replace(/^"/, '').replace(/"$/, '') || null); - stmt.run([cols[0], cols[1], cols[2], cols[3], cols[4]]); - // Each run() auto-commits! (SLOW) - } - - stmt.free(); - - const endTime = Date.now(); - const duration = (endTime - startTime) / 1000; - - console.log(` โŒ Time: ${duration.toFixed(3)}s`); - console.log(` โŒ Rate: ${Math.floor((rowLimit / duration))} rows/sec`); - - db.close(); - return duration; -} - -async function runBenchmark() { - console.log('โ•'.repeat(80)); - console.log(' PERFORMANCE COMPARISON - stop_times.txt from Irigo Dataset'); - console.log('โ•'.repeat(80)); - - const testSize = 10000; - - const unoptimized = await benchmarkUnoptimized(testSize); - const optimized = await benchmarkOptimized(testSize); - - const speedup = unoptimized / optimized; - - console.log('\n' + 'โ•'.repeat(80)); - console.log(' BENCHMARK RESULTS'); - console.log('โ•'.repeat(80)); - console.log(`\n Test size: ${testSize.toLocaleString()} rows`); - console.log(`\n Unoptimized: ${unoptimized.toFixed(3)}s`); - console.log(` Optimized: ${optimized.toFixed(3)}s`); - console.log(`\n ๐Ÿš€ SPEEDUP: ${speedup.toFixed(1)}x faster\n`); - console.log(' Optimizations applied:'); - console.log(' โœ“ Transaction wrapping (no auto-commit per row)'); - console.log(' โœ“ Batch inserts (1000 rows per INSERT)'); - console.log(' โœ“ Deferred index creation (index after data load)'); - console.log(' โœ“ PRAGMA optimizations (memory journal, no sync)'); - console.log('\n Full dataset extrapolation (295,056 rows):'); - console.log(` Unoptimized estimate: ${((295056 / testSize) * unoptimized).toFixed(1)}s (~${(((295056 / testSize) * unoptimized) / 60).toFixed(1)} minutes)`); - console.log(` Optimized actual: 5.76s (measured)`); - console.log('โ•'.repeat(80)); -} - -runBenchmark().then(() => { - console.log('\nโœ… Benchmark complete!\n'); - process.exit(0); -}).catch((error) => { - console.error('\nโŒ Benchmark failed:', error); - process.exit(1); -}); diff --git a/dummy-test/package-lock.json b/dummy-test/package-lock.json deleted file mode 100644 index 67cff1d..0000000 --- a/dummy-test/package-lock.json +++ /dev/null @@ -1,249 +0,0 @@ -{ - "name": "dummy-test", - "version": "1.0.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "dummy-test", - "version": "1.0.0", - "dependencies": { - "gtfs-sqljs": "github:sysdevrun/gtfs-sqljs#main" - } - }, - "node_modules/@protobufjs/aspromise": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", - "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/base64": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", - "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/codegen": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", - "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/eventemitter": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", - "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/fetch": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", - "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", - "license": "BSD-3-Clause", - "dependencies": { - "@protobufjs/aspromise": "^1.1.1", - "@protobufjs/inquire": "^1.1.0" - } - }, - "node_modules/@protobufjs/float": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", - "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/inquire": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", - "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/path": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", - "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/pool": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", - "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/utf8": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", - "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", - "license": "BSD-3-Clause" - }, - "node_modules/@types/node": { - "version": "24.10.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.0.tgz", - "integrity": "sha512-qzQZRBqkFsYyaSWXuEHc2WR9c0a0CXwiE5FWUvn7ZM+vdy1uZLfCunD38UzhuB7YN/J11ndbDBcTmOdxJo9Q7A==", - "license": "MIT", - "dependencies": { - "undici-types": "~7.16.0" - } - }, - "node_modules/core-util-is": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", - "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", - "license": "MIT" - }, - "node_modules/gtfs-sqljs": { - "version": "1.0.0", - "resolved": "git+ssh://git@github.com/sysdevrun/gtfs-sqljs.git#b6a1908b068a5078839a3a64b71f4c2b502b508f", - "license": "MIT", - "dependencies": { - "jszip": "^3.10.1", - "papaparse": "^5.5.3", - "protobufjs": "^7.2.5", - "sql.js": "^1.10.3" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/immediate": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", - "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==", - "license": "MIT" - }, - "node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "license": "ISC" - }, - "node_modules/isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", - "license": "MIT" - }, - "node_modules/jszip": { - "version": "3.10.1", - "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz", - "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==", - "license": "(MIT OR GPL-3.0-or-later)", - "dependencies": { - "lie": "~3.3.0", - "pako": "~1.0.2", - "readable-stream": "~2.3.6", - "setimmediate": "^1.0.5" - } - }, - "node_modules/lie": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", - "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", - "license": "MIT", - "dependencies": { - "immediate": "~3.0.5" - } - }, - "node_modules/long": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", - "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", - "license": "Apache-2.0" - }, - "node_modules/pako": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", - "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", - "license": "(MIT AND Zlib)" - }, - "node_modules/papaparse": { - "version": "5.5.3", - "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.5.3.tgz", - "integrity": "sha512-5QvjGxYVjxO59MGU2lHVYpRWBBtKHnlIAcSe1uNFCkkptUh63NFRj0FJQm7nR67puEruUci/ZkjmEFrjCAyP4A==", - "license": "MIT" - }, - "node_modules/process-nextick-args": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", - "license": "MIT" - }, - "node_modules/protobufjs": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz", - "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", - "hasInstallScript": true, - "license": "BSD-3-Clause", - "dependencies": { - "@protobufjs/aspromise": "^1.1.2", - "@protobufjs/base64": "^1.1.2", - "@protobufjs/codegen": "^2.0.4", - "@protobufjs/eventemitter": "^1.1.0", - "@protobufjs/fetch": "^1.1.0", - "@protobufjs/float": "^1.0.2", - "@protobufjs/inquire": "^1.1.0", - "@protobufjs/path": "^1.1.2", - "@protobufjs/pool": "^1.1.0", - "@protobufjs/utf8": "^1.1.0", - "@types/node": ">=13.7.0", - "long": "^5.0.0" - }, - "engines": { - "node": ">=12.0.0" - } - }, - "node_modules/readable-stream": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", - "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", - "license": "MIT", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "license": "MIT" - }, - "node_modules/setimmediate": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", - "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==", - "license": "MIT" - }, - "node_modules/sql.js": { - "version": "1.13.0", - "resolved": "https://registry.npmjs.org/sql.js/-/sql.js-1.13.0.tgz", - "integrity": "sha512-RJbVP1HRDlUUXahJ7VMTcu9Rm1Nzw+EBpoPr94vnbD4LwR715F3CcxE2G2k45PewcaZ57pjetYa+LoSJLAASgA==", - "license": "MIT" - }, - "node_modules/string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "license": "MIT", - "dependencies": { - "safe-buffer": "~5.1.0" - } - }, - "node_modules/undici-types": { - "version": "7.16.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", - "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", - "license": "MIT" - }, - "node_modules/util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "license": "MIT" - } - } -} diff --git a/dummy-test/package.json b/dummy-test/package.json deleted file mode 100644 index d801705..0000000 --- a/dummy-test/package.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "name": "dummy-test", - "version": "1.0.0", - "description": "Test installation from GitHub", - "private": true, - "dependencies": { - "gtfs-sqljs": "github:sysdevrun/gtfs-sqljs#main" - } -} diff --git a/logo-128.png b/illustrations/logo-128.png similarity index 100% rename from logo-128.png rename to illustrations/logo-128.png diff --git a/logo-16.png b/illustrations/logo-16.png similarity index 100% rename from logo-16.png rename to illustrations/logo-16.png diff --git a/logo-256.png b/illustrations/logo-256.png similarity index 100% rename from logo-256.png rename to illustrations/logo-256.png diff --git a/logo-32.png b/illustrations/logo-32.png similarity index 100% rename from logo-32.png rename to illustrations/logo-32.png diff --git a/logo-512.png b/illustrations/logo-512.png similarity index 100% rename from logo-512.png rename to illustrations/logo-512.png diff --git a/logo-64.png b/illustrations/logo-64.png similarity index 100% rename from logo-64.png rename to illustrations/logo-64.png diff --git a/logo.svg b/illustrations/logo.svg similarity index 100% rename from logo.svg rename to illustrations/logo.svg diff --git a/performance-test.js b/performance-test.js deleted file mode 100644 index b9a9285..0000000 --- a/performance-test.js +++ /dev/null @@ -1,190 +0,0 @@ -/** - * Performance Test - Compare optimized vs unoptimized loading - */ - -const { GtfsSqlJs } = require('./dist/index.js'); - -const GTFS_PATH = '/tmp/irigo-gtfs.zip'; - -async function testOptimized() { - console.log('\n๐Ÿš€ Testing OPTIMIZED version (with all improvements)...\n'); - - const startTime = Date.now(); - let lastProgress = 0; - let progressUpdates = 0; - - const gtfs = await GtfsSqlJs.fromZip(GTFS_PATH, { - onProgress: (progress) => { - progressUpdates++; - if (progress.percentComplete - lastProgress >= 10 || progress.phase === 'complete') { - console.log(` ${progress.percentComplete}% - ${progress.message}`); - lastProgress = progress.percentComplete; - } - } - }); - - const endTime = Date.now(); - const duration = ((endTime - startTime) / 1000).toFixed(2); - - // Get some stats - const stops = gtfs.getStops(); - const routes = gtfs.getRoutes(); - const trips = gtfs.getTrips(); - const stopTimes = gtfs.getStopTimes(); - - console.log(`\nโœ… Optimized Load Complete!`); - console.log(` Time: ${duration}s`); - console.log(` Progress updates: ${progressUpdates}`); - console.log(` Data loaded:`); - console.log(` - ${stops.length} stops`); - console.log(` - ${routes.length} routes`); - console.log(` - ${trips.length} trips`); - console.log(` - ${stopTimes.length} stop times`); - - gtfs.close(); - - return { duration: parseFloat(duration), stopTimes: stopTimes.length }; -} - -async function testUnoptimizedSimulation() { - console.log('\n๐ŸŒ Testing UNOPTIMIZED simulation (no transactions, no batch, with indexes first)...\n'); - console.log(' Note: This simulates the old behavior with current codebase\n'); - - const initSqlJs = require('sql.js'); - const { loadGTFSZip } = require('./dist/index.js'); - const { getAllCreateStatements } = require('./dist/index.js'); - - const startTime = Date.now(); - - // Initialize SQL.js - const SQL = await initSqlJs(); - const db = new SQL.Database(); - - console.log(' Creating tables with indexes...'); - // Old way: create tables AND indexes together - const { GTFS_SCHEMA } = require('./dist/index.js'); - - // Create tables with indexes (old way) - for (const schema of GTFS_SCHEMA) { - // Create table - const columns = schema.columns.map((col) => { - const parts = [col.name, col.type]; - if (col.primaryKey) parts.push('PRIMARY KEY'); - if (col.required && !col.primaryKey) parts.push('NOT NULL'); - return parts.join(' '); - }); - db.run(`CREATE TABLE IF NOT EXISTS ${schema.name} (${columns.join(', ')})`); - - // Create indexes immediately (OLD WAY - SLOW) - if (schema.indexes) { - for (const idx of schema.indexes) { - const unique = idx.unique ? 'UNIQUE ' : ''; - const cols = idx.columns.join(', '); - db.run(`CREATE ${unique}INDEX IF NOT EXISTS ${idx.name} ON ${schema.name} (${cols})`); - } - } - } - - console.log(' Loading ZIP file...'); - const files = await loadGTFSZip(GTFS_PATH); - - console.log(' Inserting data (no transactions, no batching)...'); - - // Simulate old insertion method: no transaction, no batching - let totalRows = 0; - for (const [fileName, content] of Object.entries(files)) { - const tableName = fileName.replace('.txt', ''); - const schema = GTFS_SCHEMA.find(s => s.name === tableName); - if (!schema) continue; - - // Parse CSV manually - const lines = content.split('\n').filter(l => l.trim()); - if (lines.length < 2) continue; - - const headers = lines[0].split(',').map(h => h.trim().replace(/"/g, '')); - const validHeaders = headers.filter(h => schema.columns.some(c => c.name === h)); - - if (validHeaders.length === 0) continue; - - const placeholders = validHeaders.map(() => '?').join(', '); - const insertSQL = `INSERT INTO ${tableName} (${validHeaders.join(', ')}) VALUES (${placeholders})`; - const stmt = db.prepare(insertSQL); - - // OLD WAY: Insert each row individually WITHOUT transaction - for (let i = 1; i < lines.length; i++) { - const values = lines[i].split(',').map(v => { - v = v.trim().replace(/^"/, '').replace(/"$/, ''); - return v === '' ? null : v; - }); - - const rowValues = validHeaders.map((h, idx) => { - const headerIdx = headers.indexOf(h); - return headerIdx >= 0 ? (values[headerIdx] || null) : null; - }); - - try { - stmt.run(rowValues); // Each run auto-commits (SLOW!) - totalRows++; - } catch (e) { - // Skip errors for demo - } - } - - stmt.free(); - console.log(` Loaded ${tableName}: ${lines.length - 1} rows`); - } - - const endTime = Date.now(); - const duration = ((endTime - startTime) / 1000).toFixed(2); - - console.log(`\nโŒ Unoptimized Load Complete`); - console.log(` Time: ${duration}s`); - console.log(` Total rows: ${totalRows}`); - - db.close(); - - return { duration: parseFloat(duration), rows: totalRows }; -} - -async function runComparison() { - console.log('โ•'.repeat(80)); - console.log(' GTFS PERFORMANCE TEST - Irigo Dataset'); - console.log(' Source: https://chouette.enroute.mobi/api/v1/datas/Irigo/gtfs.zip'); - console.log(' File: ' + GTFS_PATH); - console.log('โ•'.repeat(80)); - - try { - // Test optimized version - const optimized = await testOptimized(); - - console.log('\n' + 'โ•'.repeat(80)); - console.log(' RESULTS SUMMARY'); - console.log('โ•'.repeat(80)); - console.log(`\n Optimized version: ${optimized.duration}s`); - console.log(` Data loaded: ${optimized.stopTimes.toLocaleString()} stop times`); - console.log(`\n โœจ Performance improvements active:`); - console.log(` โœ“ Transaction wrapping (100-1000x speedup)`); - console.log(` โœ“ PRAGMA optimizations (2-5x speedup)`); - console.log(` โœ“ Deferred index creation (2-4x speedup)`); - console.log(` โœ“ Batch inserts - 1000 rows per batch (1.5-3x speedup)`); - console.log(` โœ“ ANALYZE command (better query performance)`); - console.log(` โœ“ Progress callback (${optimized.stopTimes > 0 ? 'working' : 'not available'})`); - - console.log('\n Note: Testing unoptimized version would take ~100-1000x longer'); - console.log(' (estimated 5-30 minutes for this dataset)'); - console.log('\n' + 'โ•'.repeat(80)); - - } catch (error) { - console.error('\nโŒ Error during test:', error); - process.exit(1); - } -} - -// Run the test -runComparison().then(() => { - console.log('\nโœ… Test complete!\n'); - process.exit(0); -}).catch((error) => { - console.error('\nโŒ Test failed:', error); - process.exit(1); -}); diff --git a/test-old-version.js b/test-old-version.js deleted file mode 100644 index ca372a6..0000000 --- a/test-old-version.js +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Test OLD version (before optimizations) - */ - -const { GtfsSqlJs } = require('./dist/index.js'); - -async function testOldVersion() { - console.log('\n๐ŸŒ Testing OLD VERSION (before optimizations)...\n'); - - const startTime = Date.now(); - - const gtfs = await GtfsSqlJs.fromZip('/tmp/irigo-gtfs.zip'); - - const endTime = Date.now(); - const duration = ((endTime - startTime) / 1000).toFixed(2); - - // Get stats - const stops = gtfs.getStops(); - const routes = gtfs.getRoutes(); - const trips = gtfs.getTrips(); - const stopTimes = gtfs.getStopTimes(); - - console.log(`\nโŒ OLD Version Load Complete!`); - console.log(` Time: ${duration}s`); - console.log(` Data loaded:`); - console.log(` - ${stops.length} stops`); - console.log(` - ${routes.length} routes`); - console.log(` - ${trips.length} trips`); - console.log(` - ${stopTimes.length} stop times`); - - gtfs.close(); - - return parseFloat(duration); -} - -testOldVersion().then((duration) => { - console.log('\nโœ… Test complete!'); - console.log(`\n OLD VERSION: ${duration}s`); - console.log(` (Compare to NEW VERSION: ~5.76s)`); - console.log(` Expected speedup: ~100-200x\n`); - process.exit(0); -}).catch((error) => { - console.error('\nโŒ Test failed:', error); - process.exit(1); -}); diff --git a/test-realtime-local-file.ts b/test-realtime-local-file.ts deleted file mode 100644 index 48476a7..0000000 --- a/test-realtime-local-file.ts +++ /dev/null @@ -1,222 +0,0 @@ -/** - * Test script for GTFS Realtime with local Car Jaune RT file - */ - -import { GtfsSqlJs } from './src/gtfs-sqljs'; - -async function testRealtimeLocalFile() { - console.log('๐Ÿš€ Testing GTFS-RT with local Car Jaune RT file...\n'); - - try { - // Load GTFS static data from local file - console.log('๐Ÿ“ฆ Loading GTFS static data from local file...'); - const gtfs = await GtfsSqlJs.fromZip('./website/public/car-jaune.zip', { - skipFiles: ['shapes.txt'], - stalenessThreshold: 120 - }); - console.log('โœ… GTFS static data loaded\n'); - - // Show some basic GTFS info - const agencies = gtfs.getAgencies(); - console.log(`๐Ÿ“ Agencies: ${agencies.length}`); - agencies.forEach(agency => { - console.log(` - ${agency.agency_name} (${agency.agency_id})`); - }); - - const routes = gtfs.getRoutes({ limit: 5 }); - console.log(`\n๐ŸšŒ Routes: ${gtfs.getRoutes().length} total (showing first 5):`); - routes.forEach(route => { - console.log(` - ${route.route_short_name}: ${route.route_long_name}`); - }); - - // Load realtime data from local file - console.log('\n\n๐Ÿ“ก Loading GTFS-RT data from local file: ./car-jaune-rt.pb'); - await gtfs.fetchRealtimeData(['./car-jaune-rt.pb']); - console.log('โœ… GTFS-RT data loaded\n'); - - // Check alerts - console.log('๐Ÿšจ Checking alerts...'); - const allAlerts = gtfs.getAlerts(); - console.log(` Total alerts: ${allAlerts.length}`); - - const activeAlerts = gtfs.getAlerts({ activeOnly: true }); - console.log(` Active alerts: ${activeAlerts.length}`); - - if (allAlerts.length > 0) { - console.log('\n๐Ÿ“‹ Alert Details:'); - allAlerts.forEach((alert, idx) => { - console.log(`\n Alert #${idx + 1} (ID: ${alert.id})`); - - if (alert.header_text) { - try { - const header = typeof alert.header_text === 'string' - ? JSON.parse(alert.header_text) - : alert.header_text; - console.log(` Header: ${header.translation[0]?.text || 'N/A'}`); - } catch (e) { - console.log(` Header: [Unable to parse]`); - } - } - - if (alert.description_text) { - try { - const desc = typeof alert.description_text === 'string' - ? JSON.parse(alert.description_text) - : alert.description_text; - const text = desc.translation[0]?.text || 'N/A'; - // Truncate long descriptions - const truncated = text.length > 150 ? text.substring(0, 150) + '...' : text; - console.log(` Description: ${truncated}`); - } catch (e) { - console.log(` Description: [Unable to parse]`); - } - } - - console.log(` Cause: ${alert.cause || 'N/A'}`); - console.log(` Effect: ${alert.effect || 'N/A'}`); - - if (alert.informed_entity && alert.informed_entity.length > 0) { - console.log(` Affected entities: ${alert.informed_entity.length}`); - const routes = alert.informed_entity.filter(e => e.route_id).slice(0, 5); - if (routes.length > 0) { - console.log(` Routes: ${routes.map(e => e.route_id).join(', ')}${alert.informed_entity.filter(e => e.route_id).length > 5 ? ' ...' : ''}`); - } - const stops = alert.informed_entity.filter(e => e.stop_id).slice(0, 5); - if (stops.length > 0) { - console.log(` Stops: ${stops.map(e => e.stop_id).join(', ')}${alert.informed_entity.filter(e => e.stop_id).length > 5 ? ' ...' : ''}`); - } - } - - if (alert.active_period && alert.active_period.length > 0) { - console.log(` Active periods: ${alert.active_period.length}`); - alert.active_period.slice(0, 2).forEach(period => { - const start = period.start ? new Date(period.start * 1000).toISOString() : 'N/A'; - const end = period.end ? new Date(period.end * 1000).toISOString() : 'N/A'; - console.log(` - ${start} to ${end}`); - }); - } - - console.log(` Last updated: ${new Date(alert.rt_last_updated * 1000).toISOString()}`); - }); - } - - // Check vehicle positions - console.log('\n\n๐Ÿš— Checking vehicle positions...'); - const vehiclePositions = gtfs.getVehiclePositions(); - console.log(` Total vehicle positions: ${vehiclePositions.length}`); - - if (vehiclePositions.length > 0) { - console.log('\n๐Ÿ“ Vehicle Position Details (first 5):'); - vehiclePositions.slice(0, 5).forEach((vp, idx) => { - console.log(`\n Vehicle #${idx + 1}`); - console.log(` Trip ID: ${vp.trip_id}`); - console.log(` Route ID: ${vp.route_id || 'N/A'}`); - if (vp.vehicle?.id) console.log(` Vehicle ID: ${vp.vehicle.id}`); - if (vp.vehicle?.label) console.log(` Vehicle Label: ${vp.vehicle.label}`); - if (vp.position) { - console.log(` Position: ${vp.position.latitude.toFixed(6)}, ${vp.position.longitude.toFixed(6)}`); - if (vp.position.speed !== undefined) console.log(` Speed: ${vp.position.speed.toFixed(1)} m/s`); - if (vp.position.bearing !== undefined) console.log(` Bearing: ${vp.position.bearing.toFixed(0)}ยฐ`); - } - if (vp.current_stop_sequence) console.log(` Current stop sequence: ${vp.current_stop_sequence}`); - if (vp.stop_id) console.log(` Stop ID: ${vp.stop_id}`); - if (vp.timestamp) console.log(` Timestamp: ${new Date(vp.timestamp * 1000).toISOString()}`); - console.log(` Last updated: ${new Date(vp.rt_last_updated * 1000).toISOString()}`); - }); - } - - // Check trips with realtime data - console.log('\n\n๐Ÿ• Checking trips with realtime data...'); - const tripsWithRT = gtfs.getTrips({ limit: 20, includeRealtime: true }); - console.log(` Checked ${tripsWithRT.length} trips for realtime data`); - - const tripsWithData = tripsWithRT.filter((t: any) => - t.realtime?.vehicle_position !== null || t.realtime?.trip_update !== null - ); - console.log(` Trips with RT data: ${tripsWithData.length}`); - - if (tripsWithData.length > 0) { - console.log('\n๐Ÿ“Š Trip Realtime Details (first 3):'); - tripsWithData.slice(0, 3).forEach((trip: any, idx: number) => { - console.log(`\n Trip #${idx + 1} (ID: ${trip.trip_id})`); - console.log(` Route: ${trip.route_id}`); - console.log(` Headsign: ${trip.trip_headsign || 'N/A'}`); - - if (trip.realtime?.vehicle_position) { - const vp = trip.realtime.vehicle_position; - console.log(` โœ“ Has vehicle position`); - if (vp.position) { - console.log(` Position: ${vp.position.latitude.toFixed(6)}, ${vp.position.longitude.toFixed(6)}`); - } - if (vp.vehicle?.id) { - console.log(` Vehicle ID: ${vp.vehicle.id}`); - } - } - - if (trip.realtime?.trip_update) { - const tu = trip.realtime.trip_update; - console.log(` โœ“ Has trip update`); - if (tu.delay !== undefined) { - console.log(` Delay: ${tu.delay} seconds`); - } - if (tu.schedule_relationship !== undefined) { - console.log(` Schedule relationship: ${tu.schedule_relationship}`); - } - } - }); - } - - // Check stop times with realtime - console.log('\n\nโฐ Checking stop times with realtime...'); - if (tripsWithData.length > 0) { - const sampleTrip = tripsWithData[0]; - const stopTimes = gtfs.getStopTimes({ - tripId: sampleTrip.trip_id, - includeRealtime: true, - limit: 10 - }); - - console.log(` Stop times for trip ${sampleTrip.trip_id}: ${stopTimes.length}`); - - const stopTimesWithRT = stopTimes.filter((st: any) => st.realtime); - console.log(` Stop times with RT data: ${stopTimesWithRT.length}`); - - if (stopTimesWithRT.length > 0) { - console.log('\n Stop Time RT Details (first 5):'); - stopTimesWithRT.slice(0, 5).forEach((st: any) => { - const stop = gtfs.getStopById(st.stop_id); - console.log(`\n Stop: ${stop?.stop_name || st.stop_id}`); - console.log(` Sequence: ${st.stop_sequence}`); - console.log(` Scheduled arrival: ${st.arrival_time}`); - if (st.realtime?.arrival_delay !== undefined) { - console.log(` Arrival delay: ${st.realtime.arrival_delay} seconds`); - } - if (st.realtime?.departure_delay !== undefined) { - console.log(` Departure delay: ${st.realtime.departure_delay} seconds`); - } - if (st.realtime?.schedule_relationship !== undefined) { - console.log(` Schedule relationship: ${st.realtime.schedule_relationship}`); - } - }); - } - } - - console.log('\n\nโœ… Test completed successfully!'); - console.log('\nSummary:'); - console.log(` - Alerts: ${allAlerts.length} total, ${activeAlerts.length} active`); - console.log(` - Vehicle Positions: ${vehiclePositions.length}`); - console.log(` - Trips with RT data: ${tripsWithData.length}`); - - // Cleanup - gtfs.close(); - - } catch (error) { - console.error('\nโŒ Error during test:', error); - if (error instanceof Error) { - console.error('Stack:', error.stack); - } - process.exit(1); - } -} - -testRealtimeLocalFile(); diff --git a/test-realtime-local.ts b/test-realtime-local.ts deleted file mode 100644 index c2a377b..0000000 --- a/test-realtime-local.ts +++ /dev/null @@ -1,230 +0,0 @@ -/** - * Test script for GTFS Realtime API with local Car Jaune data - * Demonstrates RT functionality and API structure - */ - -import { GtfsSqlJs, AlertCause, AlertEffect } from './src/gtfs-sqljs'; - -async function testRealtimeLocal() { - console.log('๐Ÿš€ Testing GTFS-RT API with Car Jaune data...\n'); - - try { - // Load GTFS static data from local file - console.log('๐Ÿ“ฆ Loading GTFS static data from local file...'); - const gtfs = await GtfsSqlJs.fromZip('./website/public/car-jaune.zip', { - skipFiles: ['shapes.txt'], - stalenessThreshold: 120 - }); - console.log('โœ… GTFS static data loaded\n'); - - // Show some basic GTFS info - const agencies = gtfs.getAgencies(); - console.log(`๐Ÿ“ Agencies: ${agencies.length}`); - agencies.forEach(agency => { - console.log(` - ${agency.agency_name} (${agency.agency_id})`); - }); - - const routes = gtfs.getRoutes({ limit: 5 }); - console.log(`\n๐ŸšŒ Sample Routes (showing 5 of many):`); - routes.forEach(route => { - console.log(` - ${route.route_short_name}: ${route.route_long_name}`); - }); - - const stops = gtfs.getStops({ limit: 5 }); - console.log(`\n๐Ÿ›‘ Sample Stops (showing 5 of many):`); - stops.forEach(stop => { - console.log(` - ${stop.stop_name}`); - }); - - // Demonstrate RT configuration - console.log('\n\n๐Ÿ“ก GTFS-RT Configuration:'); - console.log('-----------------------------------'); - - // Set RT feed URLs - gtfs.setRealtimeFeedUrls([ - 'https://pysae.com/api/v2/groups/car-jaune/gtfs-rt' - ]); - console.log('โœ“ RT feed URLs configured'); - console.log(` URLs: ${gtfs.getRealtimeFeedUrls().join(', ')}`); - - // Check staleness threshold - console.log(`โœ“ Staleness threshold: ${gtfs.getStalenessThreshold()} seconds`); - - // Demonstrate RT API (without fetching real data) - console.log('\n\n๐Ÿ”ง GTFS-RT API Methods Available:'); - console.log('-----------------------------------'); - - console.log('\n1. fetchRealtimeData(urls?)'); - console.log(' - Fetches and loads RT data from configured URLs'); - console.log(' - Example: await gtfs.fetchRealtimeData()'); - console.log(' - Or: await gtfs.fetchRealtimeData(["https://custom-url"])'); - - console.log('\n2. getAlerts(filters?)'); - console.log(' - Get service alerts with optional filtering'); - console.log(' - Filters: activeOnly, routeId, stopId, tripId, cause, effect, limit'); - console.log(' - Example: gtfs.getAlerts({ activeOnly: true, routeId: "1" })'); - - console.log('\n3. getAlertById(alertId)'); - console.log(' - Get specific alert by ID'); - console.log(' - Example: gtfs.getAlertById("alert_123")'); - - console.log('\n4. getVehiclePositions(filters?)'); - console.log(' - Get vehicle positions with optional filtering'); - console.log(' - Filters: tripId, routeId, vehicleId, limit'); - console.log(' - Example: gtfs.getVehiclePositions({ routeId: "1" })'); - - console.log('\n5. getVehiclePositionByTripId(tripId)'); - console.log(' - Get vehicle position for specific trip'); - console.log(' - Example: gtfs.getVehiclePositionByTripId("trip_123")'); - - console.log('\n6. getTrips({ includeRealtime: true })'); - console.log(' - Get trips with realtime data merged'); - console.log(' - Returns vehicle position and trip update data'); - console.log(' - Example: gtfs.getTrips({ routeId: "1", includeRealtime: true })'); - - console.log('\n7. getStopTimes({ includeRealtime: true })'); - console.log(' - Get stop times with realtime delays merged'); - console.log(' - Returns nested realtime object with delays'); - console.log(' - Example: gtfs.getStopTimes({ tripId: "trip_123", includeRealtime: true })'); - - console.log('\n8. clearRealtimeData()'); - console.log(' - Clear all realtime data from database'); - - // Demonstrate data structure - console.log('\n\n๐Ÿ“Š GTFS-RT Data Structures:'); - console.log('-----------------------------------'); - - console.log('\nAlert structure:'); - console.log(`{ - id: string, - active_period: [{ start?: number, end?: number }], - informed_entity: [{ route_id?, stop_id?, trip?: {...} }], - cause?: AlertCause (enum), - effect?: AlertEffect (enum), - header_text?: TranslatedString, - description_text?: TranslatedString, - rt_last_updated: number -}`); - - console.log('\nVehiclePosition structure:'); - console.log(`{ - trip_id: string, - route_id?: string, - vehicle?: { id?, label?, license_plate? }, - position?: { latitude, longitude, bearing?, speed? }, - current_stop_sequence?: number, - stop_id?: string, - current_status?: VehicleStopStatus, - timestamp?: number, - rt_last_updated: number -}`); - - console.log('\nTripWithRealtime structure:'); - console.log(`{ - ...Trip (all static trip fields), - realtime?: { - vehicle_position: VehiclePosition | null, - trip_update: { - delay?: number, - schedule_relationship?: ScheduleRelationship - } | null - } -}`); - - console.log('\nStopTimeWithRealtime structure:'); - console.log(`{ - ...StopTime (all static stop time fields), - realtime?: { - arrival_delay?: number, - departure_delay?: number, - schedule_relationship?: ScheduleRelationship - } -}`); - - // Test with empty RT data (no fetch) - console.log('\n\n๐Ÿงช Testing RT Queries (no data yet):'); - console.log('-----------------------------------'); - - const alerts = gtfs.getAlerts(); - console.log(`โœ“ getAlerts(): ${alerts.length} alerts (expected 0 without fetch)`); - - const vehiclePositions = gtfs.getVehiclePositions(); - console.log(`โœ“ getVehiclePositions(): ${vehiclePositions.length} positions (expected 0 without fetch)`); - - const tripsWithRT = gtfs.getTrips({ limit: 3, includeRealtime: true }) as any[]; - console.log(`โœ“ getTrips({ includeRealtime: true }): ${tripsWithRT.length} trips`); - console.log(` First trip realtime structure: ${JSON.stringify(tripsWithRT[0]?.realtime || {})}`); - - const stopTimesWithRT = gtfs.getStopTimes({ tripId: tripsWithRT[0]?.trip_id, limit: 3, includeRealtime: true }) as any[]; - console.log(`โœ“ getStopTimes({ includeRealtime: true }): ${stopTimesWithRT.length} stop times`); - console.log(` First stop time realtime structure: ${JSON.stringify(stopTimesWithRT[0]?.realtime || 'undefined')}`); - - // Show how to use in production - console.log('\n\n๐Ÿ“š Production Usage Example:'); - console.log('-----------------------------------'); - console.log(` -// Initialize with RT feed URLs -const gtfs = await GtfsSqlJs.fromZip('gtfs.zip', { - realtimeFeedUrls: ['https://pysae.com/api/v2/groups/car-jaune/gtfs-rt'], - stalenessThreshold: 120 -}); - -// Fetch RT data (call periodically, e.g., every 30 seconds) -await gtfs.fetchRealtimeData(); - -// Get active alerts -const alerts = gtfs.getAlerts({ activeOnly: true }); -alerts.forEach(alert => { - console.log(\`Alert: \${alert.header_text}\`); - console.log(\`Affected routes: \${alert.informed_entity.filter(e => e.route_id).length}\`); -}); - -// Get trips with vehicle positions -const trips = gtfs.getTrips({ - routeId: '1', - date: '20250105', - includeRealtime: true -}); - -trips.forEach(trip => { - if (trip.realtime?.vehicle_position?.position) { - const pos = trip.realtime.vehicle_position.position; - console.log(\`Trip \${trip.trip_id} at \${pos.latitude}, \${pos.longitude}\`); - } - if (trip.realtime?.trip_update?.delay) { - console.log(\` Delayed by \${trip.realtime.trip_update.delay} seconds\`); - } -}); - -// Get stop times with realtime delays -const stopTimes = gtfs.getStopTimes({ - stopId: 'STOP_123', - routeId: '1', - date: '20250105', - includeRealtime: true -}); - -stopTimes.forEach(st => { - const delay = st.realtime?.arrival_delay || 0; - console.log(\`\${st.arrival_time} (delay: \${delay}s)\`); -}); -`); - - console.log('\nโœ… GTFS-RT API test completed successfully!'); - console.log('\nNote: To test with real data, use the URLs:'); - console.log(' GTFS: https://pysae.com/api/v2/groups/car-jaune/gtfs/pub'); - console.log(' GTFS-RT: https://pysae.com/api/v2/groups/car-jaune/gtfs-rt'); - - // Cleanup - gtfs.close(); - - } catch (error) { - console.error('\nโŒ Error during test:', error); - if (error instanceof Error) { - console.error('Stack:', error.stack); - } - process.exit(1); - } -} - -testRealtimeLocal(); diff --git a/test-realtime.ts b/test-realtime.ts deleted file mode 100644 index f839723..0000000 --- a/test-realtime.ts +++ /dev/null @@ -1,190 +0,0 @@ -/** - * Test script for GTFS Realtime with Car Jaune data - */ - -import { GtfsSqlJs } from './src/gtfs-sqljs'; - -async function testRealtime() { - console.log('๐Ÿš€ Testing GTFS-RT with Car Jaune data...\n'); - - try { - // Load GTFS static data - console.log('๐Ÿ“ฆ Loading GTFS static data...'); - const gtfs = await GtfsSqlJs.fromZip('https://pysae.com/api/v2/groups/car-jaune/gtfs/pub', { - skipFiles: ['shapes.txt'], - realtimeFeedUrls: ['https://pysae.com/api/v2/groups/car-jaune/gtfs-rt'], - stalenessThreshold: 120 - }); - console.log('โœ… GTFS static data loaded\n'); - - // Show some basic GTFS info - const agencies = gtfs.getAgencies(); - console.log(`๐Ÿ“ Agencies: ${agencies.length}`); - agencies.forEach(agency => { - console.log(` - ${agency.agency_name}`); - }); - - const routes = gtfs.getRoutes(); - console.log(`๐ŸšŒ Routes: ${routes.length}`); - - const stops = gtfs.getStops(); - console.log(`๐Ÿ›‘ Stops: ${stops.length}\n`); - - // Fetch realtime data - console.log('๐Ÿ“ก Fetching GTFS-RT data from: https://pysae.com/api/v2/groups/car-jaune/gtfs-rt'); - await gtfs.fetchRealtimeData(); - console.log('โœ… GTFS-RT data fetched\n'); - - // Check alerts - console.log('๐Ÿšจ Checking alerts...'); - const allAlerts = gtfs.getAlerts(); - console.log(` Total alerts: ${allAlerts.length}`); - - const activeAlerts = gtfs.getAlerts({ activeOnly: true }); - console.log(` Active alerts: ${activeAlerts.length}`); - - if (activeAlerts.length > 0) { - console.log('\n๐Ÿ“‹ Active Alert Details:'); - activeAlerts.forEach((alert, idx) => { - console.log(`\n Alert #${idx + 1} (ID: ${alert.id})`); - - if (alert.header_text) { - const header = JSON.parse(alert.header_text as any); - console.log(` Header: ${header.translation[0]?.text || 'N/A'}`); - } - - if (alert.description_text) { - const desc = JSON.parse(alert.description_text as any); - console.log(` Description: ${desc.translation[0]?.text || 'N/A'}`); - } - - console.log(` Cause: ${alert.cause || 'N/A'}`); - console.log(` Effect: ${alert.effect || 'N/A'}`); - - if (alert.informed_entity && alert.informed_entity.length > 0) { - console.log(` Affected entities: ${alert.informed_entity.length}`); - alert.informed_entity.slice(0, 3).forEach(entity => { - if (entity.route_id) console.log(` - Route: ${entity.route_id}`); - if (entity.stop_id) console.log(` - Stop: ${entity.stop_id}`); - if (entity.trip?.trip_id) console.log(` - Trip: ${entity.trip.trip_id}`); - }); - } - - if (alert.active_period && alert.active_period.length > 0) { - console.log(` Active periods: ${alert.active_period.length}`); - alert.active_period.slice(0, 2).forEach(period => { - const start = period.start ? new Date(period.start * 1000).toISOString() : 'N/A'; - const end = period.end ? new Date(period.end * 1000).toISOString() : 'N/A'; - console.log(` - ${start} to ${end}`); - }); - } - }); - } - - // Check vehicle positions - console.log('\n\n๐Ÿš— Checking vehicle positions...'); - const vehiclePositions = gtfs.getVehiclePositions(); - console.log(` Total vehicle positions: ${vehiclePositions.length}`); - - if (vehiclePositions.length > 0) { - console.log('\n๐Ÿ“ Vehicle Position Details (first 5):'); - vehiclePositions.slice(0, 5).forEach((vp, idx) => { - console.log(`\n Vehicle #${idx + 1}`); - console.log(` Trip ID: ${vp.trip_id}`); - console.log(` Route ID: ${vp.route_id || 'N/A'}`); - if (vp.vehicle?.id) console.log(` Vehicle ID: ${vp.vehicle.id}`); - if (vp.position) { - console.log(` Position: ${vp.position.latitude.toFixed(6)}, ${vp.position.longitude.toFixed(6)}`); - if (vp.position.speed) console.log(` Speed: ${vp.position.speed.toFixed(1)} m/s`); - if (vp.position.bearing) console.log(` Bearing: ${vp.position.bearing.toFixed(0)}ยฐ`); - } - if (vp.current_stop_sequence) console.log(` Current stop sequence: ${vp.current_stop_sequence}`); - if (vp.timestamp) console.log(` Timestamp: ${new Date(vp.timestamp * 1000).toISOString()}`); - }); - } - - // Check trip updates - console.log('\n\n๐Ÿ• Checking trip updates...'); - // Get a sample trip with realtime data - const tripsWithRT = gtfs.getTrips({ limit: 10, includeRealtime: true }); - console.log(` Checked ${tripsWithRT.length} trips for realtime data`); - - const tripsWithData = tripsWithRT.filter((t: any) => - t.realtime?.vehicle_position !== null || t.realtime?.trip_update !== null - ); - console.log(` Trips with RT data: ${tripsWithData.length}`); - - if (tripsWithData.length > 0) { - console.log('\n๐Ÿ“Š Trip Realtime Details (first 3):'); - tripsWithData.slice(0, 3).forEach((trip: any, idx: number) => { - console.log(`\n Trip #${idx + 1} (ID: ${trip.trip_id})`); - console.log(` Route: ${trip.route_id}`); - console.log(` Headsign: ${trip.trip_headsign || 'N/A'}`); - - if (trip.realtime?.vehicle_position) { - const vp = trip.realtime.vehicle_position; - console.log(` โœ“ Has vehicle position`); - if (vp.position) { - console.log(` Position: ${vp.position.latitude.toFixed(6)}, ${vp.position.longitude.toFixed(6)}`); - } - } - - if (trip.realtime?.trip_update) { - const tu = trip.realtime.trip_update; - console.log(` โœ“ Has trip update`); - if (tu.delay !== undefined) { - console.log(` Delay: ${tu.delay} seconds`); - } - } - }); - } - - // Check stop times with realtime - console.log('\n\nโฐ Checking stop times with realtime...'); - if (tripsWithData.length > 0) { - const sampleTrip = tripsWithData[0]; - const stopTimes = gtfs.getStopTimes({ - tripId: sampleTrip.trip_id, - includeRealtime: true, - limit: 5 - }); - - console.log(` Stop times for trip ${sampleTrip.trip_id}: ${stopTimes.length}`); - - const stopTimesWithRT = stopTimes.filter((st: any) => st.realtime); - console.log(` Stop times with RT data: ${stopTimesWithRT.length}`); - - if (stopTimesWithRT.length > 0) { - console.log('\n Stop Time RT Details:'); - stopTimesWithRT.slice(0, 3).forEach((st: any) => { - const stop = gtfs.getStopById(st.stop_id); - console.log(`\n Stop: ${stop?.stop_name || st.stop_id}`); - console.log(` Scheduled arrival: ${st.arrival_time}`); - if (st.realtime?.arrival_delay) { - console.log(` Arrival delay: ${st.realtime.arrival_delay} seconds`); - } - if (st.realtime?.departure_delay) { - console.log(` Departure delay: ${st.realtime.departure_delay} seconds`); - } - if (st.realtime?.schedule_relationship !== undefined) { - console.log(` Schedule relationship: ${st.realtime.schedule_relationship}`); - } - }); - } - } - - console.log('\n\nโœ… Test completed successfully!'); - - // Cleanup - gtfs.close(); - - } catch (error) { - console.error('\nโŒ Error during test:', error); - if (error instanceof Error) { - console.error('Stack:', error.stack); - } - process.exit(1); - } -} - -testRealtime();