diff --git a/CHANGELOG.md b/CHANGELOG.md index c0c6bd9..43b4dcd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ - Replace `as Record` widening casts on `stmt.getAsObject()` across all query files with proper `ParamsObject` type from sql.js - Replace non-null assertions (`!`) with optional chaining (`?.`) for `Map.get()` calls in rt-trip-updates and stop-times - Replace `this.SQL!.Database()` non-null assertion with explicit guard in gtfs-sqljs +- Extract only known GTFS files from ZIP, skipping unrecognized files for faster extraction +- `skipFiles` option now prevents files from being extracted from the ZIP entirely, not just from being loaded into the database ## 0.2.2 diff --git a/src/gtfs-sqljs.ts b/src/gtfs-sqljs.ts index b57c4f9..6e011d5 100644 --- a/src/gtfs-sqljs.ts +++ b/src/gtfs-sqljs.ts @@ -401,7 +401,7 @@ export class GtfsSqlJs { message: 'Extracting GTFS ZIP file', }); - const files = await loadGTFSZip(zipData); + const files = await loadGTFSZip(zipData, options.skipFiles); // Load GTFS data onProgress?.({ diff --git a/src/loaders/zip-loader.ts b/src/loaders/zip-loader.ts index becca5c..7417b57 100644 --- a/src/loaders/zip-loader.ts +++ b/src/loaders/zip-loader.ts @@ -12,25 +12,37 @@ export interface GTFSFiles { /** * Extract GTFS files from pre-loaded ZIP data + * Only extracts files defined in GTFS_FILE_MAPPING, skipping unknown files. + * @param zipData - ZIP data as ArrayBuffer or Uint8Array + * @param skipFiles - Optional array of GTFS filenames to skip extracting (e.g., ['shapes.txt']) */ -export async function loadGTFSZip(zipData: ArrayBuffer | Uint8Array): Promise { +export async function loadGTFSZip(zipData: ArrayBuffer | Uint8Array, skipFiles?: string[]): Promise { // Load ZIP file const zip = await JSZip.loadAsync(zipData); - // Extract all .txt files (GTFS files) + // Build set of files to skip for fast lookup + const skipSet = skipFiles ? new Set(skipFiles) : null; + + // Extract only known GTFS files, skipping any in skipFiles const files: GTFSFiles = {}; const filePromises: Promise[] = []; zip.forEach((relativePath, file) => { - // Only process .txt files - if (!file.dir && relativePath.endsWith('.txt')) { - const fileName = relativePath.split('/').pop() || relativePath; - filePromises.push( - file.async('string').then((content) => { - files[fileName] = content; - }) - ); - } + if (file.dir) return; + + const fileName = relativePath.split('/').pop() || relativePath; + + // Only extract files that are in GTFS_FILE_MAPPING + if (!(fileName in GTFS_FILE_MAPPING)) return; + + // Skip files the caller wants excluded + if (skipSet?.has(fileName)) return; + + filePromises.push( + file.async('string').then((content) => { + files[fileName] = content; + }) + ); }); await Promise.all(filePromises); diff --git a/tests/zip-loader.test.ts b/tests/zip-loader.test.ts new file mode 100644 index 0000000..caf342b --- /dev/null +++ b/tests/zip-loader.test.ts @@ -0,0 +1,88 @@ +/** + * Tests for selective ZIP extraction + */ + +import { describe, it, expect } from 'vitest'; +import { loadGTFSZip, GTFS_FILE_MAPPING } from '../src/loaders/zip-loader'; +import fs from 'fs'; +import path from 'path'; +import JSZip from 'jszip'; + +const feedPath = path.join(__dirname, 'fixtures', 'sample-feed.zip'); + +describe('loadGTFSZip', () => { + it('should only extract files defined in GTFS_FILE_MAPPING', async () => { + const zipData = await fs.promises.readFile(feedPath); + const files = await loadGTFSZip(zipData); + + for (const fileName of Object.keys(files)) { + expect(fileName in GTFS_FILE_MAPPING).toBe(true); + } + }); + + it('should not extract unknown .txt files from ZIP', async () => { + // Create a ZIP with a known GTFS file and an unknown file + const zip = new JSZip(); + zip.file('agency.txt', 'agency_id,agency_name\nDTA,Demo'); + zip.file('custom_data.txt', 'this,should,be,ignored'); + zip.file('notes.txt', 'random notes'); + + const zipData = await zip.generateAsync({ type: 'uint8array' }); + const files = await loadGTFSZip(zipData); + + expect(files['agency.txt']).toBeDefined(); + expect(files['custom_data.txt']).toBeUndefined(); + expect(files['notes.txt']).toBeUndefined(); + expect(Object.keys(files)).toEqual(['agency.txt']); + }); + + it('should skip files listed in skipFiles', async () => { + const zipData = await fs.promises.readFile(feedPath); + const files = await loadGTFSZip(zipData, ['shapes.txt', 'frequencies.txt']); + + expect(files['shapes.txt']).toBeUndefined(); + expect(files['frequencies.txt']).toBeUndefined(); + // Other files should still be present + expect(files['agency.txt']).toBeDefined(); + expect(files['stops.txt']).toBeDefined(); + expect(files['routes.txt']).toBeDefined(); + }); + + it('should extract all known GTFS files when no skipFiles specified', async () => { + const zipData = await fs.promises.readFile(feedPath); + const files = await loadGTFSZip(zipData); + + // The sample feed has these files + expect(files['agency.txt']).toBeDefined(); + expect(files['calendar.txt']).toBeDefined(); + expect(files['routes.txt']).toBeDefined(); + expect(files['stops.txt']).toBeDefined(); + expect(files['stop_times.txt']).toBeDefined(); + expect(files['trips.txt']).toBeDefined(); + expect(files['shapes.txt']).toBeDefined(); + }); + + it('should handle files in subdirectories within the ZIP', async () => { + const zip = new JSZip(); + zip.file('feed/agency.txt', 'agency_id,agency_name\nDTA,Demo'); + zip.file('feed/unknown.txt', 'should be ignored'); + + const zipData = await zip.generateAsync({ type: 'uint8array' }); + const files = await loadGTFSZip(zipData); + + expect(files['agency.txt']).toBeDefined(); + expect(files['unknown.txt']).toBeUndefined(); + }); + + it('should not extract non-.txt files', async () => { + const zip = new JSZip(); + zip.file('agency.txt', 'agency_id,agency_name\nDTA,Demo'); + zip.file('readme.md', '# GTFS Feed'); + zip.file('data.json', '{}'); + + const zipData = await zip.generateAsync({ type: 'uint8array' }); + const files = await loadGTFSZip(zipData); + + expect(Object.keys(files)).toEqual(['agency.txt']); + }); +});