From 21562265246c2a10612e3088a79f65535e254df8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 8 Mar 2026 05:43:42 +0000 Subject: [PATCH] Extract only known GTFS files from ZIP, skip excluded files entirely Previously, loadGTFSZip extracted all .txt files from the ZIP archive, including unrecognized files that would never be used. The skipFiles option also only prevented loading into the database, not extraction. Now loadGTFSZip filters against GTFS_FILE_MAPPING during extraction and accepts a skipFiles parameter to avoid extracting excluded files at all. https://claude.ai/code/session_011hrgh6BbZURL7ECnKvZTub --- CHANGELOG.md | 2 + src/gtfs-sqljs.ts | 2 +- src/loaders/zip-loader.ts | 34 ++++++++++----- tests/zip-loader.test.ts | 88 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 114 insertions(+), 12 deletions(-) create mode 100644 tests/zip-loader.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index c0c6bd9..43b4dcd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ - Replace `as Record` widening casts on `stmt.getAsObject()` across all query files with proper `ParamsObject` type from sql.js - Replace non-null assertions (`!`) with optional chaining (`?.`) for `Map.get()` calls in rt-trip-updates and stop-times - Replace `this.SQL!.Database()` non-null assertion with explicit guard in gtfs-sqljs +- Extract only known GTFS files from ZIP, skipping unrecognized files for faster extraction +- `skipFiles` option now prevents files from being extracted from the ZIP entirely, not just from being loaded into the database ## 0.2.2 diff --git a/src/gtfs-sqljs.ts b/src/gtfs-sqljs.ts index b57c4f9..6e011d5 100644 --- a/src/gtfs-sqljs.ts +++ b/src/gtfs-sqljs.ts @@ -401,7 +401,7 @@ export class GtfsSqlJs { message: 'Extracting GTFS ZIP file', }); - const files = await loadGTFSZip(zipData); + const files = await loadGTFSZip(zipData, options.skipFiles); // Load GTFS data onProgress?.({ diff --git a/src/loaders/zip-loader.ts b/src/loaders/zip-loader.ts index becca5c..7417b57 100644 --- a/src/loaders/zip-loader.ts +++ b/src/loaders/zip-loader.ts @@ -12,25 +12,37 @@ export interface GTFSFiles { /** * Extract GTFS files from pre-loaded ZIP data + * Only extracts files defined in GTFS_FILE_MAPPING, skipping unknown files. + * @param zipData - ZIP data as ArrayBuffer or Uint8Array + * @param skipFiles - Optional array of GTFS filenames to skip extracting (e.g., ['shapes.txt']) */ -export async function loadGTFSZip(zipData: ArrayBuffer | Uint8Array): Promise { +export async function loadGTFSZip(zipData: ArrayBuffer | Uint8Array, skipFiles?: string[]): Promise { // Load ZIP file const zip = await JSZip.loadAsync(zipData); - // Extract all .txt files (GTFS files) + // Build set of files to skip for fast lookup + const skipSet = skipFiles ? new Set(skipFiles) : null; + + // Extract only known GTFS files, skipping any in skipFiles const files: GTFSFiles = {}; const filePromises: Promise[] = []; zip.forEach((relativePath, file) => { - // Only process .txt files - if (!file.dir && relativePath.endsWith('.txt')) { - const fileName = relativePath.split('/').pop() || relativePath; - filePromises.push( - file.async('string').then((content) => { - files[fileName] = content; - }) - ); - } + if (file.dir) return; + + const fileName = relativePath.split('/').pop() || relativePath; + + // Only extract files that are in GTFS_FILE_MAPPING + if (!(fileName in GTFS_FILE_MAPPING)) return; + + // Skip files the caller wants excluded + if (skipSet?.has(fileName)) return; + + filePromises.push( + file.async('string').then((content) => { + files[fileName] = content; + }) + ); }); await Promise.all(filePromises); diff --git a/tests/zip-loader.test.ts b/tests/zip-loader.test.ts new file mode 100644 index 0000000..caf342b --- /dev/null +++ b/tests/zip-loader.test.ts @@ -0,0 +1,88 @@ +/** + * Tests for selective ZIP extraction + */ + +import { describe, it, expect } from 'vitest'; +import { loadGTFSZip, GTFS_FILE_MAPPING } from '../src/loaders/zip-loader'; +import fs from 'fs'; +import path from 'path'; +import JSZip from 'jszip'; + +const feedPath = path.join(__dirname, 'fixtures', 'sample-feed.zip'); + +describe('loadGTFSZip', () => { + it('should only extract files defined in GTFS_FILE_MAPPING', async () => { + const zipData = await fs.promises.readFile(feedPath); + const files = await loadGTFSZip(zipData); + + for (const fileName of Object.keys(files)) { + expect(fileName in GTFS_FILE_MAPPING).toBe(true); + } + }); + + it('should not extract unknown .txt files from ZIP', async () => { + // Create a ZIP with a known GTFS file and an unknown file + const zip = new JSZip(); + zip.file('agency.txt', 'agency_id,agency_name\nDTA,Demo'); + zip.file('custom_data.txt', 'this,should,be,ignored'); + zip.file('notes.txt', 'random notes'); + + const zipData = await zip.generateAsync({ type: 'uint8array' }); + const files = await loadGTFSZip(zipData); + + expect(files['agency.txt']).toBeDefined(); + expect(files['custom_data.txt']).toBeUndefined(); + expect(files['notes.txt']).toBeUndefined(); + expect(Object.keys(files)).toEqual(['agency.txt']); + }); + + it('should skip files listed in skipFiles', async () => { + const zipData = await fs.promises.readFile(feedPath); + const files = await loadGTFSZip(zipData, ['shapes.txt', 'frequencies.txt']); + + expect(files['shapes.txt']).toBeUndefined(); + expect(files['frequencies.txt']).toBeUndefined(); + // Other files should still be present + expect(files['agency.txt']).toBeDefined(); + expect(files['stops.txt']).toBeDefined(); + expect(files['routes.txt']).toBeDefined(); + }); + + it('should extract all known GTFS files when no skipFiles specified', async () => { + const zipData = await fs.promises.readFile(feedPath); + const files = await loadGTFSZip(zipData); + + // The sample feed has these files + expect(files['agency.txt']).toBeDefined(); + expect(files['calendar.txt']).toBeDefined(); + expect(files['routes.txt']).toBeDefined(); + expect(files['stops.txt']).toBeDefined(); + expect(files['stop_times.txt']).toBeDefined(); + expect(files['trips.txt']).toBeDefined(); + expect(files['shapes.txt']).toBeDefined(); + }); + + it('should handle files in subdirectories within the ZIP', async () => { + const zip = new JSZip(); + zip.file('feed/agency.txt', 'agency_id,agency_name\nDTA,Demo'); + zip.file('feed/unknown.txt', 'should be ignored'); + + const zipData = await zip.generateAsync({ type: 'uint8array' }); + const files = await loadGTFSZip(zipData); + + expect(files['agency.txt']).toBeDefined(); + expect(files['unknown.txt']).toBeUndefined(); + }); + + it('should not extract non-.txt files', async () => { + const zip = new JSZip(); + zip.file('agency.txt', 'agency_id,agency_name\nDTA,Demo'); + zip.file('readme.md', '# GTFS Feed'); + zip.file('data.json', '{}'); + + const zipData = await zip.generateAsync({ type: 'uint8array' }); + const files = await loadGTFSZip(zipData); + + expect(Object.keys(files)).toEqual(['agency.txt']); + }); +});