Extract only known GTFS files from ZIP, skip excluded files entirely

CHANGELOG.md

@@ -13,6 +13,8 @@
 - Replace `as Record<string, unknown>` widening casts on `stmt.getAsObject()` across all query files with proper `ParamsObject` type from sql.js
 - Replace non-null assertions (`!`) with optional chaining (`?.`) for `Map.get()` calls in rt-trip-updates and stop-times
 - Replace `this.SQL!.Database()` non-null assertion with explicit guard in gtfs-sqljs
+- Extract only known GTFS files from ZIP, skipping unrecognized files for faster extraction
+- `skipFiles` option now prevents files from being extracted from the ZIP entirely, not just from being loaded into the database
 
 ## 0.2.2
 

src/gtfs-sqljs.ts

@@ -401,7 +401,7 @@ export class GtfsSqlJs {
       message: 'Extracting GTFS ZIP file',
     });
 
-    const files = await loadGTFSZip(zipData);
+    const files = await loadGTFSZip(zipData, options.skipFiles);
 
     // Load GTFS data
     onProgress?.({

src/loaders/zip-loader.ts

@@ -12,25 +12,37 @@ export interface GTFSFiles {
 
 /**
  * Extract GTFS files from pre-loaded ZIP data
+ * Only extracts files defined in GTFS_FILE_MAPPING, skipping unknown files.
+ * @param zipData - ZIP data as ArrayBuffer or Uint8Array
+ * @param skipFiles - Optional array of GTFS filenames to skip extracting (e.g., ['shapes.txt'])
  */
-export async function loadGTFSZip(zipData: ArrayBuffer | Uint8Array): Promise<GTFSFiles> {
+export async function loadGTFSZip(zipData: ArrayBuffer | Uint8Array, skipFiles?: string[]): Promise<GTFSFiles> {
   // Load ZIP file
   const zip = await JSZip.loadAsync(zipData);
 
-  // Extract all .txt files (GTFS files)
+  // Build set of files to skip for fast lookup
+  const skipSet = skipFiles ? new Set(skipFiles) : null;
+
+  // Extract only known GTFS files, skipping any in skipFiles
   const files: GTFSFiles = {};
   const filePromises: Promise<void>[] = [];
 
   zip.forEach((relativePath, file) => {
-    // Only process .txt files
-    if (!file.dir && relativePath.endsWith('.txt')) {
-      const fileName = relativePath.split('/').pop() || relativePath;
-      filePromises.push(
-        file.async('string').then((content) => {
-          files[fileName] = content;
-        })
-      );
-    }
+    if (file.dir) return;
+
+    const fileName = relativePath.split('/').pop() || relativePath;
+
+    // Only extract files that are in GTFS_FILE_MAPPING
+    if (!(fileName in GTFS_FILE_MAPPING)) return;
+
+    // Skip files the caller wants excluded
+    if (skipSet?.has(fileName)) return;
+
+    filePromises.push(
+      file.async('string').then((content) => {
+        files[fileName] = content;
+      })
+    );
   });
 
   await Promise.all(filePromises);

tests/zip-loader.test.ts

@@ -0,0 +1,88 @@
+/**
+ * Tests for selective ZIP extraction
+ */
+
+import { describe, it, expect } from 'vitest';
+import { loadGTFSZip, GTFS_FILE_MAPPING } from '../src/loaders/zip-loader';
+import fs from 'fs';
+import path from 'path';
+import JSZip from 'jszip';
+
+const feedPath = path.join(__dirname, 'fixtures', 'sample-feed.zip');
+
+describe('loadGTFSZip', () => {
+  it('should only extract files defined in GTFS_FILE_MAPPING', async () => {
+    const zipData = await fs.promises.readFile(feedPath);
+    const files = await loadGTFSZip(zipData);
+
+    for (const fileName of Object.keys(files)) {
+      expect(fileName in GTFS_FILE_MAPPING).toBe(true);
+    }
+  });
+
+  it('should not extract unknown .txt files from ZIP', async () => {
+    // Create a ZIP with a known GTFS file and an unknown file
+    const zip = new JSZip();
+    zip.file('agency.txt', 'agency_id,agency_name\nDTA,Demo');
+    zip.file('custom_data.txt', 'this,should,be,ignored');
+    zip.file('notes.txt', 'random notes');
+
+    const zipData = await zip.generateAsync({ type: 'uint8array' });
+    const files = await loadGTFSZip(zipData);
+
+    expect(files['agency.txt']).toBeDefined();
+    expect(files['custom_data.txt']).toBeUndefined();
+    expect(files['notes.txt']).toBeUndefined();
+    expect(Object.keys(files)).toEqual(['agency.txt']);
+  });
+
+  it('should skip files listed in skipFiles', async () => {
+    const zipData = await fs.promises.readFile(feedPath);
+    const files = await loadGTFSZip(zipData, ['shapes.txt', 'frequencies.txt']);
+
+    expect(files['shapes.txt']).toBeUndefined();
+    expect(files['frequencies.txt']).toBeUndefined();
+    // Other files should still be present
+    expect(files['agency.txt']).toBeDefined();
+    expect(files['stops.txt']).toBeDefined();
+    expect(files['routes.txt']).toBeDefined();
+  });
+
+  it('should extract all known GTFS files when no skipFiles specified', async () => {
+    const zipData = await fs.promises.readFile(feedPath);
+    const files = await loadGTFSZip(zipData);
+
+    // The sample feed has these files
+    expect(files['agency.txt']).toBeDefined();
+    expect(files['calendar.txt']).toBeDefined();
+    expect(files['routes.txt']).toBeDefined();
+    expect(files['stops.txt']).toBeDefined();
+    expect(files['stop_times.txt']).toBeDefined();
+    expect(files['trips.txt']).toBeDefined();
+    expect(files['shapes.txt']).toBeDefined();
+  });
+
+  it('should handle files in subdirectories within the ZIP', async () => {
+    const zip = new JSZip();
+    zip.file('feed/agency.txt', 'agency_id,agency_name\nDTA,Demo');
+    zip.file('feed/unknown.txt', 'should be ignored');
+
+    const zipData = await zip.generateAsync({ type: 'uint8array' });
+    const files = await loadGTFSZip(zipData);
+
+    expect(files['agency.txt']).toBeDefined();
+    expect(files['unknown.txt']).toBeUndefined();
+  });
+
+  it('should not extract non-.txt files', async () => {
+    const zip = new JSZip();
+    zip.file('agency.txt', 'agency_id,agency_name\nDTA,Demo');
+    zip.file('readme.md', '# GTFS Feed');
+    zip.file('data.json', '{}');
+
+    const zipData = await zip.generateAsync({ type: 'uint8array' });
+    const files = await loadGTFSZip(zipData);
+
+    expect(Object.keys(files)).toEqual(['agency.txt']);
+  });
+});