From 769dbeb9feb293a6bf21ca9f243ba78ee2f3fc76 Mon Sep 17 00:00:00 2001 From: ProLoser <67395+ProLoser@users.noreply.github.com> Date: Tue, 17 Nov 2026 00:51:41 +0000 Subject: [PATCH] Fix 19hz venue extraction and geocoding errors Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- 19hz/scripts/extract-venues.js | 70 ++++++++++++++-------------------- 19hz/scripts/geocode-venues.js | 8 +++- 19hz/scripts/venues.test.js | 67 ++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 44 deletions(-) create mode 100644 19hz/scripts/venues.test.js diff --git a/19hz/scripts/extract-venues.js b/19hz/scripts/extract-venues.js index 96fc4a2..7a19762 100644 --- a/19hz/scripts/extract-venues.js +++ b/19hz/scripts/extract-venues.js @@ -1,25 +1,10 @@ const fs = require('fs'); const CSV_FILES = [ - https://19hz.info/events_BayArea.csv - https://19hz.info/events_LosAngeles.csv - https://19hz.info/events_Seattle.csv - https://19hz.info/events_Atlanta.csv - https://19hz.info/events_Miami.csv - https://19hz.info/events_DC.csv - https://19hz.info/events_Toronto.csv - https://19hz.info/events_Iowa.csv - https://19hz.info/events_Texas.csv - https://19hz.info/events_PHL.csv - https://19hz.info/events_Denver.csv - https://19hz.info/events_CHI.csv - https://19hz.info/events_Detroit.csv - https://19hz.info/events_Massachusetts.csv - https://19hz.info/events_LasVegas.csv - https://19hz.info/events_Phoenix.csv - https://19hz.info/events_ORE.csv - https://19hz.info/events_BC.csv -]; + 'BayArea', 'LosAngeles', 'Seattle', 'Atlanta', 'Miami', 'DC', 'Toronto', + 'Iowa', 'Texas', 'PHL', 'Denver', 'CHI', 'Detroit', 'Massachusetts', + 'LasVegas', 'Phoenix', 'ORE', 'BC' +].map(region => `https://19hz.info/events_${region}.csv`); function parseCSV(csvText) { const rows = []; @@ -57,9 +42,11 @@ function parseCSV(csvText) { return rows; } -function extractVenuesFromCSV(filename) { - console.log(`\nProcessing ${filename}...`); - const csvText = fs.readFileSync(filename, 'utf8'); +async function extractVenuesFromCSV(url) { + console.log(`\nProcessing ${url}...`); + const response = await fetch(url); + if (!response.ok) throw new Error(`Failed to fetch ${url}: HTTP ${response.status}`); + const csvText = await response.text(); const rows = parseCSV(csvText); const venues = new Set(); @@ -77,26 +64,25 @@ function extractVenuesFromCSV(filename) { return Array.from(venues).sort(); } -console.log('Extracting venues from CSV files...'); +async function main() { + console.log('Extracting venues from CSV files...'); + const allVenues = new Map(); -const allVenues = new Map(); - -CSV_FILES.forEach(file => { - const venues = extractVenuesFromCSV(file); - venues.forEach(venue => { - const match = venue.match(/^(.+?)\s*\((.+)\)$/); - if (match) { - const name = match[1].trim(); - const address = match[2].trim(); - allVenues.set(venue, { name: venue, address: address }); - } - }); -}); - -console.log(`\nTotal unique venues across all files: ${allVenues.size}`); -console.log('\nWriting to new-venues.json...'); + for (const file of CSV_FILES) { + const venues = await extractVenuesFromCSV(file); + venues.forEach(venue => { + const match = venue.match(/^(.+?)\s*\((.+)\)$/); + if (match) allVenues.set(venue, { name: venue, address: match[2].trim() }); + }); + } -const venueArray = Array.from(allVenues.values()); -fs.writeFileSync('../new-venues.json', JSON.stringify(venueArray, null, 2)); + if (!allVenues.size) throw new Error('No venues found in CSV files'); + console.log(`\nTotal unique venues across all files: ${allVenues.size}`); + fs.writeFileSync('../new-venues.json', JSON.stringify(Array.from(allVenues.values()), null, 2)); + console.log(`Done! Wrote ${allVenues.size} venues to new-venues.json`); +} -console.log(`Done! Wrote ${venueArray.length} venues to new-venues.json`); +main().catch(error => { + console.error('Failed to extract venues:', error); + process.exitCode = 1; +}); diff --git a/19hz/scripts/geocode-venues.js b/19hz/scripts/geocode-venues.js index b18bf94..3df511f 100644 --- a/19hz/scripts/geocode-venues.js +++ b/19hz/scripts/geocode-venues.js @@ -21,6 +21,11 @@ function geocodeAddress(address) { }; https.get(url, options, (response) => { + if (response.statusCode !== 200) { + response.resume(); + reject(new Error(`Geocoding request failed: HTTP ${response.statusCode}`)); + return; + } let data = ''; response.on('data', (chunk) => { @@ -97,8 +102,7 @@ async function processVenues() { await delay(DELAY_MS); } } catch (error) { - console.log(` ✗ Error: ${error.message}`); - failCount++; + throw new Error(`Failed to geocode ${venue.name}: ${error.message}`, { cause: error }); } } diff --git a/19hz/scripts/venues.test.js b/19hz/scripts/venues.test.js new file mode 100644 index 0000000..68ca454 --- /dev/null +++ b/19hz/scripts/venues.test.js @@ -0,0 +1,67 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); + +test('venue scripts extract, merge and geocode without losing existing data', t => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), '19hz-venues-')); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const scripts = path.join(directory, 'scripts'); + fs.mkdirSync(scripts); + for (const script of ['extract-venues.js', 'merge-venues.js', 'geocode-venues.js']) { + fs.copyFileSync(path.join(__dirname, script), path.join(scripts, script)); + } + const venuesFile = path.join(directory, 'venues.json'); + const existing = { name: 'Existing', address: 'San Francisco', geometry: { lat: 1, lng: 2 } }; + fs.writeFileSync(venuesFile, JSON.stringify([existing])); + const mock = path.join(directory, 'mock.js'); + fs.writeFileSync(mock, ` + const { EventEmitter } = require('node:events'); + global.fetch = async url => ({ + ok: !process.env.FAIL_FETCH, + status: 503, + text: async () => 'date,title,genre,"New Venue (Oakland)"\\n' + }); + require('node:https').get = (url, options, callback) => { + const request = new EventEmitter(); + process.nextTick(() => { + const response = new EventEmitter(); + response.statusCode = process.env.FAIL_GEOCODE ? 503 : 200; + response.resume = () => {}; + callback(response); + if (response.statusCode === 200) { + response.emit('data', '[{"lat":"37.8","lon":"-122.2"}]'); + response.emit('end'); + } + }); + return request; + }; + `); + const run = (script, env = {}) => spawnSync(process.execPath, ['--require', mock, script], { + cwd: scripts, + env: { ...process.env, ...env }, + encoding: 'utf8' + }); + + assert.equal(run('extract-venues.js').status, 0); + assert.deepEqual(JSON.parse(fs.readFileSync(path.join(directory, 'new-venues.json'))), [ + { name: 'New Venue (Oakland)', address: 'Oakland' } + ]); + assert.equal(run('merge-venues.js').status, 0); + assert.equal(run('geocode-venues.js').status, 0); + assert.deepEqual(JSON.parse(fs.readFileSync(venuesFile)), [ + existing, + { name: 'New Venue (Oakland)', address: 'Oakland', geometry: { lat: 37.8, lng: -122.2 } } + ]); + + const fetchFailure = run('extract-venues.js', { FAIL_FETCH: '1' }); + assert.notEqual(fetchFailure.status, 0); + assert.match(fetchFailure.stderr, /HTTP 503/); + + fs.writeFileSync(venuesFile, JSON.stringify([{ name: 'New Venue', address: 'Oakland' }])); + const geocodeFailure = run('geocode-venues.js', { FAIL_GEOCODE: '1' }); + assert.notEqual(geocodeFailure.status, 0); + assert.match(geocodeFailure.stderr, /HTTP 503/); +});