Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 28 additions & 42 deletions 19hz/scripts/extract-venues.js
Original file line number Diff line number Diff line change
@@ -1,25 +1,10 @@
const fs = require('fs');

const CSV_FILES = [
https://19hz.info/events_BayArea.csv
https://19hz.info/events_LosAngeles.csv
https://19hz.info/events_Seattle.csv
https://19hz.info/events_Atlanta.csv
https://19hz.info/events_Miami.csv
https://19hz.info/events_DC.csv
https://19hz.info/events_Toronto.csv
https://19hz.info/events_Iowa.csv
https://19hz.info/events_Texas.csv
https://19hz.info/events_PHL.csv
https://19hz.info/events_Denver.csv
https://19hz.info/events_CHI.csv
https://19hz.info/events_Detroit.csv
https://19hz.info/events_Massachusetts.csv
https://19hz.info/events_LasVegas.csv
https://19hz.info/events_Phoenix.csv
https://19hz.info/events_ORE.csv
https://19hz.info/events_BC.csv
];
'BayArea', 'LosAngeles', 'Seattle', 'Atlanta', 'Miami', 'DC', 'Toronto',
'Iowa', 'Texas', 'PHL', 'Denver', 'CHI', 'Detroit', 'Massachusetts',
'LasVegas', 'Phoenix', 'ORE', 'BC'
].map(region => `https://19hz.info/events_${region}.csv`);

function parseCSV(csvText) {
const rows = [];
Expand Down Expand Up @@ -57,9 +42,11 @@ function parseCSV(csvText) {
return rows;
}

function extractVenuesFromCSV(filename) {
console.log(`\nProcessing ${filename}...`);
const csvText = fs.readFileSync(filename, 'utf8');
async function extractVenuesFromCSV(url) {
console.log(`\nProcessing ${url}...`);
const response = await fetch(url);
if (!response.ok) throw new Error(`Failed to fetch ${url}: HTTP ${response.status}`);
const csvText = await response.text();
const rows = parseCSV(csvText);

const venues = new Set();
Expand All @@ -77,26 +64,25 @@ function extractVenuesFromCSV(filename) {
return Array.from(venues).sort();
}

console.log('Extracting venues from CSV files...');
async function main() {
console.log('Extracting venues from CSV files...');
const allVenues = new Map();

const allVenues = new Map();

CSV_FILES.forEach(file => {
const venues = extractVenuesFromCSV(file);
venues.forEach(venue => {
const match = venue.match(/^(.+?)\s*\((.+)\)$/);
if (match) {
const name = match[1].trim();
const address = match[2].trim();
allVenues.set(venue, { name: venue, address: address });
}
});
});

console.log(`\nTotal unique venues across all files: ${allVenues.size}`);
console.log('\nWriting to new-venues.json...');
for (const file of CSV_FILES) {
const venues = await extractVenuesFromCSV(file);
venues.forEach(venue => {
const match = venue.match(/^(.+?)\s*\((.+)\)$/);
if (match) allVenues.set(venue, { name: venue, address: match[2].trim() });
});
}

const venueArray = Array.from(allVenues.values());
fs.writeFileSync('../new-venues.json', JSON.stringify(venueArray, null, 2));
if (!allVenues.size) throw new Error('No venues found in CSV files');
console.log(`\nTotal unique venues across all files: ${allVenues.size}`);
fs.writeFileSync('../new-venues.json', JSON.stringify(Array.from(allVenues.values()), null, 2));
console.log(`Done! Wrote ${allVenues.size} venues to new-venues.json`);
}

console.log(`Done! Wrote ${venueArray.length} venues to new-venues.json`);
main().catch(error => {
console.error('Failed to extract venues:', error);
process.exitCode = 1;
});
8 changes: 6 additions & 2 deletions 19hz/scripts/geocode-venues.js
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,11 @@ function geocodeAddress(address) {
};

https.get(url, options, (response) => {
if (response.statusCode !== 200) {
response.resume();
reject(new Error(`Geocoding request failed: HTTP ${response.statusCode}`));
return;
}
let data = '';

response.on('data', (chunk) => {
Expand Down Expand Up @@ -97,8 +102,7 @@ async function processVenues() {
await delay(DELAY_MS);
}
} catch (error) {
console.log(` ✗ Error: ${error.message}`);
failCount++;
throw new Error(`Failed to geocode ${venue.name}: ${error.message}`, { cause: error });
}
}

Expand Down
67 changes: 67 additions & 0 deletions 19hz/scripts/venues.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
const { test } = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { spawnSync } = require('node:child_process');

test('venue scripts extract, merge and geocode without losing existing data', t => {
const directory = fs.mkdtempSync(path.join(os.tmpdir(), '19hz-venues-'));
t.after(() => fs.rmSync(directory, { recursive: true, force: true }));
const scripts = path.join(directory, 'scripts');
fs.mkdirSync(scripts);
for (const script of ['extract-venues.js', 'merge-venues.js', 'geocode-venues.js']) {
fs.copyFileSync(path.join(__dirname, script), path.join(scripts, script));
}
const venuesFile = path.join(directory, 'venues.json');
const existing = { name: 'Existing', address: 'San Francisco', geometry: { lat: 1, lng: 2 } };
fs.writeFileSync(venuesFile, JSON.stringify([existing]));
const mock = path.join(directory, 'mock.js');
fs.writeFileSync(mock, `
const { EventEmitter } = require('node:events');
global.fetch = async url => ({
ok: !process.env.FAIL_FETCH,
status: 503,
text: async () => 'date,title,genre,"New Venue (Oakland)"\\n'
});
require('node:https').get = (url, options, callback) => {
const request = new EventEmitter();
process.nextTick(() => {
const response = new EventEmitter();
response.statusCode = process.env.FAIL_GEOCODE ? 503 : 200;
response.resume = () => {};
callback(response);
if (response.statusCode === 200) {
response.emit('data', '[{"lat":"37.8","lon":"-122.2"}]');
response.emit('end');
}
});
return request;
};
`);
const run = (script, env = {}) => spawnSync(process.execPath, ['--require', mock, script], {
cwd: scripts,
env: { ...process.env, ...env },
encoding: 'utf8'
});

assert.equal(run('extract-venues.js').status, 0);
assert.deepEqual(JSON.parse(fs.readFileSync(path.join(directory, 'new-venues.json'))), [
{ name: 'New Venue (Oakland)', address: 'Oakland' }
]);
assert.equal(run('merge-venues.js').status, 0);
assert.equal(run('geocode-venues.js').status, 0);
assert.deepEqual(JSON.parse(fs.readFileSync(venuesFile)), [
existing,
{ name: 'New Venue (Oakland)', address: 'Oakland', geometry: { lat: 37.8, lng: -122.2 } }
]);

const fetchFailure = run('extract-venues.js', { FAIL_FETCH: '1' });
assert.notEqual(fetchFailure.status, 0);
assert.match(fetchFailure.stderr, /HTTP 503/);

fs.writeFileSync(venuesFile, JSON.stringify([{ name: 'New Venue', address: 'Oakland' }]));
const geocodeFailure = run('geocode-venues.js', { FAIL_GEOCODE: '1' });
assert.notEqual(geocodeFailure.status, 0);
assert.match(geocodeFailure.stderr, /HTTP 503/);
});
Loading