Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 0 additions & 26 deletions .github/workflows/codeql.yml

This file was deleted.

13 changes: 12 additions & 1 deletion .github/workflows/update-19hz-venues.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,11 @@ name: Update 19hz venues

on:
workflow_dispatch:
inputs:
refresh_city_only:
description: Re-geocode existing venues whose address is only their city
type: boolean
default: false

permissions:
contents: write
Expand All @@ -23,10 +28,16 @@ jobs:
node-version: 24
- name: Extract, merge, and geocode venues
working-directory: 19hz/scripts
env:
REFRESH_CITY_ONLY: ${{ inputs.refresh_city_only }}
run: |
node extract-venues.js
node merge-venues.js
node geocode-venues.js
if [ "$REFRESH_CITY_ONLY" = "true" ]; then
node geocode-venues.js --refresh-city-only
else
node geocode-venues.js
fi
- name: Create pull request
uses: peter-evans/create-pull-request@v7
with:
Expand Down
2 changes: 2 additions & 0 deletions 19hz/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,6 @@ Variation of my funcheapsfmap based off of data scraped from 19hz with a spotify
## Venues
Venues are stored in [venues.json](./venues.json). To update them, run **Update 19hz venues** from the repository's Actions tab. The workflow runs the extraction, merge, and geocoding scripts and opens or updates a pull request when the venue file changes.

Geocoding searches for the venue name and address together. To retry existing extracted venues whose address is only a city, select **Re-geocode existing venues whose address is only their city** when starting the workflow, or run `node geocode-venues.js --refresh-city-only` from `19hz/scripts`. This can take a long time; existing coordinates are retained when a search has no result.

The repository's Actions settings must allow GitHub Actions to create pull requests. The workflow uses `GITHUB_TOKEN` with contents and pull request write permissions.
17 changes: 7 additions & 10 deletions 19hz/map.js
Original file line number Diff line number Diff line change
Expand Up @@ -286,7 +286,6 @@ window.filter = async function (filters = {}) {
return event.visible;
}).length;

// Group visible events by location for clustering
// First, hide all markers
window.events.get()?.forEach(event => {
if (!event.title || !event.geometry) return;
Expand All @@ -295,19 +294,17 @@ window.filter = async function (filters = {}) {
}
});

// Group visible events by location
const locationGroups = new Map();
const venueGroups = new Map();

window.events.get()?.forEach(event => {
if (!event.title || !event.geometry || !event.visible) return;

// Create location key with rounded coordinates (to handle minor differences)
const locationKey = `${event.geometry.lat.toFixed(6)},${event.geometry.lng.toFixed(6)}`;

if (!locationGroups.has(locationKey)) {
locationGroups.set(locationKey, []);
const venueKey = event.venue?.trim().toLowerCase() || event;

if (!venueGroups.has(venueKey)) {
venueGroups.set(venueKey, []);
}
locationGroups.get(locationKey).push(event);
venueGroups.get(venueKey).push(event);
});

// Load marker library
Expand All @@ -316,7 +313,7 @@ window.filter = async function (filters = {}) {
const PinElement = markerLibrary.PinElement;

// Create markers based on grouping
locationGroups.forEach((eventsAtLocation, locationKey) => {
venueGroups.forEach(eventsAtLocation => {
if (eventsAtLocation.length === 1) {
// Single event - show normal marker
const event = eventsAtLocation[0];
Expand Down
58 changes: 58 additions & 0 deletions 19hz/map.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
const { test } = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const vm = require('node:vm');

test('filter groups events by venue rather than shared coordinates', async () => {
const markers = [];
class AdvancedMarkerElement {
constructor(options) {
Object.assign(this, options);
markers.push(this);
}
addEventListener() {}
}
class PinElement {
constructor(options) {
Object.assign(this, options);
}
}
const map = {};
const geometry = { lat: 37.779, lng: -122.419 };
const events = [
{ title: 'First', venue: 'Venue A', geometry, date: '2026-01-01', categories: ['House'] },
{ title: 'Second', venue: 'Venue A', geometry: { lat: 37.78, lng: -122.42 }, date: '2026-01-02', categories: ['Techno'] },
{ title: 'Third', venue: 'Venue B', geometry, date: '2026-01-01', categories: ['House'] },
{ title: 'Fourth', venue: '', geometry, date: '2026-01-01', categories: ['House'] },
{ title: 'Fifth', venue: '', geometry, date: '2026-01-01', categories: ['House'] }
];
events.forEach(event => event.marker = { map, content: { style: {} } });
const form = { elements: { countEvents: {}, countCategories: {} } };
const context = {
window: { map, events: { get: () => events }, history: { replaceState() {} } },
document: { getElementById: () => form },
IntersectionObserver: class { observe() {} },
google: { maps: { importLibrary: async () => ({ AdvancedMarkerElement, PinElement }) } }
};
vm.runInNewContext(fs.readFileSync(path.join(__dirname, 'map.js'), 'utf8'), context);

await context.window.filter();
assert.equal(markers.length, 1);
assert.equal(markers[0].title, '2 events at Venue A');
assert.equal(markers[0].content.glyphText, '2');
assert.equal(events[0].marker.map, null);
assert.equal(events[1].marker.map, null);
assert.equal(events[2].marker.map, map);
assert.equal(events[3].marker.map, map);
assert.equal(events[4].marker.map, map);
assert.equal(form.elements.countEvents.innerText, 5);

await context.window.filter({ category: 'House' });
assert.equal(markers[0].map, null);
assert.equal(markers.length, 1);
assert.equal(events[0].marker.map, map);
assert.equal(events[1].marker.map, null);
assert.equal(events[2].marker.map, map);
assert.equal(form.elements.countEvents.innerText, 4);
});
27 changes: 21 additions & 6 deletions 19hz/scripts/geocode-venues.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,18 @@ function delay(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}

function getVenueQuery(venue) {
const match = venue.name?.match(/^(.+?)\s*\(([^()]*)\)\s*$/);
const name = (match ? match[1] : venue.name || '').trim();
const address = (venue.address || match?.[2] || '').trim();
return name && address ? `${name}, ${address}` : '';
}

function hasCityOnlyAddress(venue) {
const match = venue.name?.match(/^(.+?)\s*\(([^()]*)\)\s*$/);
return match && match[2].trim().toLowerCase() === venue.address?.trim().toLowerCase();
}

function geocodeAddress(address) {
return new Promise((resolve, reject) => {
const encodedAddress = encodeURIComponent(address);
Expand All @@ -35,7 +47,8 @@ function geocodeAddress(address) {
response.on('end', () => {
try {
const results = JSON.parse(data);
if (results && results.length > 0) {
if (results && results.length > 0 &&
!['administrative', 'country', 'state', 'county', 'city', 'town', 'village', 'municipality'].includes(results[0].type)) {
resolve({
lat: parseFloat(results[0].lat),
lng: parseFloat(results[0].lon)
Expand All @@ -59,8 +72,9 @@ async function processVenues() {
// Remove control characters except newlines, tabs, and carriage returns
const data = JSON.parse(rawData);

const venuesWithoutGeometry = data.filter(venue => !venue.geometry);
const venuesWithGeometry = data.filter(venue => venue.geometry);
const refreshCityOnly = process.argv.includes('--refresh-city-only');
const venuesWithoutGeometry = data.filter(venue => !venue.geometry || (refreshCityOnly && hasCityOnlyAddress(venue)));
const venuesWithGeometry = data.filter(venue => !venuesWithoutGeometry.includes(venue));

console.log(`Total venues: ${data.length}`);
console.log(`Venues with geometry: ${venuesWithGeometry.length}`);
Expand All @@ -73,16 +87,17 @@ async function processVenues() {
for (let i = 0; i < venuesWithoutGeometry.length; i++) {
const venue = venuesWithoutGeometry[i];

if (!venue.address) {
const query = getVenueQuery(venue);
if (!query) {
console.log(`[${i + 1}/${venuesWithoutGeometry.length}] Skipping ${venue.name} - no address`);
failCount++;
continue;
}

try {
console.log(`[${i + 1}/${venuesWithoutGeometry.length}] Geocoding: ${venue.name} (${venue.address})`);
console.log(`[${i + 1}/${venuesWithoutGeometry.length}] Geocoding: ${query}`);

const geometry = await geocodeAddress(venue.address);
const geometry = await geocodeAddress(query);

if (geometry) {
venue.geometry = geometry;
Expand Down
32 changes: 30 additions & 2 deletions 19hz/scripts/venues.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -25,21 +25,24 @@ test('venue scripts extract, merge and geocode without losing existing data', t
text: async () => 'date,title,genre,"New Venue (Oakland)"\\n'
});
require('node:https').get = (url, options, callback) => {
require('node:fs').appendFileSync(${JSON.stringify(path.join(directory, 'queries.txt'))}, new URL(url).searchParams.get('q') + '\\n');
const request = new EventEmitter();
process.nextTick(() => {
const response = new EventEmitter();
response.statusCode = process.env.FAIL_GEOCODE ? 503 : 200;
response.resume = () => {};
callback(response);
if (response.statusCode === 200) {
response.emit('data', '[{"lat":"37.8","lon":"-122.2"}]');
response.emit('data', process.env.NO_RESULT ? '[]' : process.env.CITY_RESULT ?
'[{"lat":"37.8","lon":"-122.2","type":"city"}]' :
'[{"lat":"37.8","lon":"-122.2","type":"building"}]');
response.emit('end');
}
});
return request;
};
`);
const run = (script, env = {}) => spawnSync(process.execPath, ['--require', mock, script], {
const run = (script, env = {}, args = []) => spawnSync(process.execPath, ['--require', mock, script, ...args], {
cwd: scripts,
env: { ...process.env, ...env },
encoding: 'utf8'
Expand All @@ -51,6 +54,7 @@ test('venue scripts extract, merge and geocode without losing existing data', t
]);
assert.equal(run('merge-venues.js').status, 0);
assert.equal(run('geocode-venues.js').status, 0);
assert.equal(fs.readFileSync(path.join(directory, 'queries.txt'), 'utf8'), 'New Venue, Oakland\n');
assert.deepEqual(JSON.parse(fs.readFileSync(venuesFile)), [
existing,
{ name: 'New Venue (Oakland)', address: 'Oakland', geometry: { lat: 37.8, lng: -122.2 } }
Expand All @@ -64,4 +68,28 @@ test('venue scripts extract, merge and geocode without losing existing data', t
const geocodeFailure = run('geocode-venues.js', { FAIL_GEOCODE: '1' });
assert.notEqual(geocodeFailure.status, 0);
assert.match(geocodeFailure.stderr, /HTTP 503/);

const oldGeometry = { lat: 10, lng: 20 };
const cityOnly = { name: '123 Main St (Oakland)', address: 'Oakland', geometry: oldGeometry };
const fullAddress = { name: 'Club', address: '123 Main St, Oakland', geometry: oldGeometry };
fs.writeFileSync(venuesFile, JSON.stringify([cityOnly, fullAddress]));
fs.writeFileSync(path.join(directory, 'queries.txt'), '');
assert.equal(run('geocode-venues.js').status, 0);
assert.equal(fs.readFileSync(path.join(directory, 'queries.txt'), 'utf8'), '');
assert.equal(run('geocode-venues.js', {}, ['--refresh-city-only']).status, 0);
assert.equal(fs.readFileSync(path.join(directory, 'queries.txt'), 'utf8'), '123 Main St, Oakland\n');
assert.deepEqual(JSON.parse(fs.readFileSync(venuesFile)), [
fullAddress,
{ ...cityOnly, geometry: { lat: 37.8, lng: -122.2 } }
]);
fs.writeFileSync(venuesFile, JSON.stringify([cityOnly]));
assert.equal(run('geocode-venues.js', { NO_RESULT: '1' }, ['--refresh-city-only']).status, 0);
assert.deepEqual(JSON.parse(fs.readFileSync(venuesFile)), [cityOnly]);
assert.equal(run('geocode-venues.js', { CITY_RESULT: '1' }, ['--refresh-city-only']).status, 0);
assert.deepEqual(JSON.parse(fs.readFileSync(venuesFile)), [cityOnly]);

fs.writeFileSync(venuesFile, JSON.stringify([{ name: 'Club', address: '123 Main St, Oakland' }]));
fs.writeFileSync(path.join(directory, 'queries.txt'), '');
assert.equal(run('geocode-venues.js').status, 0);
assert.equal(fs.readFileSync(path.join(directory, 'queries.txt'), 'utf8'), 'Club, 123 Main St, Oakland\n');
});
Loading