diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml deleted file mode 100644 index bd2cf19..0000000 --- a/.github/workflows/codeql.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: CodeQL -on: - pull_request: - branches: [main] -permissions: - actions: read - contents: read - security-events: write -jobs: - analyze: - name: Analyze (${{ matrix.language }}) - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - language: [javascript-typescript] - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Initialize CodeQL - uses: github/codeql-action/init@v3 - with: - languages: ${{ matrix.language }} - build-mode: none - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/update-19hz-venues.yml b/.github/workflows/update-19hz-venues.yml index 9cbd62b..963e5d4 100644 --- a/.github/workflows/update-19hz-venues.yml +++ b/.github/workflows/update-19hz-venues.yml @@ -2,6 +2,11 @@ name: Update 19hz venues on: workflow_dispatch: + inputs: + refresh_city_only: + description: Re-geocode existing venues whose address is only their city + type: boolean + default: false permissions: contents: write @@ -23,10 +28,16 @@ jobs: node-version: 24 - name: Extract, merge, and geocode venues working-directory: 19hz/scripts + env: + REFRESH_CITY_ONLY: ${{ inputs.refresh_city_only }} run: | node extract-venues.js node merge-venues.js - node geocode-venues.js + if [ "$REFRESH_CITY_ONLY" = "true" ]; then + node geocode-venues.js --refresh-city-only + else + node geocode-venues.js + fi - name: Create pull request uses: peter-evans/create-pull-request@v7 with: diff --git a/19hz/README.md b/19hz/README.md index f844d32..7d4be74 100644 --- a/19hz/README.md +++ b/19hz/README.md @@ -5,4 +5,6 @@ Variation of my funcheapsfmap based off of data scraped from 19hz with a spotify ## Venues Venues are stored in [venues.json](./venues.json). To update them, run **Update 19hz venues** from the repository's Actions tab. The workflow runs the extraction, merge, and geocoding scripts and opens or updates a pull request when the venue file changes. +Geocoding searches for the venue name and address together. To retry existing extracted venues whose address is only a city, select **Re-geocode existing venues whose address is only their city** when starting the workflow, or run `node geocode-venues.js --refresh-city-only` from `19hz/scripts`. This can take a long time; existing coordinates are retained when a search has no result. + The repository's Actions settings must allow GitHub Actions to create pull requests. The workflow uses `GITHUB_TOKEN` with contents and pull request write permissions. diff --git a/19hz/map.js b/19hz/map.js index 1658739..a6dc830 100644 --- a/19hz/map.js +++ b/19hz/map.js @@ -286,7 +286,6 @@ window.filter = async function (filters = {}) { return event.visible; }).length; - // Group visible events by location for clustering // First, hide all markers window.events.get()?.forEach(event => { if (!event.title || !event.geometry) return; @@ -295,19 +294,17 @@ window.filter = async function (filters = {}) { } }); - // Group visible events by location - const locationGroups = new Map(); + const venueGroups = new Map(); window.events.get()?.forEach(event => { if (!event.title || !event.geometry || !event.visible) return; - // Create location key with rounded coordinates (to handle minor differences) - const locationKey = `${event.geometry.lat.toFixed(6)},${event.geometry.lng.toFixed(6)}`; - - if (!locationGroups.has(locationKey)) { - locationGroups.set(locationKey, []); + const venueKey = event.venue?.trim().toLowerCase() || event; + + if (!venueGroups.has(venueKey)) { + venueGroups.set(venueKey, []); } - locationGroups.get(locationKey).push(event); + venueGroups.get(venueKey).push(event); }); // Load marker library @@ -316,7 +313,7 @@ window.filter = async function (filters = {}) { const PinElement = markerLibrary.PinElement; // Create markers based on grouping - locationGroups.forEach((eventsAtLocation, locationKey) => { + venueGroups.forEach(eventsAtLocation => { if (eventsAtLocation.length === 1) { // Single event - show normal marker const event = eventsAtLocation[0]; diff --git a/19hz/map.test.js b/19hz/map.test.js new file mode 100644 index 0000000..fafd750 --- /dev/null +++ b/19hz/map.test.js @@ -0,0 +1,58 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const vm = require('node:vm'); + +test('filter groups events by venue rather than shared coordinates', async () => { + const markers = []; + class AdvancedMarkerElement { + constructor(options) { + Object.assign(this, options); + markers.push(this); + } + addEventListener() {} + } + class PinElement { + constructor(options) { + Object.assign(this, options); + } + } + const map = {}; + const geometry = { lat: 37.779, lng: -122.419 }; + const events = [ + { title: 'First', venue: 'Venue A', geometry, date: '2026-01-01', categories: ['House'] }, + { title: 'Second', venue: 'Venue A', geometry: { lat: 37.78, lng: -122.42 }, date: '2026-01-02', categories: ['Techno'] }, + { title: 'Third', venue: 'Venue B', geometry, date: '2026-01-01', categories: ['House'] }, + { title: 'Fourth', venue: '', geometry, date: '2026-01-01', categories: ['House'] }, + { title: 'Fifth', venue: '', geometry, date: '2026-01-01', categories: ['House'] } + ]; + events.forEach(event => event.marker = { map, content: { style: {} } }); + const form = { elements: { countEvents: {}, countCategories: {} } }; + const context = { + window: { map, events: { get: () => events }, history: { replaceState() {} } }, + document: { getElementById: () => form }, + IntersectionObserver: class { observe() {} }, + google: { maps: { importLibrary: async () => ({ AdvancedMarkerElement, PinElement }) } } + }; + vm.runInNewContext(fs.readFileSync(path.join(__dirname, 'map.js'), 'utf8'), context); + + await context.window.filter(); + assert.equal(markers.length, 1); + assert.equal(markers[0].title, '2 events at Venue A'); + assert.equal(markers[0].content.glyphText, '2'); + assert.equal(events[0].marker.map, null); + assert.equal(events[1].marker.map, null); + assert.equal(events[2].marker.map, map); + assert.equal(events[3].marker.map, map); + assert.equal(events[4].marker.map, map); + assert.equal(form.elements.countEvents.innerText, 5); + + await context.window.filter({ category: 'House' }); + assert.equal(markers[0].map, null); + assert.equal(markers.length, 1); + assert.equal(events[0].marker.map, map); + assert.equal(events[1].marker.map, null); + assert.equal(events[2].marker.map, map); + assert.equal(form.elements.countEvents.innerText, 4); +}); diff --git a/19hz/scripts/geocode-venues.js b/19hz/scripts/geocode-venues.js index 3df511f..f12e1d8 100644 --- a/19hz/scripts/geocode-venues.js +++ b/19hz/scripts/geocode-venues.js @@ -9,6 +9,18 @@ function delay(ms) { return new Promise(resolve => setTimeout(resolve, ms)); } +function getVenueQuery(venue) { + const match = venue.name?.match(/^(.+?)\s*\(([^()]*)\)\s*$/); + const name = (match ? match[1] : venue.name || '').trim(); + const address = (venue.address || match?.[2] || '').trim(); + return name && address ? `${name}, ${address}` : ''; +} + +function hasCityOnlyAddress(venue) { + const match = venue.name?.match(/^(.+?)\s*\(([^()]*)\)\s*$/); + return match && match[2].trim().toLowerCase() === venue.address?.trim().toLowerCase(); +} + function geocodeAddress(address) { return new Promise((resolve, reject) => { const encodedAddress = encodeURIComponent(address); @@ -35,7 +47,8 @@ function geocodeAddress(address) { response.on('end', () => { try { const results = JSON.parse(data); - if (results && results.length > 0) { + if (results && results.length > 0 && + !['administrative', 'country', 'state', 'county', 'city', 'town', 'village', 'municipality'].includes(results[0].type)) { resolve({ lat: parseFloat(results[0].lat), lng: parseFloat(results[0].lon) @@ -59,8 +72,9 @@ async function processVenues() { // Remove control characters except newlines, tabs, and carriage returns const data = JSON.parse(rawData); - const venuesWithoutGeometry = data.filter(venue => !venue.geometry); - const venuesWithGeometry = data.filter(venue => venue.geometry); + const refreshCityOnly = process.argv.includes('--refresh-city-only'); + const venuesWithoutGeometry = data.filter(venue => !venue.geometry || (refreshCityOnly && hasCityOnlyAddress(venue))); + const venuesWithGeometry = data.filter(venue => !venuesWithoutGeometry.includes(venue)); console.log(`Total venues: ${data.length}`); console.log(`Venues with geometry: ${venuesWithGeometry.length}`); @@ -73,16 +87,17 @@ async function processVenues() { for (let i = 0; i < venuesWithoutGeometry.length; i++) { const venue = venuesWithoutGeometry[i]; - if (!venue.address) { + const query = getVenueQuery(venue); + if (!query) { console.log(`[${i + 1}/${venuesWithoutGeometry.length}] Skipping ${venue.name} - no address`); failCount++; continue; } try { - console.log(`[${i + 1}/${venuesWithoutGeometry.length}] Geocoding: ${venue.name} (${venue.address})`); + console.log(`[${i + 1}/${venuesWithoutGeometry.length}] Geocoding: ${query}`); - const geometry = await geocodeAddress(venue.address); + const geometry = await geocodeAddress(query); if (geometry) { venue.geometry = geometry; diff --git a/19hz/scripts/venues.test.js b/19hz/scripts/venues.test.js index 68ca454..e93de14 100644 --- a/19hz/scripts/venues.test.js +++ b/19hz/scripts/venues.test.js @@ -25,6 +25,7 @@ test('venue scripts extract, merge and geocode without losing existing data', t text: async () => 'date,title,genre,"New Venue (Oakland)"\\n' }); require('node:https').get = (url, options, callback) => { + require('node:fs').appendFileSync(${JSON.stringify(path.join(directory, 'queries.txt'))}, new URL(url).searchParams.get('q') + '\\n'); const request = new EventEmitter(); process.nextTick(() => { const response = new EventEmitter(); @@ -32,14 +33,16 @@ test('venue scripts extract, merge and geocode without losing existing data', t response.resume = () => {}; callback(response); if (response.statusCode === 200) { - response.emit('data', '[{"lat":"37.8","lon":"-122.2"}]'); + response.emit('data', process.env.NO_RESULT ? '[]' : process.env.CITY_RESULT ? + '[{"lat":"37.8","lon":"-122.2","type":"city"}]' : + '[{"lat":"37.8","lon":"-122.2","type":"building"}]'); response.emit('end'); } }); return request; }; `); - const run = (script, env = {}) => spawnSync(process.execPath, ['--require', mock, script], { + const run = (script, env = {}, args = []) => spawnSync(process.execPath, ['--require', mock, script, ...args], { cwd: scripts, env: { ...process.env, ...env }, encoding: 'utf8' @@ -51,6 +54,7 @@ test('venue scripts extract, merge and geocode without losing existing data', t ]); assert.equal(run('merge-venues.js').status, 0); assert.equal(run('geocode-venues.js').status, 0); + assert.equal(fs.readFileSync(path.join(directory, 'queries.txt'), 'utf8'), 'New Venue, Oakland\n'); assert.deepEqual(JSON.parse(fs.readFileSync(venuesFile)), [ existing, { name: 'New Venue (Oakland)', address: 'Oakland', geometry: { lat: 37.8, lng: -122.2 } } @@ -64,4 +68,28 @@ test('venue scripts extract, merge and geocode without losing existing data', t const geocodeFailure = run('geocode-venues.js', { FAIL_GEOCODE: '1' }); assert.notEqual(geocodeFailure.status, 0); assert.match(geocodeFailure.stderr, /HTTP 503/); + + const oldGeometry = { lat: 10, lng: 20 }; + const cityOnly = { name: '123 Main St (Oakland)', address: 'Oakland', geometry: oldGeometry }; + const fullAddress = { name: 'Club', address: '123 Main St, Oakland', geometry: oldGeometry }; + fs.writeFileSync(venuesFile, JSON.stringify([cityOnly, fullAddress])); + fs.writeFileSync(path.join(directory, 'queries.txt'), ''); + assert.equal(run('geocode-venues.js').status, 0); + assert.equal(fs.readFileSync(path.join(directory, 'queries.txt'), 'utf8'), ''); + assert.equal(run('geocode-venues.js', {}, ['--refresh-city-only']).status, 0); + assert.equal(fs.readFileSync(path.join(directory, 'queries.txt'), 'utf8'), '123 Main St, Oakland\n'); + assert.deepEqual(JSON.parse(fs.readFileSync(venuesFile)), [ + fullAddress, + { ...cityOnly, geometry: { lat: 37.8, lng: -122.2 } } + ]); + fs.writeFileSync(venuesFile, JSON.stringify([cityOnly])); + assert.equal(run('geocode-venues.js', { NO_RESULT: '1' }, ['--refresh-city-only']).status, 0); + assert.deepEqual(JSON.parse(fs.readFileSync(venuesFile)), [cityOnly]); + assert.equal(run('geocode-venues.js', { CITY_RESULT: '1' }, ['--refresh-city-only']).status, 0); + assert.deepEqual(JSON.parse(fs.readFileSync(venuesFile)), [cityOnly]); + + fs.writeFileSync(venuesFile, JSON.stringify([{ name: 'Club', address: '123 Main St, Oakland' }])); + fs.writeFileSync(path.join(directory, 'queries.txt'), ''); + assert.equal(run('geocode-venues.js').status, 0); + assert.equal(fs.readFileSync(path.join(directory, 'queries.txt'), 'utf8'), 'Club, 123 Main St, Oakland\n'); });