Reduce non-intersecting map archive downloads

cmd/update-download-regions/geometry.go

@@ -0,0 +1,159 @@
+package main
+
+import (
+	"fmt"
+	"math"
+	"slices"
+	"sort"
+
+	"github.com/paulmach/orb"
+	"github.com/paulmach/orb/clip"
+	"github.com/paulmach/orb/planar"
+)
+
+// Must match settings.GROUP_AREA_BOX_DEGREES, which defines the runtime archive grid.
+const archiveDegrees = 2
+
+type coordinate struct {
+	latitude  int
+	longitude int
+}
+
+type archiveRange [3]int
+
+func archivesForBounds(bounds orb.Bound) []coordinate {
+	var coordinates []coordinate
+	for latitude := archiveStart(bounds.Min[1]); latitude < archiveStop(bounds.Max[1]); latitude += archiveDegrees {
+		for longitude := archiveStart(bounds.Min[0]); longitude < archiveStop(bounds.Max[0]); longitude += archiveDegrees {
+			coordinates = append(coordinates, coordinate{latitude: latitude, longitude: longitude})
+		}
+	}
+	return coordinates
+}
+
+func archivesForGeometry(polygons orb.MultiPolygon) []coordinate {
+	selected := make(map[coordinate]struct{})
+	for _, polygon := range polygons {
+		bounds := polygon.Bound()
+		for latitude := archiveStart(bounds.Min[1]); latitude < archiveStop(bounds.Max[1]); latitude += archiveDegrees {
+			for longitude := archiveStart(bounds.Min[0]); longitude < archiveStop(bounds.Max[0]); longitude += archiveDegrees {
+				archive := orb.Bound{
+					Min: orb.Point{float64(longitude), float64(latitude)},
+					Max: orb.Point{float64(longitude + archiveDegrees), float64(latitude + archiveDegrees)},
+				}
+				if polygonIntersectsBound(polygon, archive, bounds) {
+					selected[coordinate{latitude: latitude, longitude: longitude}] = struct{}{}
+				}
+			}
+		}
+	}
+
+	coordinates := make([]coordinate, 0, len(selected))
+	for coordinate := range selected {
+		coordinates = append(coordinates, coordinate)
+	}
+	sort.Slice(coordinates, func(first, second int) bool {
+		if coordinates[first].latitude != coordinates[second].latitude {
+			return coordinates[first].latitude < coordinates[second].latitude
+		}
+		return coordinates[first].longitude < coordinates[second].longitude
+	})
+	return coordinates
+}
+
+func scopeGeometry(polygons orb.MultiPolygon, bounds orb.Bound, path string) (orb.MultiPolygon, error) {
+	if len(polygons) == 0 {
+		return nil, fmt.Errorf("%s has no source geometry", path)
+	}
+	for _, polygon := range polygons {
+		for _, ring := range polygon {
+			if ringCrossesAntimeridian(ring) {
+				return nil, fmt.Errorf("%s has unsupported source geometry", path)
+			}
+		}
+	}
+
+	seed := orb.Bound{
+		Min: orb.Point{float64(archiveStart(bounds.Min[0])), float64(archiveStart(bounds.Min[1]))},
+		Max: orb.Point{float64(archiveStop(bounds.Max[0])), float64(archiveStop(bounds.Max[1]))},
+	}
+
+	// Use the existing bounds to choose components, then keep each selected
+	// component whole so normal border changes do not get clipped.
+	selected := make(orb.MultiPolygon, 0, len(polygons))
+	for _, polygon := range polygons {
+		if polygonIntersectsBound(polygon, seed, polygon.Bound()) {
+			selected = append(selected, polygon)
+		}
+	}
+	if len(selected) == 0 {
+		return nil, fmt.Errorf("%s does not intersect its bounding_box", path)
+	}
+	return selected, nil
+}
+
+func compactRanges(coordinates []coordinate) []archiveRange {
+	var ranges []archiveRange
+	for index := 0; index < len(coordinates); {
+		latitude := coordinates[index].latitude
+		minimumLongitude := coordinates[index].longitude
+		maximumLongitude := minimumLongitude + archiveDegrees
+		index++
+
+		for index < len(coordinates) && coordinates[index].latitude == latitude && coordinates[index].longitude == maximumLongitude {
+			maximumLongitude += archiveDegrees
+			index++
+		}
+		ranges = append(ranges, archiveRange{latitude, minimumLongitude, maximumLongitude})
+	}
+	return ranges
+}
+
+func archiveStart(value float64) int {
+	return int(math.Floor(value/archiveDegrees)) * archiveDegrees
+}
+
+func archiveStop(value float64) int {
+	return int(math.Ceil(value/archiveDegrees)) * archiveDegrees
+}
+
+func coordinatesEqual(first, second []coordinate) bool {
+	return slices.Equal(first, second)
+}
+
+func polygonIntersectsBound(polygon orb.Polygon, target, bounds orb.Bound) bool {
+	if !bounds.Intersects(target) {
+		return false
+	}
+	for _, ring := range polygon {
+		if len(ring) == 1 && target.Contains(ring[0]) {
+			return true
+		}
+		if len(ring) < 2 {
+			continue
+		}
+		line := orb.LineString(ring).Clone()
+		if line[0] != line[len(line)-1] {
+			line = append(line, line[0])
+		}
+		if len(clip.LineString(target, line)) > 0 {
+			return true
+		}
+	}
+
+	for _, corner := range target.ToRing()[:4] {
+		if planar.PolygonContains(polygon, corner) {
+			return true
+		}
+	}
+	return false
+}
+
+func ringCrossesAntimeridian(ring orb.Ring) bool {
+	for index, start := range ring {
+		if math.Abs(start[0]-ring[(index+1)%len(ring)][0]) > 180 {
+			return true
+		}
+	}
+	return false
+}

cmd/update-download-regions/main.go

@@ -0,0 +1,179 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"sort"
+
+	"github.com/paulmach/orb"
+)
+
+const downloadMenuPath = "settings/download_menu.json"
+
+type menuBounds struct {
+	MinLat float64 `json:"min_lat"`
+	MinLon float64 `json:"min_lon"`
+	MaxLat float64 `json:"max_lat"`
+	MaxLon float64 `json:"max_lon"`
+}
+
+type menuLocation struct {
+	BoundingBox menuBounds `json:"bounding_box"`
+}
+
+type (
+	downloadMenu   map[string]map[string]menuLocation
+	downloadRanges map[string]map[string][]archiveRange
+)
+
+type summary struct {
+	locations int
+	ranged    int
+	legacy    int
+	selected  int
+}
+
+func (summary summary) String() string {
+	return fmt.Sprintf(
+		"%d regions, %d with explicit ranges; %d legacy archive occurrences -> %d selected",
+		summary.locations, summary.ranged, summary.legacy, summary.selected,
+	)
+}
+
+func main() {
+	write := flag.Bool("write", false, "write updated archive ranges to the download menu instead of checking them")
+	flag.Parse()
+	if err := updateDownloadMenu(*write); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func updateDownloadMenu(write bool) error {
+	rawMenu, err := os.ReadFile(downloadMenuPath)
+	if err != nil {
+		return err
+	}
+	menuDocument, err := parseJSONObject(rawMenu)
+	if err != nil {
+		return err
+	}
+	var menu downloadMenu
+	if err := json.Unmarshal(rawMenu, &menu); err != nil {
+		return err
+	}
+
+	cacheDirectory, err := os.UserCacheDir()
+	if err != nil {
+		return err
+	}
+	ranges, summary, err := generateDownloadRanges(menu, filepath.Join(cacheDirectory, "mapd", "download-region-sources"))
+	if err != nil {
+		return err
+	}
+
+	newline := "\n"
+	if bytes.Contains(rawMenu, []byte("\r\n")) {
+		newline = "\r\n"
+	}
+	if err := inlineArchiveRanges(&menuDocument, ranges, newline); err != nil {
+		return err
+	}
+	expected := append(renderJSONObject(menuDocument, 0, newline), newline...)
+	changed := !bytes.Equal(rawMenu, expected)
+
+	if changed && !write {
+		return fmt.Errorf("download menu is stale (%s); run with --write", summary)
+	}
+	if changed {
+		if err := os.WriteFile(downloadMenuPath, expected, 0o644); err != nil {
+			return err
+		}
+		fmt.Printf("updated %s (%s)\n", downloadMenuPath, summary)
+	} else {
+		fmt.Printf("download menu is up to date (%s)\n", summary)
+	}
+	return nil
+}
+
+func generateDownloadRanges(menu downloadMenu, cacheDirectory string) (downloadRanges, summary, error) {
+	countryCodes := sortedKeys(menu["nation"])
+	stateCodes := sortedKeys(menu["us_state"])
+	if len(countryCodes)+len(stateCodes) == 0 {
+		return nil, summary{}, fmt.Errorf("download menu contains no nation or us_state regions")
+	}
+
+	countryPath, err := fetchSource(countrySource, cacheDirectory)
+	if err != nil {
+		return nil, summary{}, err
+	}
+	countryGeometries, err := loadCountryGeometries(countryPath, countryCodes)
+	if err != nil {
+		return nil, summary{}, err
+	}
+
+	statePath, err := fetchSource(stateSource, cacheDirectory)
+	if err != nil {
+		return nil, summary{}, err
+	}
+	stateGeometries, err := loadStateGeometries(statePath, stateCodes)
+	if err != nil {
+		return nil, summary{}, err
+	}
+
+	geometries := map[string]regionGeometries{
+		"nation":   countryGeometries,
+		"us_state": stateGeometries,
+	}
+	ranges := make(downloadRanges)
+	var result summary
+	for _, section := range []string{"nation", "us_state"} {
+		for _, code := range sortedKeys(menu[section]) {
+			path := section + "." + code
+			bounds, err := menu[section][code].BoundingBox.bound()
+			if err != nil {
+				return nil, summary{}, fmt.Errorf("%s: %w", path, err)
+			}
+			legacy := archivesForBounds(bounds)
+			scoped, err := scopeGeometry(geometries[section][code], bounds, path)
+			if err != nil {
+				return nil, summary{}, err
+			}
+			selected := archivesForGeometry(scoped)
+
+			result.locations++
+			result.legacy += len(legacy)
+			result.selected += len(selected)
+			if coordinatesEqual(selected, legacy) {
+				continue
+			}
+			if ranges[section] == nil {
+				ranges[section] = make(map[string][]archiveRange)
+			}
+			ranges[section][code] = compactRanges(selected)
+			result.ranged++
+		}
+	}
+	return ranges, result, nil
+}
+
+func (bounds menuBounds) bound() (orb.Bound, error) {
+	result := orb.Bound{Min: orb.Point{bounds.MinLon, bounds.MinLat}, Max: orb.Point{bounds.MaxLon, bounds.MaxLat}}
+	if result.Min[0] < -180 || result.Max[0] > 180 || result.Min[0] >= result.Max[0] || result.Min[1] < -90 || result.Max[1] > 90 || result.Min[1] >= result.Max[1] {
+		return orb.Bound{}, fmt.Errorf("invalid bounding_box")
+	}
+	return result, nil
+}
+
+func sortedKeys[T any](values map[string]T) []string {
+	keys := make([]string, 0, len(values))
+	for key := range values {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	return keys
+}