From eb23a2ab25557cb4aa62384878cf589a2b4512bf Mon Sep 17 00:00:00 2001 From: FrogAi <91348155+FrogAi@users.noreply.github.com> Date: Fri, 21 Aug 2026 19:38:09 -0700 Subject: [PATCH 1/2] Optimize offline map downloads --- docs/overriding-internal-defaults.md | 28 +- scripts/update_download_regions.py | 312 +++++++++++++++ settings/download.go | 83 ++-- settings/download_menu.json | 552 ++++++++++++++++++--------- 4 files changed, 757 insertions(+), 218 deletions(-) create mode 100644 scripts/update_download_regions.py diff --git a/docs/overriding-internal-defaults.md b/docs/overriding-internal-defaults.md index 49f86a6..a33b6f6 100644 --- a/docs/overriding-internal-defaults.md +++ b/docs/overriding-internal-defaults.md @@ -38,8 +38,9 @@ values are always loaded upon starting mapd. ## Download Menu The download menu file is used for two purposes. When triggering a download, the -area names given to mapd are used to locate the appropriate bounding box from -the download menu file. The download menu file is also used to create a dynamic +area names given to mapd locate an entry in the download menu. Its non-empty +`archive_ranges` select the archives when provided; otherwise its `bounding_box` +is used. The download menu file is also used to create a dynamic menu in the mapd cli for selecting areas to download. This means that additional areas not provided by mapd can be added as options for downloads by copying the download\_menu.json to /data/openpilot/mapd\_download\_menu.json and then adding @@ -52,7 +53,7 @@ any desired areas to the file. The structure is as follows: "definitions": { "area_menu": { "type": "object", - "additionalProperties": {"$ref": "#/definitions/area"}, + "additionalProperties": {"$ref": "#/definitions/area"} }, "area": { "type": "object", @@ -83,6 +84,20 @@ any desired areas to the file. The structure is as follows: "max_lat" ] }, + "archive_ranges": { + "description": "Rows of [minimum latitude, inclusive minimum longitude, exclusive maximum longitude] for 2-degree latitude bands.", + "type": "array", + "minItems": 1, + "items": { + "type": "array", + "items": { + "type": "integer", + "multipleOf": 2 + }, + "minItems": 3, + "maxItems": 3 + } + }, "submenu": { "type": "string" } @@ -96,6 +111,13 @@ any desired areas to the file. The structure is as follows: } ``` +`archive_ranges` is optional, but takes precedence over `bounding_box` when +non-empty. Remove it to restore bounding-box selection. The default ranges can +be checked or regenerated with `uv run scripts/update_download_regions.py` and +`uv run scripts/update_download_regions.py --write`, respectively. +Pass `--menu PATH` to process another menu; the updater regenerates whichever of +the `nation` and `us_state` sections are present and leaves other sections unchanged. + Note the optional submenu value in an area. The submenu value allows for chaining of the menus when requesting a download, so the submenu value should exactly match a top level key in the main object. This value is not used by the diff --git a/scripts/update_download_regions.py b/scripts/update_download_regions.py new file mode 100644 index 0000000..c97411e --- /dev/null +++ b/scripts/update_download_regions.py @@ -0,0 +1,312 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.12,<3.14" +# dependencies = [ +# "numpy==2.5.2", +# "pyshp==3.1.6", +# "shapely==2.1.2", +# ] +# /// +import argparse +import copy +import hashlib +import io +import json +import math +import shapefile +import sys +import tempfile +import urllib.request +import zipfile + +from itertools import pairwise, product +from pathlib import Path +from shapely import make_valid +from shapely.geometry import GeometryCollection, MultiPolygon, Polygon, box, shape +from shapely.ops import unary_union + +DESCRIPTION = """Update the default menu's archive ranges from pinned boundary sources. + +Run with uv run scripts/update_download_regions.py to check the menu or add +--write to update it. To adopt a newer boundary release, update its source +specification in this script, then review the generated diff. + +The existing menu remains authoritative for which regions and disconnected +territories it contains. New countries and newly relevant detached territories +require an intentional menu change; this script never makes that policy choice. +""" + +ARCHIVE_DEGREES = 2 + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_MENU_PATH = REPOSITORY_ROOT / "settings" / "download_menu.json" +DEFAULT_CACHE_DIRECTORY = Path(tempfile.gettempdir()) / "mapd-download-region-sources" + +# Natural Earth data is in the public domain. +COUNTRY_SOURCE = { + "name": "Natural Earth 10m Admin 0 countries", + "url": "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/f1890d9f152c896d250a77557a5751a93d494776/geojson/ne_10m_admin_0_countries.geojson", + "filename": "ne_10m_admin_0_countries.geojson", + "sha256": "239eec57ac17f100a11e2536cffc56752c318b50ae765b0918ff7aab4ce8f255", +} + +# United States Census Bureau data is in the public domain. +STATE_SOURCE = { + "name": "Census TIGER/Line states", + "url": "https://www2.census.gov/geo/tiger/TIGER2025/STATE/tl_2025_us_state.zip", + "filename": "tl_2025_us_state.zip", + "sha256": "59a220888a8d9be8117c4fcd38f542bd02d81abf0d198c78113595ad540dd957", +} + +# Natural Earth uses nonstandard ISO_A2 values for these menu entries. +COUNTRY_SELECTORS = { + "FR": ("ADM0_A3", "FRA"), + "NO": ("ADM0_A3", "NOR"), + "TW": ("ADM0_A3", "TWN"), +} + +# mapd historically used GM for Guam; Census uses the standard GU code. +STATE_CODE_ALIASES = {"GM": "GU"} + + +class UpdateError(RuntimeError): + pass + + +def sha256_bytes(data): + return hashlib.sha256(data).hexdigest() + + +def fetch_source(specification, cache_directory): + cache_directory.mkdir(parents=True, exist_ok=True) + destination = cache_directory / specification["filename"] + if destination.is_file() and sha256_bytes(destination.read_bytes()) == specification["sha256"]: + return destination + + request = urllib.request.Request(specification["url"], headers={"User-Agent": "mapd-download-region-updater/1"}) + with urllib.request.urlopen(request, timeout=120) as response: + content = response.read() + if sha256_bytes(content) != specification["sha256"]: + raise UpdateError(f"{specification['name']} no longer matches its pinned hash") + destination.write_bytes(content) + return destination + + +def polygon_components(geometry): + if isinstance(geometry, Polygon): + return [geometry] + if isinstance(geometry, MultiPolygon): + return list(geometry.geoms) + if isinstance(geometry, GeometryCollection): + return [polygon for part in geometry.geoms for polygon in polygon_components(part)] + return [] + + +def has_antimeridian_jump(geometry): + return any( + abs(first[0] - second[0]) > 180 + for polygon in polygon_components(geometry) + for ring in (polygon.exterior, *polygon.interiors) + for first, second in pairwise(ring.coords) + ) + + +def scoped_geometry(geometry, bounds, path): + if not geometry.is_valid: + geometry = make_valid(geometry) + + components = polygon_components(geometry) + if has_antimeridian_jump(geometry): + raise UpdateError(f"{path} has unsupported source geometry") + + latitude_archives = archive_axis(bounds["min_lat"], bounds["max_lat"]) + longitude_archives = archive_axis(bounds["min_lon"], bounds["max_lon"]) + seed = box(longitude_archives.start, latitude_archives.start, longitude_archives.stop, latitude_archives.stop) + selected_components = [component for component in components if component.intersects(seed)] + if not selected_components: + raise UpdateError(f"{path} does not intersect its bounding_box") + + # Use the existing archive-aligned scope to choose components, then keep each + # selected component whole so small bounding-box errors do not clip its archives. + return unary_union(selected_components) + + +def archive_axis(minimum, maximum): + return range( + math.floor(minimum / ARCHIVE_DEGREES) * ARCHIVE_DEGREES, + math.ceil(maximum / ARCHIVE_DEGREES) * ARCHIVE_DEGREES, + ARCHIVE_DEGREES, + ) + + +def archives_for_bounds(bounds): + return list( + product( + archive_axis(bounds["min_lat"], bounds["max_lat"]), + archive_axis(bounds["min_lon"], bounds["max_lon"]), + ) + ) + + +def archives_for_geometry(geometry): + min_longitude, min_latitude, max_longitude, max_latitude = geometry.bounds + coordinates = product( + archive_axis(min_latitude, max_latitude), + archive_axis(min_longitude, max_longitude), + ) + return [ + (latitude, longitude) + for latitude, longitude in coordinates + if geometry.intersects(box(longitude, latitude, longitude + ARCHIVE_DEGREES, latitude + ARCHIVE_DEGREES)) + ] + + +def compact_ranges(coordinates): + sorted_coordinates = sorted(coordinates) + ranges = [] + index = 0 + while index < len(sorted_coordinates): + latitude, min_longitude = sorted_coordinates[index] + max_longitude = min_longitude + ARCHIVE_DEGREES + index += 1 + while index < len(sorted_coordinates): + next_latitude, next_longitude = sorted_coordinates[index] + if next_latitude != latitude or next_longitude != max_longitude: + break + max_longitude += ARCHIVE_DEGREES + index += 1 + ranges.append([latitude, min_longitude, max_longitude]) + return ranges + + +def load_country_geometries(source_path, codes): + features = json.loads(source_path.read_text(encoding="utf-8"))["features"] + + geometries = {} + for code in codes: + field, value = COUNTRY_SELECTORS.get(code, ("ISO_A2", code)) + matches = [feature for feature in features if feature.get("properties", {}).get(field) == value] + if len(matches) != 1: + raise UpdateError(f"nation.{code} matched {len(matches)} source features") + geometries[code] = shape(matches[0]["geometry"]) + return geometries + + +def load_state_geometries(source_path, codes): + with zipfile.ZipFile(source_path) as source_zip: + source_names = source_zip.namelist() + shape_name = next(name for name in source_names if name.lower().endswith(".shp")) + database_name = next(name for name in source_names if name.lower().endswith(".dbf")) + reader = shapefile.Reader( + shp=io.BytesIO(source_zip.read(shape_name)), + dbf=io.BytesIO(source_zip.read(database_name)), + ) + source_geometries = {} + for record in reader.iterShapeRecords(): + source_geometries[record.record.as_dict()["STUSPS"]] = shape(record.shape.__geo_interface__) + + geometries = {} + for menu_code in codes: + source_code = STATE_CODE_ALIASES.get(menu_code, menu_code) + if source_code not in source_geometries: + raise UpdateError(f"us_state.{menu_code} has no source feature") + geometries[menu_code] = source_geometries[source_code] + return geometries + + +def update_menu(menu, geometries): + summary = {"legacy": 0, "locations": 0, "ranged": 0, "selected": 0} + + for section_name, section_geometries in geometries.items(): + entries = menu.get(section_name, {}) + for code, entry in entries.items(): + path = f"{section_name}.{code}" + bounds = entry["bounding_box"] + legacy_coordinates = archives_for_bounds(bounds) + geometry = scoped_geometry(section_geometries[code], bounds, path) + selected_coordinates = archives_for_geometry(geometry) + ranges = compact_ranges(selected_coordinates) + uses_archive_ranges = selected_coordinates != legacy_coordinates + if uses_archive_ranges: + entry["archive_ranges"] = ranges + else: + entry.pop("archive_ranges", None) + + summary["legacy"] += len(legacy_coordinates) + summary["locations"] += 1 + summary["ranged"] += int(uses_archive_ranges) + summary["selected"] += len(selected_coordinates) + return menu, summary + + +def render_menu(menu, newline): + rendered_menu = copy.deepcopy(menu) + replacements = {} + replacement_index = 0 + for entries in rendered_menu.values(): + for entry in entries.values(): + if "archive_ranges" not in entry: + continue + token = f"__MAPD_ARCHIVE_RANGES_{replacement_index}__" + replacements[token] = entry["archive_ranges"] + entry["archive_ranges"] = token + replacement_index += 1 + + rendered = json.dumps(rendered_menu, ensure_ascii=False, indent=2) + "\n" + for token, ranges in replacements.items(): + rendered = rendered.replace(json.dumps(token), json.dumps(ranges)) + return rendered.replace("\n", newline) + + +def parse_arguments(): + parser = argparse.ArgumentParser(description=DESCRIPTION, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--menu", type=Path, default=DEFAULT_MENU_PATH) + parser.add_argument("--cache-dir", type=Path, default=DEFAULT_CACHE_DIRECTORY) + + mode = parser.add_mutually_exclusive_group() + mode.add_argument("--check", action="store_true", help="check without changing the menu (default)") + mode.add_argument("--write", action="store_true", help="update the menu") + + return parser.parse_args() + + +def run(arguments): + raw_menu = arguments.menu.read_bytes() + newline = "\r\n" if b"\r\n" in raw_menu else "\n" + menu = json.loads(raw_menu.decode("utf-8")) + + country_source = fetch_source(COUNTRY_SOURCE, arguments.cache_dir) + state_source = fetch_source(STATE_SOURCE, arguments.cache_dir) + + country_geometries = load_country_geometries(country_source, menu.get("nation", {})) + state_geometries = load_state_geometries(state_source, menu.get("us_state", {})) + + updated_menu, summary = update_menu(menu, {"nation": country_geometries, "us_state": state_geometries}) + expected = render_menu(updated_menu, newline).encode("utf-8") + + message = ("{locations} regions, {ranged} with explicit ranges; {legacy} legacy archive occurrences -> {selected} selected").format(**summary) + if not arguments.write: + if raw_menu != expected: + print(f"download menu is stale ({message}); run with --write", file=sys.stderr) + return 1 + print(f"download menu is up to date ({message})") + return 0 + + if raw_menu == expected: + print(f"download menu already up to date ({message})") + return 0 + arguments.menu.write_bytes(expected) + print(f"updated {arguments.menu} ({message})") + return 0 + + +def main(): + try: + return run(parse_arguments()) + except (UpdateError, OSError, ValueError, zipfile.BadZipFile) as error: + print(f"error: {error}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/settings/download.go b/settings/download.go index 466b553..0249862 100644 --- a/settings/download.go +++ b/settings/download.go @@ -18,9 +18,17 @@ import ( ) type LocationData struct { - BoundingBox Bounds `json:"bounding_box"` - FullName string `json:"full_name"` - Submenu string `json:"submenu"` + ArchiveRanges []ArchiveRange `json:"archive_ranges,omitempty"` + BoundingBox Bounds `json:"bounding_box"` + FullName string `json:"full_name"` + Submenu string `json:"submenu"` +} + +// ArchiveRange is [minimum latitude, inclusive minimum longitude, exclusive maximum longitude] for a 2-degree latitude band. +type ArchiveRange [3]int + +func (r ArchiveRange) coordinates() (latitude, minLongitude, maxLongitude int) { + return r[0], r[1], r[2] } type DownloadMenu map[string]map[string]LocationData @@ -107,19 +115,20 @@ type download struct { cancelChan chan bool } -func (p *DownloadProgress) addLocationDetails(path string) { +func (p *DownloadProgress) addLocationDetails(path string, location LocationData) { p.LocationDetails[path] = &DownloadLocationDetail{ - TotalFiles: countFilesForBounds(getBoundsForPath(path)), + TotalFiles: countFilesForLocation(location), } } func Download(paths string, progressChan chan DownloadProgress, cancelChan chan bool) { slog.Info("download", "paths", paths) pathsSplit := strings.Split(paths, ",") + menu := GetDownloadMenu() d := download{ progress: DownloadProgress{ LocationsToDownload: pathsSplit, - TotalFiles: countTotalFiles(pathsSplit), + TotalFiles: countTotalFiles(menu, pathsSplit), LocationDetails: make(map[string]*DownloadLocationDetail), Active: true, }, @@ -128,12 +137,12 @@ func Download(paths string, progressChan chan DownloadProgress, cancelChan chan } for _, p := range pathsSplit { - d.progress.addLocationDetails(p) - location := getDataForPath(p) - slog.Info("downloading nation", "nation", location.FullName) - err, canceled := d.downloadBounds(location.BoundingBox, p) + location := getDataForPath(menu, p) + d.progress.addLocationDetails(p, location) + slog.Info("downloading location", "location", location.FullName) + err, canceled := d.downloadLocation(location, p) if err != nil { - slog.Warn("failed to download nation", "error", err, "nation", location.FullName) + slog.Warn("failed to download location", "error", err, "location", location.FullName) } if canceled { d.progress.Canceled = true @@ -162,14 +171,25 @@ func adjustedBounds(bounds Bounds) (int, int, int, int) { return minLat, minLon, maxLat, maxLon } -func (d *download) downloadBounds(bounds Bounds, locationName string) (err error, cancel bool) { - slog.Info("Downloading Bounds", "min_lat", bounds.MinLat, "min_lon", bounds.MinLon, "max_lat", bounds.MaxLat, "max_lon", bounds.MaxLon) +func archiveRangesForLocation(location LocationData) []ArchiveRange { + if len(location.ArchiveRanges) > 0 { + return location.ArchiveRanges + } + + minLat, minLon, maxLat, maxLon := adjustedBounds(location.BoundingBox) + var archiveRanges []ArchiveRange + for lat := minLat; lat < maxLat; lat += GROUP_AREA_BOX_DEGREES { + archiveRanges = append(archiveRanges, ArchiveRange{lat, minLon, maxLon}) + } + return archiveRanges +} + +func (d *download) downloadLocation(location LocationData, locationName string) (err error, cancel bool) { + slog.Info("Downloading Location", "location", locationName) - // clip given bounds to file areas - minLat, minLon, maxLat, maxLon := adjustedBounds(bounds) - d.progress.LocationDetails[locationName].TotalFiles = countFilesForBounds(bounds) - for i := minLat; i < maxLat; i += GROUP_AREA_BOX_DEGREES { - for j := minLon; j < maxLon; j += GROUP_AREA_BOX_DEGREES { + for _, archiveRange := range archiveRangesForLocation(location) { + latitude, minLongitude, maxLongitude := archiveRange.coordinates() + for longitude := minLongitude; longitude < maxLongitude; longitude += GROUP_AREA_BOX_DEGREES { select { // nonblocking update of progress case d.progressChan <- d.progress: default: @@ -182,7 +202,7 @@ func (d *download) downloadBounds(bounds Bounds, locationName string) (err error default: } - filename := fmt.Sprintf("offline/%d/%d.tar.gz", i, j) + filename := fmt.Sprintf("offline/%d/%d.tar.gz", latitude, longitude) url := fmt.Sprintf("https://map-data.pfeifer.dev/%s", filename) outputName := filepath.Join(params.GetBaseOpPath(), "tmp", filename) err := os.MkdirAll(filepath.Dir(outputName), 0o775) @@ -269,22 +289,27 @@ func (d *download) downloadBounds(bounds Bounds, locationName string) (err error slog.Warn("could not remove temporary download directory", "error", err) } - slog.Info("Finished Downloading Bounds", "min_lat", bounds.MinLat, "min_lon", bounds.MinLon, "max_lat", bounds.MaxLat, "max_lon", bounds.MaxLon) + slog.Info("Finished Downloading Location", "location", locationName) return nil, false } -func countFilesForBounds(bounds Bounds) int { - minLat, minLon, maxLat, maxLon := adjustedBounds(bounds) - return ((maxLat - minLat) / GROUP_AREA_BOX_DEGREES) * ((maxLon - minLon) / GROUP_AREA_BOX_DEGREES) +func countFilesForLocation(location LocationData) int { + totalFiles := 0 + for _, archiveRange := range archiveRangesForLocation(location) { + _, minLongitude, maxLongitude := archiveRange.coordinates() + for longitude := minLongitude; longitude < maxLongitude; longitude += GROUP_AREA_BOX_DEGREES { + totalFiles++ + } + } + return totalFiles } -func getDataForPath(path string) LocationData { +func getDataForPath(menu DownloadMenu, path string) LocationData { parts := strings.Split(path, ".") if len(parts) < 2 { slog.Warn("ignoring invalid download path", "path", path) return LocationData{} } - menu := GetDownloadMenu() box := menu[parts[0]][parts[1]] if len(parts) > 2 { for i := range len(parts) - 2 { @@ -294,15 +319,11 @@ func getDataForPath(path string) LocationData { return box } -func getBoundsForPath(path string) Bounds { - return getDataForPath(path).BoundingBox -} - -func countTotalFiles(paths []string) int { +func countTotalFiles(menu DownloadMenu, paths []string) int { totalFiles := 0 for _, p := range paths { - totalFiles += countFilesForBounds(getBoundsForPath(p)) + totalFiles += countFilesForLocation(getDataForPath(menu, p)) } return totalFiles diff --git a/settings/download_menu.json b/settings/download_menu.json index 7d713ef..e44fee6 100644 --- a/settings/download_menu.json +++ b/settings/download_menu.json @@ -7,7 +7,8 @@ "min_lat": 29.32, "max_lon": 75.16, "max_lat": 38.49 - } + }, + "archive_ranges": [[28, 60, 68], [30, 60, 70], [32, 60, 72], [34, 60, 72], [36, 62, 76], [38, 70, 72]] }, "AO": { "full_name": "Angola", @@ -16,7 +17,8 @@ "min_lat": -17.93, "max_lon": 24.08, "max_lat": -4.44 - } + }, + "archive_ranges": [[-20, 20, 22], [-18, 10, 24], [-16, 10, 22], [-14, 12, 26], [-12, 12, 26], [-10, 12, 24], [-8, 12, 22], [-6, 12, 18]] }, "AL": { "full_name": "Albania", @@ -34,7 +36,8 @@ "min_lat": 22.5, "max_lon": 56.4, "max_lat": 26.06 - } + }, + "archive_ranges": [[22, 50, 56], [24, 50, 58], [26, 56, 58]] }, "AR": { "full_name": "Argentina", @@ -43,7 +46,8 @@ "min_lat": -55.25, "max_lon": -53.63, "max_lat": -21.83 - } + }, + "archive_ranges": [[-56, -70, -62], [-54, -72, -66], [-52, -74, -66], [-50, -74, -64], [-48, -74, -64], [-46, -74, -64], [-44, -74, -62], [-42, -72, -62], [-40, -72, -56], [-38, -72, -56], [-36, -72, -56], [-34, -72, -58], [-32, -72, -56], [-30, -72, -54], [-28, -70, -52], [-26, -70, -52], [-24, -68, -60], [-22, -68, -62]] }, "AM": { "full_name": "Armenia", @@ -52,7 +56,8 @@ "min_lat": 38.74, "max_lon": 46.51, "max_lat": 41.25 - } + }, + "archive_ranges": [[38, 44, 48], [40, 42, 46]] }, "AQ": { "full_name": "Antarctica", @@ -61,7 +66,8 @@ "min_lat": -90, "max_lon": 180, "max_lat": -63.27 - } + }, + "archive_ranges": [[-90, -180, 180], [-88, -180, 180], [-86, -180, 180], [-84, -176, 178], [-82, -164, -58], [-82, -56, 164], [-80, -166, -64], [-80, -62, -58], [-80, -52, -42], [-80, -38, 168], [-78, -160, -66], [-78, -50, -44], [-78, -36, 170], [-76, -148, -60], [-76, -28, 166], [-74, -128, -112], [-74, -106, -58], [-74, -22, -20], [-74, -18, 172], [-72, -104, -96], [-72, -78, -60], [-72, -14, 172], [-70, -92, -90], [-70, -76, -60], [-70, 14, 18], [-70, 32, 162], [-68, -70, -60], [-68, 42, 70], [-68, 80, 148], [-68, 162, 166], [-66, -68, -56], [-66, 52, 56], [-66, 92, 94], [-66, 100, 106], [-66, 110, 114], [-64, -64, -54], [-62, -60, -56]] }, "TF": { "full_name": "French Southern Territories", @@ -79,7 +85,8 @@ "min_lat": -43.63, "max_lon": 153.57, "max_lat": -10.67 - } + }, + "archive_ranges": [[-44, 144, 150], [-42, 142, 150], [-40, 140, 150], [-38, 136, 152], [-36, 114, 120], [-36, 122, 124], [-36, 134, 152], [-34, 114, 130], [-34, 132, 154], [-32, 114, 154], [-30, 112, 154], [-28, 112, 154], [-26, 112, 154], [-24, 112, 152], [-22, 112, 152], [-20, 118, 150], [-18, 122, 148], [-16, 124, 138], [-16, 140, 146], [-14, 124, 138], [-14, 140, 144], [-12, 130, 138], [-12, 140, 144]] }, "AT": { "full_name": "Austria", @@ -88,7 +95,8 @@ "min_lat": 46.43, "max_lon": 16.98, "max_lat": 49.04 - } + }, + "archive_ranges": [[46, 8, 18], [48, 12, 18]] }, "AZ": { "full_name": "Azerbaijan", @@ -97,7 +105,8 @@ "min_lat": 38.27, "max_lon": 50.39, "max_lat": 41.86 - } + }, + "archive_ranges": [[38, 44, 50], [40, 44, 52]] }, "BI": { "full_name": "Burundi", @@ -115,7 +124,8 @@ "min_lat": 49.53, "max_lon": 6.16, "max_lat": 51.48 - } + }, + "archive_ranges": [[48, 4, 6], [50, 2, 8]] }, "BJ": { "full_name": "Benin", @@ -124,7 +134,8 @@ "min_lat": 6.14, "max_lon": 3.8, "max_lat": 12.24 - } + }, + "archive_ranges": [[6, 0, 4], [8, 0, 4], [10, 0, 4], [12, 2, 4]] }, "BF": { "full_name": "Burkina Faso", @@ -133,7 +144,8 @@ "min_lat": 9.61, "max_lon": 2.18, "max_lat": 15.12 - } + }, + "archive_ranges": [[8, -6, -2], [10, -6, 4], [12, -6, 4], [14, -4, 2]] }, "BD": { "full_name": "Bangladesh", @@ -142,7 +154,8 @@ "min_lat": 20.67, "max_lon": 92.67, "max_lat": 26.45 - } + }, + "archive_ranges": [[20, 88, 94], [22, 88, 94], [24, 88, 94], [26, 88, 90]] }, "BG": { "full_name": "Bulgaria", @@ -151,7 +164,8 @@ "min_lat": 41.23, "max_lon": 28.56, "max_lat": 44.23 - } + }, + "archive_ranges": [[40, 22, 30], [42, 22, 30], [44, 22, 24], [44, 26, 28]] }, "BS": { "full_name": "Bahamas", @@ -160,7 +174,8 @@ "min_lat": 23.71, "max_lon": -77, "max_lat": 27.04 - } + }, + "archive_ranges": [[22, -80, -74], [24, -80, -76], [26, -80, -76]] }, "BA": { "full_name": "Bosnia and Herzegovina", @@ -169,7 +184,8 @@ "min_lat": 42.65, "max_lon": 19.6, "max_lat": 45.23 - } + }, + "archive_ranges": [[42, 16, 20], [44, 14, 20]] }, "BY": { "full_name": "Belarus", @@ -178,7 +194,8 @@ "min_lat": 51.32, "max_lon": 32.69, "max_lat": 56.17 - } + }, + "archive_ranges": [[50, 22, 32], [52, 22, 34], [54, 24, 32], [56, 26, 30]] }, "BZ": { "full_name": "Belize", @@ -187,7 +204,8 @@ "min_lat": 15.89, "max_lon": -88.11, "max_lat": 18.5 - } + }, + "archive_ranges": [[14, -90, -88], [16, -90, -86], [18, -90, -86]] }, "BO": { "full_name": "Bolivia", @@ -196,7 +214,8 @@ "min_lat": -22.87, "max_lon": -57.5, "max_lat": -9.76 - } + }, + "archive_ranges": [[-24, -70, -62], [-22, -70, -56], [-20, -70, -56], [-18, -70, -56], [-16, -70, -60], [-14, -70, -60], [-12, -70, -64], [-10, -68, -64]] }, "BR": { "full_name": "Brazil", @@ -205,7 +224,8 @@ "min_lat": -33.77, "max_lon": -34.73, "max_lat": 5.24 - } + }, + "archive_ranges": [[-34, -54, -50], [-32, -58, -50], [-30, -58, -48], [-28, -56, -48], [-26, -56, -46], [-24, -58, -40], [-22, -60, -40], [-20, -60, -38], [-18, -62, -38], [-16, -62, -38], [-14, -66, -36], [-12, -74, -36], [-10, -74, -34], [-8, -76, -34], [-6, -74, -34], [-4, -70, -38], [-2, -72, -44], [0, -72, -48], [2, -70, -50], [4, -66, -58], [4, -52, -50]] }, "BN": { "full_name": "Brunei", @@ -214,7 +234,8 @@ "min_lat": 4.01, "max_lon": 115.45, "max_lat": 5.45 - } + }, + "archive_ranges": [[4, 112, 116]] }, "BT": { "full_name": "Bhutan", @@ -223,7 +244,8 @@ "min_lat": 26.72, "max_lon": 92.1, "max_lat": 28.3 - } + }, + "archive_ranges": [[26, 88, 94], [28, 88, 92]] }, "BW": { "full_name": "Botswana", @@ -232,7 +254,8 @@ "min_lat": -26.83, "max_lon": 29.43, "max_lat": -17.66 - } + }, + "archive_ranges": [[-28, 20, 24], [-26, 18, 28], [-24, 18, 30], [-22, 20, 30], [-20, 20, 28], [-18, 22, 26]] }, "CF": { "full_name": "Central African Republic", @@ -241,7 +264,8 @@ "min_lat": 2.27, "max_lon": 27.37, "max_lat": 11.14 - } + }, + "archive_ranges": [[2, 14, 20], [4, 14, 28], [6, 14, 28], [8, 16, 26], [10, 20, 24]] }, "CA": { "full_name": "Canada", @@ -250,7 +274,8 @@ "min_lat": 41.68, "max_lon": -52.65, "max_lat": 73.23 - } + }, + "archive_ranges": [[40, -84, -80], [42, -84, -76], [42, -68, -64], [42, -62, -58], [44, -84, -70], [44, -68, -58], [46, -90, -52], [48, -128, -52], [50, -132, -54], [52, -134, -54], [54, -134, -56], [56, -134, -86], [56, -82, -60], [58, -140, -92], [58, -82, -62], [60, -142, -92], [60, -82, -64], [62, -142, -64], [64, -142, -62], [66, -142, -80], [66, -78, -60], [68, -142, -64], [70, -132, -66], [72, -126, -74], [74, -126, -116], [74, -100, -90]] }, "CH": { "full_name": "Switzerland", @@ -259,7 +284,8 @@ "min_lat": 45.78, "max_lon": 10.44, "max_lat": 47.83 - } + }, + "archive_ranges": [[44, 6, 10], [46, 4, 12]] }, "CL": { "full_name": "Chile", @@ -268,7 +294,8 @@ "min_lat": -55.61, "max_lon": -66.96, "max_lat": -17.58 - } + }, + "archive_ranges": [[-56, -74, -66], [-54, -76, -68], [-52, -76, -70], [-50, -76, -72], [-48, -76, -70], [-46, -76, -70], [-44, -76, -70], [-42, -76, -70], [-40, -74, -70], [-38, -74, -70], [-36, -74, -68], [-34, -72, -68], [-32, -72, -68], [-30, -72, -68], [-28, -72, -68], [-26, -72, -66], [-24, -72, -66], [-22, -72, -68], [-20, -72, -68], [-18, -70, -68]] }, "CN": { "full_name": "China", @@ -277,7 +304,8 @@ "min_lat": 18.2, "max_lon": 135.03, "max_lat": 53.46 - } + }, + "archive_ranges": [[18, 108, 112], [20, 98, 102], [20, 106, 114], [22, 96, 118], [24, 96, 120], [26, 84, 94], [26, 98, 122], [28, 82, 124], [30, 78, 124], [32, 78, 122], [34, 74, 122], [36, 74, 124], [38, 72, 126], [40, 72, 130], [42, 78, 132], [44, 78, 96], [44, 110, 134], [46, 82, 92], [46, 114, 136], [48, 84, 90], [48, 114, 136], [50, 118, 128], [52, 120, 128]] }, "CI": { "full_name": "Ivory Coast", @@ -286,7 +314,8 @@ "min_lat": 4.34, "max_lon": -2.56, "max_lat": 10.52 - } + }, + "archive_ranges": [[4, -8, -2], [6, -10, -2], [8, -10, -2], [10, -10, -4]] }, "CM": { "full_name": "Cameroon", @@ -295,7 +324,8 @@ "min_lat": 1.73, "max_lon": 16.01, "max_lat": 12.86 - } + }, + "archive_ranges": [[0, 14, 18], [2, 8, 18], [4, 8, 16], [6, 8, 16], [8, 12, 16], [10, 12, 16], [12, 14, 16]] }, "CD": { "full_name": "Congo (Kinshasa)", @@ -304,7 +334,8 @@ "min_lat": -13.26, "max_lon": 31.17, "max_lat": 5.26 - } + }, + "archive_ranges": [[-14, 26, 30], [-12, 22, 30], [-10, 16, 32], [-8, 12, 14], [-8, 16, 32], [-6, 12, 30], [-4, 14, 30], [-2, 16, 30], [0, 16, 32], [2, 18, 32], [4, 18, 32]] }, "CG": { "full_name": "Congo (Brazzaville)", @@ -313,7 +344,8 @@ "min_lat": -5.04, "max_lon": 18.45, "max_lat": 3.73 - } + }, + "archive_ranges": [[-6, 10, 16], [-4, 10, 18], [-2, 12, 18], [0, 12, 20], [2, 12, 20]] }, "CO": { "full_name": "Colombia", @@ -322,7 +354,8 @@ "min_lat": -4.3, "max_lon": -66.88, "max_lat": 12.44 - } + }, + "archive_ranges": [[-6, -72, -68], [-4, -74, -68], [-2, -76, -68], [0, -80, -66], [2, -80, -66], [4, -78, -66], [6, -78, -66], [8, -78, -72], [10, -76, -70], [12, -74, -70]] }, "CR": { "full_name": "Costa Rica", @@ -340,7 +373,8 @@ "min_lat": 19.86, "max_lon": -74.18, "max_lat": 23.19 - } + }, + "archive_ranges": [[18, -78, -74], [20, -86, -74], [22, -86, -76]] }, "CY": { "full_name": "Cyprus", @@ -349,7 +383,8 @@ "min_lat": 34.57, "max_lon": 34, "max_lat": 35.17 - } + }, + "archive_ranges": [[34, 32, 36]] }, "CZ": { "full_name": "Czech Republic", @@ -367,7 +402,8 @@ "min_lat": 47.3, "max_lon": 15.02, "max_lat": 54.98 - } + }, + "archive_ranges": [[46, 6, 14], [48, 6, 14], [50, 4, 16], [52, 6, 16], [54, 6, 16]] }, "DJ": { "full_name": "Djibouti", @@ -376,7 +412,8 @@ "min_lat": 10.93, "max_lon": 43.32, "max_lat": 12.7 - } + }, + "archive_ranges": [[10, 40, 44], [12, 42, 44]] }, "DK": { "full_name": "Denmark", @@ -394,7 +431,8 @@ "min_lat": 17.6, "max_lon": -68.32, "max_lat": 19.88 - } + }, + "archive_ranges": [[16, -72, -70], [18, -74, -68]] }, "DZ": { "full_name": "Algeria", @@ -403,7 +441,8 @@ "min_lat": 19.06, "max_lon": 12, "max_lat": 37.12 - } + }, + "archive_ranges": [[18, 2, 8], [20, -2, 10], [22, -4, 12], [24, -8, 12], [26, -10, 10], [28, -10, 10], [30, -6, 10], [32, -4, 10], [34, -4, 10], [36, 0, 10]] }, "EC": { "full_name": "Ecuador", @@ -412,7 +451,8 @@ "min_lat": -4.96, "max_lon": -75.23, "max_lat": 1.38 - } + }, + "archive_ranges": [[-6, -82, -78], [-4, -82, -74], [-2, -82, -74], [0, -82, -74]] }, "EG": { "full_name": "Egypt", @@ -421,7 +461,8 @@ "min_lat": 22, "max_lon": 36.87, "max_lat": 31.59 - } + }, + "archive_ranges": [[20, 24, 38], [22, 24, 38], [24, 24, 36], [26, 24, 36], [28, 24, 36], [30, 24, 36]] }, "ER": { "full_name": "Eritrea", @@ -430,7 +471,8 @@ "min_lat": 12.46, "max_lon": 43.08, "max_lat": 18 - } + }, + "archive_ranges": [[12, 40, 44], [14, 36, 42], [16, 36, 42], [18, 38, 40]] }, "ES": { "full_name": "Spain", @@ -439,7 +481,8 @@ "min_lat": 35.95, "max_lon": 3.04, "max_lat": 43.75 - } + }, + "archive_ranges": [[34, -6, -2], [36, -8, 0], [38, -8, 6], [40, -10, 6], [42, -10, 4]] }, "EE": { "full_name": "Estonia", @@ -448,7 +491,8 @@ "min_lat": 57.47, "max_lon": 28.13, "max_lat": 59.61 - } + }, + "archive_ranges": [[56, 20, 28], [58, 20, 30]] }, "ET": { "full_name": "Ethiopia", @@ -457,7 +501,8 @@ "min_lat": 3.42, "max_lon": 47.79, "max_lat": 14.96 - } + }, + "archive_ranges": [[2, 36, 42], [4, 34, 46], [6, 32, 48], [8, 32, 48], [10, 34, 44], [12, 34, 44], [14, 36, 42]] }, "FI": { "full_name": "Finland", @@ -466,7 +511,8 @@ "min_lat": 59.85, "max_lon": 31.52, "max_lat": 70.16 - } + }, + "archive_ranges": [[58, 22, 26], [60, 20, 32], [62, 20, 32], [64, 22, 32], [66, 22, 32], [68, 20, 30], [70, 26, 28]] }, "FJ": { "full_name": "Fiji", @@ -475,7 +521,8 @@ "min_lat": -18.29, "max_lon": 180, "max_lat": -16.02 - } + }, + "archive_ranges": [[-20, -180, -178], [-20, 176, 180], [-18, -180, -178], [-18, 176, 180]] }, "FK": { "full_name": "Falkland Islands", @@ -484,7 +531,8 @@ "min_lat": -52.3, "max_lon": -57.75, "max_lat": -51.1 - } + }, + "archive_ranges": [[-54, -62, -58], [-52, -62, -56]] }, "FR": { "full_name": "France", @@ -493,7 +541,8 @@ "min_lat": 42.5, "max_lon": 9.56, "max_lat": 51.15 - } + }, + "archive_ranges": [[40, 8, 10], [42, -2, 10], [44, -2, 8], [46, -6, 8], [48, -6, 10], [50, 0, 6]] }, "GA": { "full_name": "Gabon", @@ -502,7 +551,8 @@ "min_lat": -3.98, "max_lon": 14.43, "max_lat": 2.33 - } + }, + "archive_ranges": [[-4, 8, 16], [-2, 8, 16], [0, 8, 16], [2, 10, 14]] }, "GB": { "full_name": "United Kingdom", @@ -511,7 +561,8 @@ "min_lat": 49.96, "max_lon": 1.68, "max_lat": 58.64 - } + }, + "archive_ranges": [[48, -8, -4], [50, -6, 2], [52, -6, 2], [54, -10, 0], [56, -8, 0], [58, -8, 0], [60, -2, 0]] }, "GE": { "full_name": "Georgia", @@ -520,7 +571,8 @@ "min_lat": 41.06, "max_lon": 46.64, "max_lat": 43.55 - } + }, + "archive_ranges": [[40, 40, 48], [42, 38, 48]] }, "GH": { "full_name": "Ghana", @@ -538,7 +590,8 @@ "min_lat": 7.31, "max_lon": -7.83, "max_lat": 12.59 - } + }, + "archive_ranges": [[6, -10, -8], [8, -16, -6], [10, -16, -6], [12, -14, -8]] }, "GM": { "full_name": "Gambia", @@ -574,7 +627,8 @@ "min_lat": 34.92, "max_lon": 26.6, "max_lat": 41.83 - } + }, + "archive_ranges": [[34, 22, 28], [36, 20, 30], [38, 18, 28], [40, 20, 28]] }, "GL": { "full_name": "Greenland", @@ -583,7 +637,8 @@ "min_lat": 60.04, "max_lon": -12.21, "max_lat": 83.65 - } + }, + "archive_ranges": [[58, -46, -42], [60, -50, -42], [62, -52, -40], [64, -54, -34], [66, -54, -32], [68, -56, -22], [70, -56, -20], [72, -58, -20], [74, -68, -66], [74, -62, -16], [76, -74, -16], [78, -74, -16], [80, -68, -10], [82, -60, -18]] }, "GT": { "full_name": "Guatemala", @@ -592,7 +647,8 @@ "min_lat": 13.74, "max_lon": -88.23, "max_lat": 17.82 - } + }, + "archive_ranges": [[12, -92, -88], [14, -94, -88], [16, -92, -88]] }, "GY": { "full_name": "Guyana", @@ -601,7 +657,8 @@ "min_lat": 1.27, "max_lon": -56.54, "max_lat": 8.37 - } + }, + "archive_ranges": [[0, -60, -56], [2, -62, -56], [4, -62, -56], [6, -62, -56], [8, -62, -58]] }, "HN": { "full_name": "Honduras", @@ -610,7 +667,8 @@ "min_lat": 12.98, "max_lon": -83.15, "max_lat": 16.01 - } + }, + "archive_ranges": [[12, -90, -84], [14, -90, -82], [16, -88, -82]] }, "HR": { "full_name": "Croatia", @@ -619,7 +677,8 @@ "min_lat": 42.48, "max_lon": 19.39, "max_lat": 46.5 - } + }, + "archive_ranges": [[42, 14, 20], [44, 12, 20], [46, 14, 18]] }, "HT": { "full_name": "Haiti", @@ -628,7 +687,8 @@ "min_lat": 18.03, "max_lon": -71.62, "max_lat": 19.92 - } + }, + "archive_ranges": [[18, -76, -70], [20, -74, -72]] }, "HU": { "full_name": "Hungary", @@ -637,7 +697,8 @@ "min_lat": 45.76, "max_lon": 22.71, "max_lat": 48.62 - } + }, + "archive_ranges": [[44, 16, 20], [46, 16, 24], [48, 16, 24]] }, "ID": { "full_name": "Indonesia", @@ -646,7 +707,8 @@ "min_lat": -10.36, "max_lon": 141.03, "max_lat": 5.48 - } + }, + "archive_ranges": [[-12, 118, 126], [-10, 110, 132], [-10, 136, 142], [-8, 104, 116], [-8, 120, 122], [-8, 124, 132], [-8, 134, 142], [-6, 102, 108], [-6, 110, 142], [-4, 98, 142], [-2, 98, 140], [0, 96, 132], [0, 134, 136], [2, 94, 102], [2, 104, 110], [2, 114, 120], [2, 124, 130], [4, 94, 100], [4, 106, 110], [4, 114, 118], [4, 126, 128]] }, "IN": { "full_name": "India", @@ -655,7 +717,8 @@ "min_lat": 7.97, "max_lon": 97.4, "max_lat": 35.49 - } + }, + "archive_ranges": [[6, 92, 94], [8, 72, 74], [8, 76, 80], [8, 92, 94], [10, 72, 80], [10, 92, 94], [12, 74, 82], [12, 92, 96], [14, 72, 82], [16, 72, 84], [18, 72, 88], [20, 68, 90], [20, 92, 94], [22, 68, 96], [24, 68, 96], [26, 68, 98], [28, 70, 82], [28, 88, 90], [28, 92, 98], [30, 72, 82], [32, 72, 80], [34, 72, 80]] }, "IE": { "full_name": "Ireland", @@ -664,7 +727,8 @@ "min_lat": 51.67, "max_lon": -6.03, "max_lat": 55.13 - } + }, + "archive_ranges": [[50, -12, -6], [52, -12, -4], [54, -12, -6]] }, "IR": { "full_name": "Iran", @@ -673,7 +737,8 @@ "min_lat": 25.08, "max_lon": 63.32, "max_lat": 39.71 - } + }, + "archive_ranges": [[24, 54, 62], [26, 50, 64], [28, 48, 64], [30, 46, 62], [32, 44, 62], [34, 44, 62], [36, 44, 62], [38, 44, 50], [38, 54, 58]] }, "IQ": { "full_name": "Iraq", @@ -682,7 +747,8 @@ "min_lat": 29.1, "max_lon": 48.57, "max_lat": 37.39 - } + }, + "archive_ranges": [[28, 42, 50], [30, 38, 50], [32, 38, 48], [34, 40, 48], [36, 40, 46]] }, "IS": { "full_name": "Iceland", @@ -691,7 +757,8 @@ "min_lat": 63.5, "max_lon": -13.61, "max_lat": 66.53 - } + }, + "archive_ranges": [[62, -24, -16], [64, -26, -12], [66, -24, -14]] }, "IL": { "full_name": "Israel", @@ -709,7 +776,8 @@ "min_lat": 36.62, "max_lon": 18.48, "max_lat": 47.12 - } + }, + "archive_ranges": [[36, 10, 18], [38, 8, 10], [38, 12, 20], [40, 8, 20], [42, 6, 16], [44, 6, 14], [46, 6, 14]] }, "JM": { "full_name": "Jamaica", @@ -718,7 +786,8 @@ "min_lat": 17.7, "max_lon": -76.2, "max_lat": 18.52 - } + }, + "archive_ranges": [[16, -78, -76], [18, -80, -76]] }, "JO": { "full_name": "Jordan", @@ -727,7 +796,8 @@ "min_lat": 29.2, "max_lon": 39.2, "max_lat": 33.38 - } + }, + "archive_ranges": [[28, 34, 38], [30, 34, 40], [32, 34, 40]] }, "JP": { "full_name": "Japan", @@ -736,7 +806,8 @@ "min_lat": 31.03, "max_lon": 145.54, "max_lat": 45.55 - } + }, + "archive_ranges": [[28, 128, 130], [30, 128, 132], [30, 140, 142], [32, 128, 140], [34, 128, 142], [36, 132, 142], [38, 138, 144], [40, 138, 144], [42, 138, 146], [44, 140, 146]] }, "KZ": { "full_name": "Kazakhstan", @@ -745,7 +816,8 @@ "min_lat": 40.66, "max_lon": 87.36, "max_lat": 55.39 - } + }, + "archive_ranges": [[40, 52, 56], [40, 66, 72], [42, 50, 56], [42, 60, 82], [44, 48, 84], [46, 46, 86], [48, 46, 88], [50, 46, 86], [52, 60, 80], [54, 60, 78]] }, "KE": { "full_name": "Kenya", @@ -754,7 +826,8 @@ "min_lat": -4.68, "max_lon": 41.86, "max_lat": 5.51 - } + }, + "archive_ranges": [[-6, 38, 40], [-4, 34, 42], [-2, 32, 42], [0, 32, 42], [2, 34, 42], [4, 32, 38], [4, 40, 42]] }, "KG": { "full_name": "Kyrgyzstan", @@ -763,7 +836,8 @@ "min_lat": 39.28, "max_lon": 80.26, "max_lat": 43.3 - } + }, + "archive_ranges": [[38, 68, 74], [40, 68, 80], [42, 70, 82]] }, "KH": { "full_name": "Cambodia", @@ -781,7 +855,8 @@ "min_lat": 34.39, "max_lon": 129.47, "max_lat": 38.61 - } + }, + "archive_ranges": [[34, 124, 130], [36, 126, 130], [38, 126, 130]] }, "KW": { "full_name": "Kuwait", @@ -799,7 +874,8 @@ "min_lat": 13.88, "max_lon": 107.56, "max_lat": 22.46 - } + }, + "archive_ranges": [[12, 104, 108], [14, 104, 108], [16, 100, 108], [18, 100, 106], [20, 100, 106], [22, 100, 104]] }, "LB": { "full_name": "Lebanon", @@ -817,7 +893,8 @@ "min_lat": 4.36, "max_lon": -7.54, "max_lat": 8.54 - } + }, + "archive_ranges": [[4, -12, -6], [6, -12, -6], [8, -12, -8]] }, "LY": { "full_name": "Libya", @@ -826,7 +903,8 @@ "min_lat": 19.58, "max_lon": 25.16, "max_lat": 33.14 - } + }, + "archive_ranges": [[18, 22, 26], [20, 18, 26], [22, 10, 26], [24, 8, 26], [26, 8, 26], [28, 8, 26], [30, 8, 26], [32, 10, 16], [32, 18, 26]] }, "LK": { "full_name": "Sri Lanka", @@ -835,7 +913,8 @@ "min_lat": 5.97, "max_lon": 81.79, "max_lat": 9.82 - } + }, + "archive_ranges": [[4, 80, 82], [6, 78, 82], [8, 78, 82]] }, "LS": { "full_name": "Lesotho", @@ -853,7 +932,8 @@ "min_lat": 53.91, "max_lon": 26.59, "max_lat": 56.37 - } + }, + "archive_ranges": [[52, 22, 26], [54, 20, 28], [56, 20, 26]] }, "LU": { "full_name": "Luxembourg", @@ -871,7 +951,8 @@ "min_lat": 55.62, "max_lon": 28.18, "max_lat": 57.97 - } + }, + "archive_ranges": [[54, 24, 28], [56, 20, 30], [58, 24, 26]] }, "MA": { "full_name": "Morocco", @@ -880,7 +961,8 @@ "min_lat": 21.42, "max_lon": -1.12, "max_lat": 35.76 - } + }, + "archive_ranges": [[20, -18, -14], [22, -18, -12], [24, -16, -12], [26, -16, -8], [28, -14, -4], [30, -10, -2], [32, -10, 0], [34, -8, 0]] }, "MD": { "full_name": "Moldova", @@ -889,7 +971,8 @@ "min_lat": 45.49, "max_lon": 30.02, "max_lat": 48.47 - } + }, + "archive_ranges": [[44, 28, 30], [46, 26, 32], [48, 26, 30]] }, "MG": { "full_name": "Madagascar", @@ -898,7 +981,8 @@ "min_lat": -25.6, "max_lon": 50.48, "max_lat": -12.04 - } + }, + "archive_ranges": [[-26, 42, 48], [-24, 42, 50], [-22, 42, 50], [-20, 44, 50], [-18, 42, 52], [-16, 44, 52], [-14, 46, 52], [-12, 48, 50]] }, "MX": { "full_name": "Mexico", @@ -907,7 +991,8 @@ "min_lat": 14.54, "max_lon": -86.81, "max_lat": 32.72 - } + }, + "archive_ranges": [[14, -98, -90], [16, -104, -88], [18, -116, -114], [18, -112, -110], [18, -106, -86], [20, -108, -96], [20, -92, -86], [22, -112, -96], [22, -90, -88], [24, -116, -96], [26, -116, -96], [28, -116, -98], [30, -118, -104], [32, -118, -112]] }, "MK": { "full_name": "Macedonia", @@ -925,7 +1010,8 @@ "min_lat": 10.1, "max_lon": 4.27, "max_lat": 24.97 - } + }, + "archive_ranges": [[10, -12, -4], [12, -14, -2], [14, -14, 4], [16, -6, 6], [18, -8, 6], [20, -8, 4], [22, -8, 0], [24, -8, -2]] }, "MM": { "full_name": "Myanmar", @@ -934,7 +1020,8 @@ "min_lat": 9.93, "max_lon": 101.18, "max_lat": 28.34 - } + }, + "archive_ranges": [[8, 98, 100], [10, 96, 100], [12, 96, 100], [14, 92, 100], [16, 94, 100], [18, 92, 100], [20, 92, 102], [22, 92, 100], [24, 92, 100], [26, 94, 100], [28, 96, 100]] }, "ME": { "full_name": "Montenegro", @@ -943,7 +1030,8 @@ "min_lat": 41.88, "max_lon": 20.34, "max_lat": 43.52 - } + }, + "archive_ranges": [[40, 18, 20], [42, 18, 22]] }, "MN": { "full_name": "Mongolia", @@ -952,7 +1040,8 @@ "min_lat": 41.6, "max_lon": 119.77, "max_lat": 52.05 - } + }, + "archive_ranges": [[40, 102, 106], [42, 94, 112], [44, 90, 118], [46, 88, 120], [48, 86, 120], [50, 88, 108], [50, 112, 118], [52, 98, 100]] }, "MZ": { "full_name": "Mozambique", @@ -961,7 +1050,8 @@ "min_lat": -26.74, "max_lon": 40.78, "max_lat": -10.32 - } + }, + "archive_ranges": [[-28, 32, 34], [-26, 30, 36], [-24, 30, 36], [-22, 30, 36], [-20, 32, 38], [-18, 30, 42], [-16, 30, 42], [-14, 34, 42], [-12, 34, 42]] }, "MR": { "full_name": "Mauritania", @@ -970,7 +1060,8 @@ "min_lat": 14.62, "max_lon": -4.92, "max_lat": 27.4 - } + }, + "archive_ranges": [[14, -18, -16], [14, -14, -4], [16, -18, -4], [18, -18, -4], [20, -18, -6], [22, -14, -6], [24, -14, -4], [26, -10, -6]] }, "MW": { "full_name": "Malawi", @@ -979,7 +1070,8 @@ "min_lat": -16.8, "max_lon": 35.77, "max_lat": -9.23 - } + }, + "archive_ranges": [[-18, 34, 36], [-16, 32, 36], [-14, 32, 36], [-12, 32, 36], [-10, 32, 36]] }, "MY": { "full_name": "Malaysia", @@ -988,7 +1080,8 @@ "min_lat": 0.77, "max_lon": 119.18, "max_lat": 6.93 - } + }, + "archive_ranges": [[0, 102, 106], [0, 108, 116], [2, 100, 106], [2, 108, 116], [4, 100, 104], [4, 112, 120], [6, 100, 104], [6, 116, 120]] }, "NA": { "full_name": "Namibia", @@ -997,7 +1090,8 @@ "min_lat": -29.05, "max_lon": 25.08, "max_lat": -16.94 - } + }, + "archive_ranges": [[-30, 14, 20], [-28, 14, 20], [-26, 14, 20], [-24, 14, 22], [-22, 12, 22], [-20, 10, 26], [-18, 10, 26]] }, "NC": { "full_name": "New Caledonia", @@ -1006,7 +1100,8 @@ "min_lat": -22.4, "max_lon": 167.12, "max_lat": -20.11 - } + }, + "archive_ranges": [[-24, 166, 168], [-22, 162, 170]] }, "NE": { "full_name": "Niger", @@ -1015,7 +1110,8 @@ "min_lat": 11.66, "max_lon": 15.9, "max_lat": 23.47 - } + }, + "archive_ranges": [[10, 2, 4], [12, 0, 14], [14, 0, 16], [16, 2, 16], [18, 4, 16], [20, 6, 16], [22, 8, 16]] }, "NG": { "full_name": "Nigeria", @@ -1024,7 +1120,8 @@ "min_lat": 4.24, "max_lon": 14.58, "max_lat": 13.87 - } + }, + "archive_ranges": [[4, 4, 10], [6, 2, 14], [8, 2, 14], [10, 2, 16], [12, 2, 16]] }, "NI": { "full_name": "Nicaragua", @@ -1042,7 +1139,8 @@ "min_lat": 50.8, "max_lon": 7.09, "max_lat": 53.51 - } + }, + "archive_ranges": [[50, 2, 8], [52, 4, 8]] }, "NO": { "full_name": "Norway", @@ -1051,7 +1149,8 @@ "min_lat": 58.08, "max_lon": 31.29, "max_lat": 70.92 - } + }, + "archive_ranges": [[56, 6, 8], [58, 4, 14], [60, 4, 14], [62, 4, 14], [64, 8, 16], [66, 12, 18], [68, 12, 32], [70, 18, 32]] }, "NP": { "full_name": "Nepal", @@ -1060,7 +1159,8 @@ "min_lat": 26.4, "max_lon": 88.17, "max_lat": 30.42 - } + }, + "archive_ranges": [[26, 80, 90], [28, 80, 88], [30, 80, 84]] }, "NZ": { "full_name": "New Zealand", @@ -1069,7 +1169,8 @@ "min_lat": -46.64, "max_lon": 178.52, "max_lat": -34.45 - } + }, + "archive_ranges": [[-50, 166, 168], [-48, 166, 172], [-46, 166, 172], [-44, 168, 176], [-42, 170, 178], [-40, 172, 180], [-38, 172, 180], [-36, 172, 176]] }, "OM": { "full_name": "Oman", @@ -1078,7 +1179,8 @@ "min_lat": 16.65, "max_lon": 59.81, "max_lat": 26.4 - } + }, + "archive_ranges": [[16, 52, 58], [18, 50, 58], [20, 54, 60], [22, 54, 60], [24, 54, 58], [26, 56, 58]] }, "PK": { "full_name": "Pakistan", @@ -1087,7 +1189,8 @@ "min_lat": 23.69, "max_lon": 77.84, "max_lat": 37.13 - } + }, + "archive_ranges": [[22, 66, 70], [24, 60, 72], [26, 60, 72], [28, 60, 74], [30, 66, 76], [32, 68, 76], [34, 68, 78], [36, 70, 78]] }, "PA": { "full_name": "Panama", @@ -1096,7 +1199,8 @@ "min_lat": 7.22, "max_lon": -77.24, "max_lat": 9.61 - } + }, + "archive_ranges": [[6, -82, -76], [8, -84, -76]] }, "PE": { "full_name": "Peru", @@ -1105,7 +1209,8 @@ "min_lat": -18.35, "max_lon": -68.67, "max_lat": -0.06 - } + }, + "archive_ranges": [[-20, -72, -68], [-18, -76, -68], [-16, -78, -68], [-14, -78, -68], [-12, -80, -68], [-10, -80, -70], [-8, -82, -72], [-6, -82, -68], [-4, -82, -70], [-2, -76, -72]] }, "PH": { "full_name": "Philippines", @@ -1114,7 +1219,8 @@ "min_lat": 5.58, "max_lon": 126.54, "max_lat": 18.51 - } + }, + "archive_ranges": [[4, 118, 122], [4, 124, 126], [6, 116, 128], [8, 116, 128], [10, 118, 128], [12, 118, 126], [14, 118, 126], [16, 118, 124], [18, 120, 124]] }, "PG": { "full_name": "Papua New Guinea", @@ -1123,7 +1229,8 @@ "min_lat": -10.65, "max_lon": 156.02, "max_lat": -2.5 - } + }, + "archive_ranges": [[-12, 146, 156], [-10, 140, 154], [-8, 140, 152], [-8, 154, 156], [-6, 140, 156], [-4, 140, 156], [-2, 146, 148]] }, "PL": { "full_name": "Poland", @@ -1132,7 +1239,8 @@ "min_lat": 49.03, "max_lon": 24.03, "max_lat": 54.85 - } + }, + "archive_ranges": [[48, 16, 24], [50, 14, 26], [52, 14, 24], [54, 14, 24]] }, "PR": { "full_name": "Puerto Rico", @@ -1150,7 +1258,8 @@ "min_lat": 37.67, "max_lon": 130.78, "max_lat": 42.99 - } + }, + "archive_ranges": [[36, 124, 128], [38, 124, 130], [40, 124, 132], [42, 128, 132]] }, "PT": { "full_name": "Portugal", @@ -1159,7 +1268,8 @@ "min_lat": 36.84, "max_lon": -6.39, "max_lat": 42.28 - } + }, + "archive_ranges": [[36, -10, -6], [38, -10, -6], [40, -10, -6], [42, -10, -8]] }, "PY": { "full_name": "Paraguay", @@ -1168,7 +1278,8 @@ "min_lat": -27.55, "max_lon": -54.29, "max_lat": -19.34 - } + }, + "archive_ranges": [[-28, -60, -54], [-26, -62, -54], [-24, -64, -54], [-22, -64, -56], [-20, -62, -58]] }, "QA": { "full_name": "Qatar", @@ -1186,7 +1297,8 @@ "min_lat": 43.69, "max_lon": 29.63, "max_lat": 48.22 - } + }, + "archive_ranges": [[42, 22, 30], [44, 20, 30], [46, 20, 30], [48, 22, 24], [48, 26, 28]] }, "RU": { "full_name": "Russia", @@ -1195,7 +1307,8 @@ "min_lat": 41.15, "max_lon": 180, "max_lat": 81.25 - } + }, + "archive_ranges": [[40, 46, 50], [42, 38, 50], [42, 130, 136], [42, 144, 148], [44, 32, 50], [44, 130, 138], [44, 140, 152], [46, 32, 50], [46, 130, 144], [46, 148, 154], [48, 36, 50], [48, 84, 90], [48, 94, 98], [48, 106, 120], [48, 126, 146], [48, 152, 156], [50, 34, 62], [50, 78, 122], [50, 126, 146], [50, 154, 160], [52, 30, 64], [52, 72, 144], [52, 154, 162], [54, 18, 24], [54, 28, 144], [54, 154, 170], [56, 26, 142], [56, 154, 164], [58, 26, 168], [58, 170, 172], [60, 26, 176], [62, 28, 180], [64, -180, -168], [64, 28, 180], [66, -180, -168], [66, 28, 180], [68, -180, -176], [68, 28, 180], [70, -180, -174], [70, 50, 62], [70, 66, 164], [70, 168, 172], [70, 178, 180], [72, 50, 60], [72, 68, 130], [72, 134, 150], [74, 54, 68], [74, 78, 118], [74, 134, 152], [76, 60, 70], [76, 88, 114], [76, 136, 144], [76, 148, 150], [76, 152, 154], [76, 156, 158], [78, 50, 52], [78, 58, 60], [78, 76, 78], [78, 90, 108], [80, 36, 38], [80, 44, 66], [80, 78, 82], [80, 88, 100]] }, "RW": { "full_name": "Rwanda", @@ -1213,7 +1326,8 @@ "min_lat": 16.35, "max_lon": 55.67, "max_lat": 32.16 - } + }, + "archive_ranges": [[16, 40, 50], [18, 40, 56], [20, 38, 56], [22, 38, 56], [24, 36, 52], [26, 34, 52], [28, 34, 50], [30, 36, 44], [32, 38, 40]] }, "SD": { "full_name": "Sudan", @@ -1222,7 +1336,8 @@ "min_lat": 8.62, "max_lon": 38.41, "max_lat": 22 - } + }, + "archive_ranges": [[8, 22, 36], [10, 22, 36], [12, 20, 38], [14, 22, 38], [16, 22, 40], [18, 22, 40], [20, 24, 38], [22, 30, 32]] }, "SS": { "full_name": "South Sudan", @@ -1231,7 +1346,8 @@ "min_lat": 3.51, "max_lon": 35.3, "max_lat": 12.25 - } + }, + "archive_ranges": [[2, 30, 34], [4, 26, 36], [6, 24, 36], [8, 24, 36], [10, 24, 34], [12, 32, 34]] }, "SN": { "full_name": "Senegal", @@ -1240,7 +1356,8 @@ "min_lat": 12.33, "max_lon": -11.47, "max_lat": 16.6 - } + }, + "archive_ranges": [[12, -18, -10], [14, -18, -10], [16, -18, -12]] }, "SB": { "full_name": "Solomon Islands", @@ -1249,7 +1366,8 @@ "min_lat": -10.83, "max_lon": 162.4, "max_lat": -6.6 - } + }, + "archive_ranges": [[-12, 158, 164], [-10, 156, 164], [-8, 156, 162]] }, "SL": { "full_name": "Sierra Leone", @@ -1258,7 +1376,8 @@ "min_lat": 6.79, "max_lon": -10.23, "max_lat": 10.05 - } + }, + "archive_ranges": [[6, -14, -10], [8, -14, -10]] }, "SV": { "full_name": "El Salvador", @@ -1267,7 +1386,8 @@ "min_lat": 13.15, "max_lon": -87.72, "max_lat": 14.42 - } + }, + "archive_ranges": [[12, -92, -86], [14, -90, -88]] }, "SO": { "full_name": "Somalia", @@ -1276,7 +1396,8 @@ "min_lat": -1.68, "max_lon": 51.13, "max_lat": 12.02 - } + }, + "archive_ranges": [[-2, 40, 44], [0, 40, 46], [2, 40, 48], [4, 40, 50], [6, 44, 50], [8, 46, 52], [10, 48, 52]] }, "RS": { "full_name": "Serbia", @@ -1285,7 +1406,8 @@ "min_lat": 42.25, "max_lon": 22.99, "max_lat": 46.17 - } + }, + "archive_ranges": [[42, 18, 24], [44, 18, 24], [46, 18, 22]] }, "SR": { "full_name": "Suriname", @@ -1294,7 +1416,8 @@ "min_lat": 1.82, "max_lon": -53.96, "max_lat": 6.03 - } + }, + "archive_ranges": [[0, -58, -54], [2, -60, -52], [4, -60, -52], [6, -58, -56]] }, "SK": { "full_name": "Slovakia", @@ -1303,7 +1426,8 @@ "min_lat": 47.76, "max_lon": 22.56, "max_lat": 49.57 - } + }, + "archive_ranges": [[46, 16, 20], [48, 16, 24]] }, "SI": { "full_name": "Slovenia", @@ -1312,7 +1436,8 @@ "min_lat": 45.45, "max_lon": 16.56, "max_lat": 46.85 - } + }, + "archive_ranges": [[44, 12, 16], [46, 12, 18]] }, "SE": { "full_name": "Sweden", @@ -1321,7 +1446,8 @@ "min_lat": 55.36, "max_lon": 23.9, "max_lat": 69.11 - } + }, + "archive_ranges": [[54, 12, 16], [56, 10, 20], [58, 10, 20], [60, 12, 20], [62, 10, 22], [64, 12, 26], [66, 14, 26], [68, 16, 24]] }, "SZ": { "full_name": "Swaziland", @@ -1339,7 +1465,8 @@ "min_lat": 32.31, "max_lon": 42.35, "max_lat": 37.23 - } + }, + "archive_ranges": [[32, 34, 42], [34, 34, 42], [36, 36, 44]] }, "TD": { "full_name": "Chad", @@ -1348,7 +1475,8 @@ "min_lat": 7.42, "max_lon": 23.89, "max_lat": 23.41 - } + }, + "archive_ranges": [[6, 14, 18], [8, 12, 22], [10, 14, 24], [12, 12, 24], [14, 12, 24], [16, 14, 24], [18, 14, 24], [20, 14, 24], [22, 14, 20]] }, "TG": { "full_name": "Togo", @@ -1357,7 +1485,8 @@ "min_lat": 5.93, "max_lon": 1.87, "max_lat": 11.02 - } + }, + "archive_ranges": [[6, 0, 2], [8, 0, 2], [10, -2, 2]] }, "TH": { "full_name": "Thailand", @@ -1366,7 +1495,8 @@ "min_lat": 5.69, "max_lon": 105.59, "max_lat": 20.42 - } + }, + "archive_ranges": [[4, 100, 102], [6, 98, 104], [8, 96, 102], [10, 98, 100], [10, 102, 104], [12, 98, 104], [14, 98, 106], [16, 96, 106], [18, 96, 106], [20, 98, 102]] }, "TJ": { "full_name": "Tajikistan", @@ -1375,7 +1505,8 @@ "min_lat": 36.74, "max_lon": 74.98, "max_lat": 40.96 - } + }, + "archive_ranges": [[36, 66, 76], [38, 66, 76], [40, 68, 72]] }, "TM": { "full_name": "Turkmenistan", @@ -1384,7 +1515,8 @@ "min_lat": 35.27, "max_lon": 66.55, "max_lat": 42.75 - } + }, + "archive_ranges": [[34, 60, 66], [36, 52, 68], [38, 52, 68], [40, 52, 64], [42, 52, 62]] }, "TL": { "full_name": "East Timor", @@ -1411,7 +1543,8 @@ "min_lat": 30.31, "max_lon": 11.49, "max_lat": 37.35 - } + }, + "archive_ranges": [[30, 8, 12], [32, 6, 12], [34, 6, 12], [36, 8, 12]] }, "TR": { "full_name": "Turkey", @@ -1420,7 +1553,8 @@ "min_lat": 35.82, "max_lon": 44.79, "max_lat": 42.14 - } + }, + "archive_ranges": [[34, 34, 38], [36, 26, 46], [38, 24, 46], [40, 24, 46], [42, 26, 28], [42, 32, 36]] }, "TW": { "full_name": "Taiwan", @@ -1429,7 +1563,8 @@ "min_lat": 21.97, "max_lon": 121.95, "max_lat": 25.3 - } + }, + "archive_ranges": [[20, 120, 122], [22, 120, 122], [24, 120, 124]] }, "TZ": { "full_name": "Tanzania", @@ -1438,7 +1573,8 @@ "min_lat": -11.72, "max_lon": 40.32, "max_lat": -0.95 - } + }, + "archive_ranges": [[-12, 34, 42], [-10, 30, 40], [-8, 28, 40], [-6, 28, 40], [-4, 30, 40], [-2, 30, 36]] }, "UG": { "full_name": "Uganda", @@ -1447,7 +1583,8 @@ "min_lat": -1.44, "max_lon": 35.04, "max_lat": 4.25 - } + }, + "archive_ranges": [[-2, 28, 34], [0, 28, 36], [2, 30, 36], [4, 32, 36]] }, "UA": { "full_name": "Ukraine", @@ -1456,7 +1593,8 @@ "min_lat": 44.36, "max_lon": 40.08, "max_lat": 52.34 - } + }, + "archive_ranges": [[44, 28, 32], [44, 34, 36], [46, 22, 40], [48, 22, 42], [50, 22, 40], [52, 30, 36]] }, "UY": { "full_name": "Uruguay", @@ -1475,7 +1613,8 @@ "max_lon": -66.96, "max_lat": 49.5 }, - "submenu": "us_state" + "submenu": "us_state", + "archive_ranges": [[24, -98, -96], [24, -84, -80], [26, -100, -96], [26, -84, -80], [28, -106, -88], [28, -86, -80], [30, -114, -80], [32, -122, -76], [34, -122, -74], [36, -124, -74], [38, -126, -74], [40, -126, -68], [42, -126, -68], [44, -126, -82], [44, -78, -66], [46, -126, -82], [46, -72, -66], [48, -126, -86]] }, "UZ": { "full_name": "Uzbekistan", @@ -1484,7 +1623,8 @@ "min_lat": 37.14, "max_lon": 73.06, "max_lat": 45.59 - } + }, + "archive_ranges": [[36, 66, 70], [38, 62, 72], [40, 54, 74], [42, 54, 68], [42, 70, 72], [44, 54, 62]] }, "VE": { "full_name": "Venezuela", @@ -1493,7 +1633,8 @@ "min_lat": 0.72, "max_lon": -59.76, "max_lat": 12.16 - } + }, + "archive_ranges": [[0, -68, -62], [2, -68, -62], [4, -68, -60], [6, -74, -60], [8, -74, -58], [10, -74, -60], [12, -72, -66]] }, "VN": { "full_name": "Vietnam", @@ -1502,7 +1643,8 @@ "min_lat": 8.6, "max_lon": 109.34, "max_lat": 23.35 - } + }, + "archive_ranges": [[8, 104, 108], [10, 102, 110], [12, 106, 110], [14, 106, 110], [16, 104, 110], [18, 102, 108], [20, 102, 108], [22, 102, 108]] }, "VU": { "full_name": "Vanuatu", @@ -1511,7 +1653,8 @@ "min_lat": -16.6, "max_lon": 167.84, "max_lat": -14.63 - } + }, + "archive_ranges": [[-18, 166, 170], [-16, 166, 170]] }, "PS": { "full_name": "West Bank", @@ -1529,7 +1672,8 @@ "min_lat": 12.59, "max_lon": 53.11, "max_lat": 19 - } + }, + "archive_ranges": [[12, 42, 50], [12, 52, 56], [14, 42, 54], [16, 42, 54], [18, 48, 54]] }, "ZA": { "full_name": "South Africa", @@ -1538,7 +1682,8 @@ "min_lat": -34.82, "max_lon": 32.83, "max_lat": -22.09 - } + }, + "archive_ranges": [[-36, 18, 26], [-34, 16, 30], [-32, 16, 32], [-30, 16, 34], [-28, 18, 34], [-26, 18, 32], [-24, 26, 32]] }, "ZM": { "full_name": "Zambia", @@ -1547,7 +1692,8 @@ "min_lat": -17.96, "max_lon": 33.49, "max_lat": -8.24 - } + }, + "archive_ranges": [[-20, 24, 28], [-18, 20, 30], [-16, 20, 34], [-14, 20, 34], [-12, 22, 34], [-10, 28, 34]] }, "ZW": { "full_name": "Zimbabwe", @@ -1556,7 +1702,8 @@ "min_lat": -22.27, "max_lon": 32.85, "max_lat": -15.51 - } + }, + "archive_ranges": [[-24, 28, 32], [-22, 26, 34], [-20, 24, 34], [-18, 24, 34], [-16, 28, 32]] } }, "us_state": { @@ -1576,7 +1723,8 @@ "min_lat": 51.229087747767466, "max_lon": 179.77488070600702, "max_lat": 71.352561 - } + }, + "archive_ranges": [[50, -180, -172], [50, 176, 180], [52, -178, -164], [52, 172, 180], [54, -168, -154], [54, -136, -128], [56, -172, -168], [56, -162, -152], [56, -138, -130], [58, -168, -132], [60, -174, -172], [60, -168, -138], [62, -172, -140], [64, -170, -140], [66, -168, -140], [68, -168, -140], [70, -164, -142]] }, "AZ": { "full_name": "Arizona", @@ -1585,7 +1733,8 @@ "min_lat": 31.332406253852533, "max_lon": -109.04483902389023, "max_lat": 37.0039183311733 - } + }, + "archive_ranges": [[30, -114, -108], [32, -116, -108], [34, -116, -108], [36, -116, -108]] }, "AR": { "full_name": "Arkansas", @@ -1594,7 +1743,8 @@ "min_lat": 33.00413641175411, "max_lon": -89.65547287402873, "max_lat": 36.49965029279292 - } + }, + "archive_ranges": [[32, -96, -90], [34, -96, -88], [36, -96, -88]] }, "CA": { "full_name": "California", @@ -1603,7 +1753,8 @@ "min_lat": 32.5342307609976, "max_lon": -114.13445790587905, "max_lat": 42.00965914828148 - } + }, + "archive_ranges": [[32, -122, -114], [34, -122, -114], [36, -124, -114], [38, -126, -118], [40, -126, -118], [42, -124, -120]] }, "CO": { "full_name": "Colorado", @@ -1648,7 +1799,8 @@ "min_lat": 24.51490854927549, "max_lon": -80.03257567895679, "max_lat": 31.000809213282125 - } + }, + "archive_ranges": [[24, -84, -80], [26, -84, -78], [28, -86, -80], [30, -88, -80]] }, "GA": { "full_name": "Georgia", @@ -1657,7 +1809,8 @@ "min_lat": 30.35909162440624, "max_lon": -80.84375612136121, "max_lat": 35.000591132701324 - } + }, + "archive_ranges": [[30, -86, -80], [32, -86, -80], [34, -86, -82]] }, "HI": { "full_name": "Hawaii", @@ -1666,7 +1819,8 @@ "min_lat": 18.91727560534605, "max_lon": -154.80833743387433, "max_lat": 22.23238695135951 - } + }, + "archive_ranges": [[18, -158, -154], [20, -162, -154], [22, -162, -158]] }, "ID": { "full_name": "Idaho", @@ -1675,7 +1829,8 @@ "min_lat": 41.988182656016555, "max_lon": -111.04407577795777, "max_lat": 49.00068691035909 - } + }, + "archive_ranges": [[40, -118, -110], [42, -118, -110], [44, -118, -110], [46, -118, -114], [48, -118, -116]] }, "IL": { "full_name": "Illinois", @@ -1684,7 +1839,8 @@ "min_lat": 36.97041500324003, "max_lon": -87.4947178902789, "max_lat": 42.508772828518275 - } + }, + "archive_ranges": [[36, -92, -88], [38, -92, -86], [40, -92, -86], [42, -92, -86]] }, "IN": { "full_name": "Indiana", @@ -1693,7 +1849,8 @@ "min_lat": 37.77191769456694, "max_lon": -84.78480092560925, "max_lat": 41.760531838008376 - } + }, + "archive_ranges": [[36, -90, -84], [38, -90, -84], [40, -88, -84]] }, "IA": { "full_name": "Iowa", @@ -1720,7 +1877,8 @@ "min_lat": 36.49707311372113, "max_lon": -81.96720514115141, "max_lat": 39.14641319952199 - } + }, + "archive_ranges": [[36, -90, -80], [38, -88, -82]] }, "LA": { "full_name": "Louisiana", @@ -1729,7 +1887,8 @@ "min_lat": 28.929616299252984, "max_lon": -88.81557807968079, "max_lat": 33.01959948618486 - } + }, + "archive_ranges": [[28, -94, -88], [30, -96, -88], [32, -96, -90]] }, "ME": { "full_name": "Maine", @@ -1738,7 +1897,8 @@ "min_lat": 43.059430090190894, "max_lon": -66.9819027206272, "max_lat": 47.459533825428245 - } + }, + "archive_ranges": [[42, -72, -68], [44, -72, -66], [46, -72, -66]] }, "MD": { "full_name": "Maryland", @@ -1747,7 +1907,8 @@ "min_lat": 37.91709878227782, "max_lon": -75.05063561675617, "max_lat": 39.72284225191251 - } + }, + "archive_ranges": [[36, -78, -74], [38, -80, -74]] }, "MA": { "full_name": "Massachusetts", @@ -1765,7 +1926,8 @@ "min_lat": 41.696102361213605, "max_lon": -82.4158668902689, "max_lat": 48.190593622126215 - } + }, + "archive_ranges": [[40, -88, -82], [42, -88, -82], [44, -90, -82], [46, -92, -82], [48, -90, -86]] }, "MN": { "full_name": "Minnesota", @@ -1774,7 +1936,8 @@ "min_lat": 43.49926865177651, "max_lon": -89.4903653503535, "max_lat": 49.384686592055914 - } + }, + "archive_ranges": [[42, -98, -90], [44, -98, -90], [46, -98, -88], [48, -98, -88]] }, "MS": { "full_name": "Mississippi", @@ -1792,7 +1955,8 @@ "min_lat": 35.99538225441254, "max_lon": -89.09913230512305, "max_lat": 40.613687151061505 - } + }, + "archive_ranges": [[34, -92, -88], [36, -96, -88], [38, -96, -90], [40, -96, -90]] }, "MT": { "full_name": "Montana", @@ -1801,7 +1965,8 @@ "min_lat": 44.35832834237342, "max_lon": -104.04136319773197, "max_lat": 49.00154597004969 - } + }, + "archive_ranges": [[44, -116, -104], [46, -118, -104], [48, -118, -104]] }, "NE": { "full_name": "Nebraska", @@ -1810,7 +1975,8 @@ "min_lat": 40.00031853197531, "max_lon": -95.30861091290913, "max_lat": 43.001014031230305 - } + }, + "archive_ranges": [[38, -96, -94], [40, -106, -94], [42, -106, -96]] }, "NV": { "full_name": "Nevada", @@ -1819,7 +1985,8 @@ "min_lat": 35.00145019239192, "max_lon": -114.04113626206261, "max_lat": 42.0019276110661 - } + }, + "archive_ranges": [[34, -116, -114], [36, -120, -114], [38, -122, -114], [40, -122, -114], [42, -118, -114]] }, "NH": { "full_name": "New Hampshire", @@ -1846,7 +2013,8 @@ "min_lat": 31.332406253852533, "max_lon": -103.0004679397794, "max_lat": 37.00048209241092 - } + }, + "archive_ranges": [[30, -110, -104], [32, -110, -102], [34, -110, -102], [36, -110, -102]] }, "NY": { "full_name": "New York", @@ -1855,7 +2023,8 @@ "min_lat": 40.502009391283906, "max_lon": -71.85616396303963, "max_lat": 45.01550900568005 - } + }, + "archive_ranges": [[40, -80, -70], [42, -80, -72], [44, -78, -72]] }, "NC": { "full_name": "North Carolina", @@ -1864,7 +2033,8 @@ "min_lat": 33.85116926668266, "max_lon": -75.45981513195132, "max_lat": 36.5881334409244 - } + }, + "archive_ranges": [[32, -80, -76], [34, -86, -74], [36, -84, -74]] }, "ND": { "full_name": "North Dakota", @@ -1882,7 +2052,8 @@ "min_lat": 38.40504468653686, "max_lon": -80.52071966199662, "max_lat": 41.97787393972939 - } + }, + "archive_ranges": [[38, -86, -80], [40, -86, -80], [42, -82, -80]] }, "OK": { "full_name": "Oklahoma", @@ -1891,7 +2062,8 @@ "min_lat": 33.61664597114971, "max_lon": -94.43282317863178, "max_lat": 37.002200211792115 - } + }, + "archive_ranges": [[32, -100, -94], [34, -102, -94], [36, -104, -94]] }, "OR": { "full_name": "Oregon", @@ -1900,7 +2072,8 @@ "min_lat": 41.99161889477894, "max_lon": -116.46390970729706, "max_lat": 46.26801803457034 - } + }, + "archive_ranges": [[40, -126, -116], [42, -126, -116], [44, -126, -116], [46, -126, -122], [46, -120, -116]] }, "PA": { "full_name": "Pennsylvania", @@ -1909,7 +2082,8 @@ "min_lat": 39.720265072840725, "max_lon": -74.69529551145511, "max_lat": 42.269954234532335 - } + }, + "archive_ranges": [[38, -82, -74], [40, -82, -74], [42, -82, -76]] }, "PR": { "full_name": "Puerto Rico", @@ -1936,7 +2110,8 @@ "min_lat": 32.03425802107021, "max_lon": -78.53942937789378, "max_lat": 35.21535605535055 - } + }, + "archive_ranges": [[30, -82, -80], [32, -84, -78], [34, -84, -78]] }, "SD": { "full_name": "South Dakota", @@ -1954,7 +2129,8 @@ "min_lat": 34.98255087919878, "max_lon": -81.64775797577975, "max_lat": 36.67833470843708 - } + }, + "archive_ranges": [[34, -92, -82], [36, -90, -80]] }, "TX": { "full_name": "Texas", @@ -1963,7 +2139,8 @@ "min_lat": 25.840437651866516, "max_lon": -93.5175532104321, "max_lat": 36.50050935248352 - } + }, + "archive_ranges": [[24, -98, -96], [26, -100, -96], [28, -106, -92], [30, -108, -92], [32, -108, -94], [34, -104, -96], [36, -104, -100]] }, "UT": { "full_name": "Utah", @@ -1972,7 +2149,8 @@ "min_lat": 36.99790491333913, "max_lon": -109.04124972989729, "max_lat": 42.0019276110661 - } + }, + "archive_ranges": [[36, -116, -108], [38, -116, -108], [40, -116, -108], [42, -114, -110]] }, "VT": { "full_name": "Vermont", @@ -1981,7 +2159,8 @@ "min_lat": 42.726973989929895, "max_lon": -71.49364526975269, "max_lat": 45.01550900568005 - } + }, + "archive_ranges": [[42, -74, -72], [44, -74, -70]] }, "VI": { "full_name": "Virgin Islands", @@ -1999,7 +2178,8 @@ "min_lat": 36.540885157941574, "max_lon": -75.24086819838197, "max_lat": 39.46598340442404 - } + }, + "archive_ranges": [[36, -84, -74], [38, -80, -74]] }, "WA": { "full_name": "Washington", @@ -2008,7 +2188,8 @@ "min_lat": 45.54383071539715, "max_lon": -116.9161607504075, "max_lat": 49.00240502974029 - } + }, + "archive_ranges": [[44, -124, -116], [46, -126, -116], [48, -126, -116]] }, "WV": { "full_name": "West Virginia", @@ -2017,7 +2198,8 @@ "min_lat": 37.2015020600106, "max_lon": -77.72107034750347, "max_lat": 40.63859988208881 - } + }, + "archive_ranges": [[36, -84, -78], [38, -84, -76], [40, -82, -80]] }, "WI": { "full_name": "Wisconsin", @@ -2026,7 +2208,8 @@ "min_lat": 42.49159163470634, "max_lon": -86.82351991359913, "max_lat": 47.07725226311263 - } + }, + "archive_ranges": [[42, -92, -86], [44, -94, -86], [46, -94, -88]] }, "WY": { "full_name": "Wyoming", @@ -2044,7 +2227,8 @@ "min_lat": 14.110836636456362, "max_lon": 146.0821779942799, "max_lat": 18.81247032309323 - } + }, + "archive_ranges": [[14, 144, 148], [16, 144, 148], [18, 144, 146], [20, 144, 146]] }, "GM": { "full_name": "Guam", From f538b5e13dd0755939b8839c9c4e7640ebd4e90b Mon Sep 17 00:00:00 2001 From: FrogAi <91348155+FrogAi@users.noreply.github.com> Date: Sat, 22 Aug 2026 13:19:41 -0700 Subject: [PATCH 2/2] Move download region updater to Go --- cmd/update-download-regions/geometry.go | 159 ++++++++++++ cmd/update-download-regions/main.go | 179 ++++++++++++++ cmd/update-download-regions/menu.go | 136 +++++++++++ cmd/update-download-regions/sources.go | 235 ++++++++++++++++++ docs/overriding-internal-defaults.md | 19 +- go.mod | 3 +- go.sum | 2 + scripts/update_download_regions.py | 312 ------------------------ settings/download.go | 64 +++-- 9 files changed, 770 insertions(+), 339 deletions(-) create mode 100644 cmd/update-download-regions/geometry.go create mode 100644 cmd/update-download-regions/main.go create mode 100644 cmd/update-download-regions/menu.go create mode 100644 cmd/update-download-regions/sources.go delete mode 100644 scripts/update_download_regions.py diff --git a/cmd/update-download-regions/geometry.go b/cmd/update-download-regions/geometry.go new file mode 100644 index 0000000..424aa74 --- /dev/null +++ b/cmd/update-download-regions/geometry.go @@ -0,0 +1,159 @@ +package main + +import ( + "fmt" + "math" + "slices" + "sort" + + "github.com/paulmach/orb" + "github.com/paulmach/orb/clip" + "github.com/paulmach/orb/planar" +) + +// Must match settings.GROUP_AREA_BOX_DEGREES, which defines the runtime archive grid. +const archiveDegrees = 2 + +type coordinate struct { + latitude int + longitude int +} + +type archiveRange [3]int + +func archivesForBounds(bounds orb.Bound) []coordinate { + var coordinates []coordinate + for latitude := archiveStart(bounds.Min[1]); latitude < archiveStop(bounds.Max[1]); latitude += archiveDegrees { + for longitude := archiveStart(bounds.Min[0]); longitude < archiveStop(bounds.Max[0]); longitude += archiveDegrees { + coordinates = append(coordinates, coordinate{latitude: latitude, longitude: longitude}) + } + } + return coordinates +} + +func archivesForGeometry(polygons orb.MultiPolygon) []coordinate { + selected := make(map[coordinate]struct{}) + for _, polygon := range polygons { + bounds := polygon.Bound() + for latitude := archiveStart(bounds.Min[1]); latitude < archiveStop(bounds.Max[1]); latitude += archiveDegrees { + for longitude := archiveStart(bounds.Min[0]); longitude < archiveStop(bounds.Max[0]); longitude += archiveDegrees { + archive := orb.Bound{ + Min: orb.Point{float64(longitude), float64(latitude)}, + Max: orb.Point{float64(longitude + archiveDegrees), float64(latitude + archiveDegrees)}, + } + if polygonIntersectsBound(polygon, archive, bounds) { + selected[coordinate{latitude: latitude, longitude: longitude}] = struct{}{} + } + } + } + } + + coordinates := make([]coordinate, 0, len(selected)) + for coordinate := range selected { + coordinates = append(coordinates, coordinate) + } + sort.Slice(coordinates, func(first, second int) bool { + if coordinates[first].latitude != coordinates[second].latitude { + return coordinates[first].latitude < coordinates[second].latitude + } + return coordinates[first].longitude < coordinates[second].longitude + }) + return coordinates +} + +func scopeGeometry(polygons orb.MultiPolygon, bounds orb.Bound, path string) (orb.MultiPolygon, error) { + if len(polygons) == 0 { + return nil, fmt.Errorf("%s has no source geometry", path) + } + for _, polygon := range polygons { + for _, ring := range polygon { + if ringCrossesAntimeridian(ring) { + return nil, fmt.Errorf("%s has unsupported source geometry", path) + } + } + } + + seed := orb.Bound{ + Min: orb.Point{float64(archiveStart(bounds.Min[0])), float64(archiveStart(bounds.Min[1]))}, + Max: orb.Point{float64(archiveStop(bounds.Max[0])), float64(archiveStop(bounds.Max[1]))}, + } + + // Use the existing bounds to choose components, then keep each selected + // component whole so normal border changes do not get clipped. + selected := make(orb.MultiPolygon, 0, len(polygons)) + for _, polygon := range polygons { + if polygonIntersectsBound(polygon, seed, polygon.Bound()) { + selected = append(selected, polygon) + } + } + if len(selected) == 0 { + return nil, fmt.Errorf("%s does not intersect its bounding_box", path) + } + return selected, nil +} + +func compactRanges(coordinates []coordinate) []archiveRange { + var ranges []archiveRange + for index := 0; index < len(coordinates); { + latitude := coordinates[index].latitude + minimumLongitude := coordinates[index].longitude + maximumLongitude := minimumLongitude + archiveDegrees + index++ + + for index < len(coordinates) && coordinates[index].latitude == latitude && coordinates[index].longitude == maximumLongitude { + maximumLongitude += archiveDegrees + index++ + } + ranges = append(ranges, archiveRange{latitude, minimumLongitude, maximumLongitude}) + } + return ranges +} + +func archiveStart(value float64) int { + return int(math.Floor(value/archiveDegrees)) * archiveDegrees +} + +func archiveStop(value float64) int { + return int(math.Ceil(value/archiveDegrees)) * archiveDegrees +} + +func coordinatesEqual(first, second []coordinate) bool { + return slices.Equal(first, second) +} + +func polygonIntersectsBound(polygon orb.Polygon, target, bounds orb.Bound) bool { + if !bounds.Intersects(target) { + return false + } + for _, ring := range polygon { + if len(ring) == 1 && target.Contains(ring[0]) { + return true + } + if len(ring) < 2 { + continue + } + line := orb.LineString(ring).Clone() + if line[0] != line[len(line)-1] { + line = append(line, line[0]) + } + if len(clip.LineString(target, line)) > 0 { + return true + } + } + + for _, corner := range target.ToRing()[:4] { + if planar.PolygonContains(polygon, corner) { + return true + } + } + return false +} + +func ringCrossesAntimeridian(ring orb.Ring) bool { + for index, start := range ring { + if math.Abs(start[0]-ring[(index+1)%len(ring)][0]) > 180 { + return true + } + } + return false +} diff --git a/cmd/update-download-regions/main.go b/cmd/update-download-regions/main.go new file mode 100644 index 0000000..9e22725 --- /dev/null +++ b/cmd/update-download-regions/main.go @@ -0,0 +1,179 @@ +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "log" + "os" + "path/filepath" + "sort" + + "github.com/paulmach/orb" +) + +const downloadMenuPath = "settings/download_menu.json" + +type menuBounds struct { + MinLat float64 `json:"min_lat"` + MinLon float64 `json:"min_lon"` + MaxLat float64 `json:"max_lat"` + MaxLon float64 `json:"max_lon"` +} + +type menuLocation struct { + BoundingBox menuBounds `json:"bounding_box"` +} + +type ( + downloadMenu map[string]map[string]menuLocation + downloadRanges map[string]map[string][]archiveRange +) + +type summary struct { + locations int + ranged int + legacy int + selected int +} + +func (summary summary) String() string { + return fmt.Sprintf( + "%d regions, %d with explicit ranges; %d legacy archive occurrences -> %d selected", + summary.locations, summary.ranged, summary.legacy, summary.selected, + ) +} + +func main() { + write := flag.Bool("write", false, "write updated archive ranges to the download menu instead of checking them") + flag.Parse() + if err := updateDownloadMenu(*write); err != nil { + log.Fatal(err) + } +} + +func updateDownloadMenu(write bool) error { + rawMenu, err := os.ReadFile(downloadMenuPath) + if err != nil { + return err + } + menuDocument, err := parseJSONObject(rawMenu) + if err != nil { + return err + } + var menu downloadMenu + if err := json.Unmarshal(rawMenu, &menu); err != nil { + return err + } + + cacheDirectory, err := os.UserCacheDir() + if err != nil { + return err + } + ranges, summary, err := generateDownloadRanges(menu, filepath.Join(cacheDirectory, "mapd", "download-region-sources")) + if err != nil { + return err + } + + newline := "\n" + if bytes.Contains(rawMenu, []byte("\r\n")) { + newline = "\r\n" + } + if err := inlineArchiveRanges(&menuDocument, ranges, newline); err != nil { + return err + } + expected := append(renderJSONObject(menuDocument, 0, newline), newline...) + changed := !bytes.Equal(rawMenu, expected) + + if changed && !write { + return fmt.Errorf("download menu is stale (%s); run with --write", summary) + } + if changed { + if err := os.WriteFile(downloadMenuPath, expected, 0o644); err != nil { + return err + } + fmt.Printf("updated %s (%s)\n", downloadMenuPath, summary) + } else { + fmt.Printf("download menu is up to date (%s)\n", summary) + } + return nil +} + +func generateDownloadRanges(menu downloadMenu, cacheDirectory string) (downloadRanges, summary, error) { + countryCodes := sortedKeys(menu["nation"]) + stateCodes := sortedKeys(menu["us_state"]) + if len(countryCodes)+len(stateCodes) == 0 { + return nil, summary{}, fmt.Errorf("download menu contains no nation or us_state regions") + } + + countryPath, err := fetchSource(countrySource, cacheDirectory) + if err != nil { + return nil, summary{}, err + } + countryGeometries, err := loadCountryGeometries(countryPath, countryCodes) + if err != nil { + return nil, summary{}, err + } + + statePath, err := fetchSource(stateSource, cacheDirectory) + if err != nil { + return nil, summary{}, err + } + stateGeometries, err := loadStateGeometries(statePath, stateCodes) + if err != nil { + return nil, summary{}, err + } + + geometries := map[string]regionGeometries{ + "nation": countryGeometries, + "us_state": stateGeometries, + } + ranges := make(downloadRanges) + var result summary + for _, section := range []string{"nation", "us_state"} { + for _, code := range sortedKeys(menu[section]) { + path := section + "." + code + bounds, err := menu[section][code].BoundingBox.bound() + if err != nil { + return nil, summary{}, fmt.Errorf("%s: %w", path, err) + } + legacy := archivesForBounds(bounds) + scoped, err := scopeGeometry(geometries[section][code], bounds, path) + if err != nil { + return nil, summary{}, err + } + selected := archivesForGeometry(scoped) + + result.locations++ + result.legacy += len(legacy) + result.selected += len(selected) + if coordinatesEqual(selected, legacy) { + continue + } + if ranges[section] == nil { + ranges[section] = make(map[string][]archiveRange) + } + ranges[section][code] = compactRanges(selected) + result.ranged++ + } + } + return ranges, result, nil +} + +func (bounds menuBounds) bound() (orb.Bound, error) { + result := orb.Bound{Min: orb.Point{bounds.MinLon, bounds.MinLat}, Max: orb.Point{bounds.MaxLon, bounds.MaxLat}} + if result.Min[0] < -180 || result.Max[0] > 180 || result.Min[0] >= result.Max[0] || result.Min[1] < -90 || result.Max[1] > 90 || result.Min[1] >= result.Max[1] { + return orb.Bound{}, fmt.Errorf("invalid bounding_box") + } + return result, nil +} + +func sortedKeys[T any](values map[string]T) []string { + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} diff --git a/cmd/update-download-regions/menu.go b/cmd/update-download-regions/menu.go new file mode 100644 index 0000000..adf888c --- /dev/null +++ b/cmd/update-download-regions/menu.go @@ -0,0 +1,136 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "strings" +) + +type jsonField struct { + name string + value json.RawMessage +} + +type jsonObject []jsonField + +func parseJSONObject(data []byte) (jsonObject, error) { + decoder := json.NewDecoder(bytes.NewReader(data)) + token, err := decoder.Token() + if err != nil { + return nil, err + } + if delimiter, ok := token.(json.Delim); !ok || delimiter != '{' { + return nil, fmt.Errorf("expected JSON object") + } + + var object jsonObject + names := make(map[string]struct{}) + for decoder.More() { + token, err := decoder.Token() + if err != nil { + return nil, err + } + name := token.(string) + if _, exists := names[name]; exists { + return nil, fmt.Errorf("duplicate JSON field %q", name) + } + names[name] = struct{}{} + + var value json.RawMessage + if err := decoder.Decode(&value); err != nil { + return nil, err + } + object = append(object, jsonField{name: name, value: value}) + } + if _, err := decoder.Token(); err != nil { + return nil, err + } + if _, err := decoder.Token(); err != io.EOF { + if err == nil { + err = fmt.Errorf("unexpected data after JSON object") + } + return nil, err + } + return object, nil +} + +func (object jsonObject) field(name string) (*jsonField, bool) { + for index := range object { + if object[index].name == name { + return &object[index], true + } + } + return nil, false +} + +func (object *jsonObject) set(name string, value json.RawMessage) { + if field, exists := object.field(name); exists { + field.value = value + return + } + *object = append(*object, jsonField{name: name, value: value}) +} + +func (object *jsonObject) delete(name string) { + for index := range *object { + if (*object)[index].name == name { + *object = append((*object)[:index], (*object)[index+1:]...) + return + } + } +} + +func renderJSONObject(object jsonObject, indent int, newline string) json.RawMessage { + if len(object) == 0 { + return json.RawMessage("{}") + } + + var output strings.Builder + output.WriteByte('{') + for index, field := range object { + if index > 0 { + output.WriteByte(',') + } + output.WriteString(newline) + output.WriteString(strings.Repeat(" ", indent+2)) + name, _ := json.Marshal(field.name) + output.Write(name) + output.WriteString(": ") + output.Write(bytes.TrimSpace(field.value)) + } + output.WriteString(newline) + output.WriteString(strings.Repeat(" ", indent)) + output.WriteByte('}') + return json.RawMessage(output.String()) +} + +func inlineArchiveRanges(menu *jsonObject, ranges downloadRanges, newline string) error { + for _, sectionName := range []string{"nation", "us_state"} { + sectionField, exists := menu.field(sectionName) + if !exists { + continue + } + section, err := parseJSONObject(sectionField.value) + if err != nil { + return fmt.Errorf("%s: %w", sectionName, err) + } + + for index := range section { + location, err := parseJSONObject(section[index].value) + if err != nil { + return fmt.Errorf("%s.%s: %w", sectionName, section[index].name, err) + } + if archiveRanges, exists := ranges[sectionName][section[index].name]; exists { + encoded, _ := json.Marshal(archiveRanges) + location.set("archive_ranges", bytes.ReplaceAll(encoded, []byte(","), []byte(", "))) + } else { + location.delete("archive_ranges") + } + section[index].value = renderJSONObject(location, 4, newline) + } + sectionField.value = renderJSONObject(section, 2, newline) + } + return nil +} diff --git a/cmd/update-download-regions/sources.go b/cmd/update-download-regions/sources.go new file mode 100644 index 0000000..51053e8 --- /dev/null +++ b/cmd/update-download-regions/sources.go @@ -0,0 +1,235 @@ +package main + +import ( + "crypto/sha256" + "fmt" + "io" + "math" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + shp "github.com/jonas-p/go-shp" + "github.com/paulmach/orb" + "github.com/paulmach/orb/geojson" + "github.com/paulmach/orb/planar" +) + +type source struct { + name string + url string + filename string + sha256 string +} + +var countrySource = source{ + name: "Natural Earth 10m Admin 0 countries", + url: "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/f1890d9f152c896d250a77557a5751a93d494776/geojson/ne_10m_admin_0_countries.geojson", + filename: "ne_10m_admin_0_countries.geojson", + sha256: "239eec57ac17f100a11e2536cffc56752c318b50ae765b0918ff7aab4ce8f255", +} + +var stateSource = source{ + name: "Census TIGER/Line states", + url: "https://www2.census.gov/geo/tiger/TIGER2025/STATE/tl_2025_us_state.zip", + filename: "tl_2025_us_state.zip", + sha256: "59a220888a8d9be8117c4fcd38f542bd02d81abf0d198c78113595ad540dd957", +} + +var countrySelectors = map[string][2]string{ + "FR": {"ADM0_A3", "FRA"}, + "NO": {"ADM0_A3", "NOR"}, + "TW": {"ADM0_A3", "TWN"}, +} + +var stateCodeAliases = map[string]string{"GM": "GU"} + +type regionGeometries map[string]orb.MultiPolygon + +func fetchSource(specification source, cacheDirectory string) (string, error) { + if err := os.MkdirAll(cacheDirectory, 0o755); err != nil { + return "", err + } + destination := filepath.Join(cacheDirectory, specification.filename) + if content, err := os.ReadFile(destination); err == nil && hashBytes(content) == specification.sha256 { + return destination, nil + } + + request, err := http.NewRequest(http.MethodGet, specification.url, nil) + if err != nil { + return "", err + } + request.Header.Set("User-Agent", "mapd-download-region-updater/1") + + client := &http.Client{Timeout: 120 * time.Second} + response, err := client.Do(request) + if err != nil { + return "", err + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return "", fmt.Errorf("%s download returned %s", specification.name, response.Status) + } + content, err := io.ReadAll(response.Body) + if err != nil { + return "", err + } + if hashBytes(content) != specification.sha256 { + return "", fmt.Errorf("%s no longer matches its pinned hash", specification.name) + } + if err := os.WriteFile(destination, content, 0o644); err != nil { + return "", err + } + return destination, nil +} + +func hashBytes(content []byte) string { + hash := sha256.Sum256(content) + return fmt.Sprintf("%x", hash) +} + +func loadCountryGeometries(sourcePath string, codes []string) (regionGeometries, error) { + content, err := os.ReadFile(sourcePath) + if err != nil { + return nil, err + } + collection, err := geojson.UnmarshalFeatureCollection(content) + if err != nil { + return nil, err + } + + geometries := make(regionGeometries, len(codes)) + for _, code := range codes { + selector := [2]string{"ISO_A2", code} + if specialSelector, exists := countrySelectors[code]; exists { + selector = specialSelector + } + + var matches []orb.MultiPolygon + for _, feature := range collection.Features { + value, exists := feature.Properties[selector[0]].(string) + if !exists || value != selector[1] { + continue + } + switch geometry := feature.Geometry.(type) { + case orb.Polygon: + matches = append(matches, orb.MultiPolygon{geometry}) + case orb.MultiPolygon: + matches = append(matches, geometry) + default: + return nil, fmt.Errorf("nation.%s has unsupported GeoJSON geometry %T", code, geometry) + } + } + if len(matches) != 1 { + return nil, fmt.Errorf("nation.%s matched %d source features", code, len(matches)) + } + geometries[code] = matches[0] + } + return geometries, nil +} + +func loadStateGeometries(sourcePath string, codes []string) (regionGeometries, error) { + reader, err := shp.OpenZip(sourcePath) + if err != nil { + return nil, err + } + defer reader.Close() + + codeField := -1 + for index, field := range reader.Fields() { + if field.String() == "STUSPS" { + codeField = index + break + } + } + if codeField == -1 { + return nil, fmt.Errorf("Census source has no STUSPS field") + } + + sourceGeometries := make(regionGeometries) + for reader.Next() { + _, shape := reader.Shape() + shapePolygon, ok := shape.(*shp.Polygon) + if !ok { + return nil, fmt.Errorf("Census source contains %T instead of polygons", shape) + } + polygons, err := polygonsFromShape(shapePolygon) + if err != nil { + return nil, err + } + code := strings.TrimSpace(reader.Attribute(codeField)) + if _, exists := sourceGeometries[code]; exists { + return nil, fmt.Errorf("Census source contains duplicate STUSPS %q", code) + } + sourceGeometries[code] = polygons + } + if err := reader.Err(); err != nil { + return nil, err + } + + geometries := make(regionGeometries, len(codes)) + for _, menuCode := range codes { + sourceCode := menuCode + if alias, exists := stateCodeAliases[menuCode]; exists { + sourceCode = alias + } + geometry, exists := sourceGeometries[sourceCode] + if !exists { + return nil, fmt.Errorf("us_state.%s has no source feature", menuCode) + } + geometries[menuCode] = geometry + } + return geometries, nil +} + +func polygonsFromShape(shape *shp.Polygon) (orb.MultiPolygon, error) { + var exteriors []orb.Ring + var holes []orb.Ring + for index, start := range shape.Parts { + end := len(shape.Points) + if index+1 < len(shape.Parts) { + end = int(shape.Parts[index+1]) + } + if int(start) < 0 || int(start) >= end || end > len(shape.Points) { + return nil, fmt.Errorf("Census source contains invalid polygon parts") + } + ring := make(orb.Ring, end-int(start)) + for pointIndex, sourcePoint := range shape.Points[start:end] { + ring[pointIndex] = orb.Point{sourcePoint.X, sourcePoint.Y} + } + if ring.Orientation() == orb.CW { + exteriors = append(exteriors, ring) + } else { + holes = append(holes, ring) + } + } + + polygons := make(orb.MultiPolygon, len(exteriors)) + for index, exterior := range exteriors { + polygons[index] = orb.Polygon{exterior} + } + for _, hole := range holes { + owner := smallestContainingPolygon(polygons, hole[0]) + if owner == -1 { + polygons = append(polygons, orb.Polygon{hole}) + } else { + polygons[owner] = append(polygons[owner], hole) + } + } + return polygons, nil +} + +func smallestContainingPolygon(polygons orb.MultiPolygon, target orb.Point) int { + owner := -1 + ownerArea := math.Inf(1) + for index, polygon := range polygons { + area := math.Abs(planar.Area(polygon[0])) + if area < ownerArea && planar.RingContains(polygon[0], target) { + owner = index + ownerArea = area + } + } + return owner +} diff --git a/docs/overriding-internal-defaults.md b/docs/overriding-internal-defaults.md index a33b6f6..ea1d35f 100644 --- a/docs/overriding-internal-defaults.md +++ b/docs/overriding-internal-defaults.md @@ -53,7 +53,7 @@ any desired areas to the file. The structure is as follows: "definitions": { "area_menu": { "type": "object", - "additionalProperties": {"$ref": "#/definitions/area"} + "additionalProperties": {"$ref": "#/definitions/area"}, }, "area": { "type": "object", @@ -111,15 +111,18 @@ any desired areas to the file. The structure is as follows: } ``` -`archive_ranges` is optional, but takes precedence over `bounding_box` when -non-empty. Remove it to restore bounding-box selection. The default ranges can -be checked or regenerated with `uv run scripts/update_download_regions.py` and -`uv run scripts/update_download_regions.py --write`, respectively. -Pass `--menu PATH` to process another menu; the updater regenerates whichever of -the `nation` and `us_state` sections are present and leaves other sections unchanged. - Note the optional submenu value in an area. The submenu value allows for chaining of the menus when requesting a download, so the submenu value should exactly match a top level key in the main object. This value is not used by the cli tui however, so selecting an entry with a submenu in the tui will just result in that entry being downloaded. + +`archive_ranges` is optional, but takes precedence over `bounding_box` when +non-empty. Each row must contain exactly three grid-aligned integers within the +world bounds, and its maximum longitude must be greater than its minimum. +Remove it to restore bounding-box selection. From the repository root, check +the generated ranges with `go run ./cmd/update-download-regions` or update them +with `go run ./cmd/update-download-regions --write`. Run the check after changing +menu bounds or source pins. Existing menu bounds select which disconnected +source components belong to each region, so new countries and newly relevant +detached territories require an intentional menu change. diff --git a/go.mod b/go.mod index 5b178c2..ddc0935 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,8 @@ require ( github.com/charmbracelet/bubbletea v1.3.10 github.com/charmbracelet/lipgloss v1.1.0 github.com/gofrs/flock v0.13.0 + github.com/jonas-p/go-shp v0.1.1 + github.com/paulmach/orb v0.1.3 github.com/paulmach/osm v0.8.0 github.com/pfeiferj/gomsgq v0.1.11 github.com/pkg/errors v0.9.1 @@ -33,7 +35,6 @@ require ( github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect - github.com/paulmach/orb v0.1.3 // indirect github.com/paulmach/protoscan v0.2.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/sahilm/fuzzy v0.1.1 // indirect diff --git a/go.sum b/go.sum index 4c17ae9..cab59e2 100644 --- a/go.sum +++ b/go.sum @@ -39,6 +39,8 @@ github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/jonas-p/go-shp v0.1.1 h1:LY81nN67DBCz6VNFn2kS64CjmnDo9IP8rmSkTvhO9jE= +github.com/jonas-p/go-shp v0.1.1/go.mod h1:MRIhyxDQ6VVp0oYeD7yPGr5RSTNScUFKCDsI5DR7PtI= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= diff --git a/scripts/update_download_regions.py b/scripts/update_download_regions.py deleted file mode 100644 index c97411e..0000000 --- a/scripts/update_download_regions.py +++ /dev/null @@ -1,312 +0,0 @@ -#!/usr/bin/env -S uv run --script -# /// script -# requires-python = ">=3.12,<3.14" -# dependencies = [ -# "numpy==2.5.2", -# "pyshp==3.1.6", -# "shapely==2.1.2", -# ] -# /// -import argparse -import copy -import hashlib -import io -import json -import math -import shapefile -import sys -import tempfile -import urllib.request -import zipfile - -from itertools import pairwise, product -from pathlib import Path -from shapely import make_valid -from shapely.geometry import GeometryCollection, MultiPolygon, Polygon, box, shape -from shapely.ops import unary_union - -DESCRIPTION = """Update the default menu's archive ranges from pinned boundary sources. - -Run with uv run scripts/update_download_regions.py to check the menu or add ---write to update it. To adopt a newer boundary release, update its source -specification in this script, then review the generated diff. - -The existing menu remains authoritative for which regions and disconnected -territories it contains. New countries and newly relevant detached territories -require an intentional menu change; this script never makes that policy choice. -""" - -ARCHIVE_DEGREES = 2 - -REPOSITORY_ROOT = Path(__file__).resolve().parents[1] -DEFAULT_MENU_PATH = REPOSITORY_ROOT / "settings" / "download_menu.json" -DEFAULT_CACHE_DIRECTORY = Path(tempfile.gettempdir()) / "mapd-download-region-sources" - -# Natural Earth data is in the public domain. -COUNTRY_SOURCE = { - "name": "Natural Earth 10m Admin 0 countries", - "url": "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/f1890d9f152c896d250a77557a5751a93d494776/geojson/ne_10m_admin_0_countries.geojson", - "filename": "ne_10m_admin_0_countries.geojson", - "sha256": "239eec57ac17f100a11e2536cffc56752c318b50ae765b0918ff7aab4ce8f255", -} - -# United States Census Bureau data is in the public domain. -STATE_SOURCE = { - "name": "Census TIGER/Line states", - "url": "https://www2.census.gov/geo/tiger/TIGER2025/STATE/tl_2025_us_state.zip", - "filename": "tl_2025_us_state.zip", - "sha256": "59a220888a8d9be8117c4fcd38f542bd02d81abf0d198c78113595ad540dd957", -} - -# Natural Earth uses nonstandard ISO_A2 values for these menu entries. -COUNTRY_SELECTORS = { - "FR": ("ADM0_A3", "FRA"), - "NO": ("ADM0_A3", "NOR"), - "TW": ("ADM0_A3", "TWN"), -} - -# mapd historically used GM for Guam; Census uses the standard GU code. -STATE_CODE_ALIASES = {"GM": "GU"} - - -class UpdateError(RuntimeError): - pass - - -def sha256_bytes(data): - return hashlib.sha256(data).hexdigest() - - -def fetch_source(specification, cache_directory): - cache_directory.mkdir(parents=True, exist_ok=True) - destination = cache_directory / specification["filename"] - if destination.is_file() and sha256_bytes(destination.read_bytes()) == specification["sha256"]: - return destination - - request = urllib.request.Request(specification["url"], headers={"User-Agent": "mapd-download-region-updater/1"}) - with urllib.request.urlopen(request, timeout=120) as response: - content = response.read() - if sha256_bytes(content) != specification["sha256"]: - raise UpdateError(f"{specification['name']} no longer matches its pinned hash") - destination.write_bytes(content) - return destination - - -def polygon_components(geometry): - if isinstance(geometry, Polygon): - return [geometry] - if isinstance(geometry, MultiPolygon): - return list(geometry.geoms) - if isinstance(geometry, GeometryCollection): - return [polygon for part in geometry.geoms for polygon in polygon_components(part)] - return [] - - -def has_antimeridian_jump(geometry): - return any( - abs(first[0] - second[0]) > 180 - for polygon in polygon_components(geometry) - for ring in (polygon.exterior, *polygon.interiors) - for first, second in pairwise(ring.coords) - ) - - -def scoped_geometry(geometry, bounds, path): - if not geometry.is_valid: - geometry = make_valid(geometry) - - components = polygon_components(geometry) - if has_antimeridian_jump(geometry): - raise UpdateError(f"{path} has unsupported source geometry") - - latitude_archives = archive_axis(bounds["min_lat"], bounds["max_lat"]) - longitude_archives = archive_axis(bounds["min_lon"], bounds["max_lon"]) - seed = box(longitude_archives.start, latitude_archives.start, longitude_archives.stop, latitude_archives.stop) - selected_components = [component for component in components if component.intersects(seed)] - if not selected_components: - raise UpdateError(f"{path} does not intersect its bounding_box") - - # Use the existing archive-aligned scope to choose components, then keep each - # selected component whole so small bounding-box errors do not clip its archives. - return unary_union(selected_components) - - -def archive_axis(minimum, maximum): - return range( - math.floor(minimum / ARCHIVE_DEGREES) * ARCHIVE_DEGREES, - math.ceil(maximum / ARCHIVE_DEGREES) * ARCHIVE_DEGREES, - ARCHIVE_DEGREES, - ) - - -def archives_for_bounds(bounds): - return list( - product( - archive_axis(bounds["min_lat"], bounds["max_lat"]), - archive_axis(bounds["min_lon"], bounds["max_lon"]), - ) - ) - - -def archives_for_geometry(geometry): - min_longitude, min_latitude, max_longitude, max_latitude = geometry.bounds - coordinates = product( - archive_axis(min_latitude, max_latitude), - archive_axis(min_longitude, max_longitude), - ) - return [ - (latitude, longitude) - for latitude, longitude in coordinates - if geometry.intersects(box(longitude, latitude, longitude + ARCHIVE_DEGREES, latitude + ARCHIVE_DEGREES)) - ] - - -def compact_ranges(coordinates): - sorted_coordinates = sorted(coordinates) - ranges = [] - index = 0 - while index < len(sorted_coordinates): - latitude, min_longitude = sorted_coordinates[index] - max_longitude = min_longitude + ARCHIVE_DEGREES - index += 1 - while index < len(sorted_coordinates): - next_latitude, next_longitude = sorted_coordinates[index] - if next_latitude != latitude or next_longitude != max_longitude: - break - max_longitude += ARCHIVE_DEGREES - index += 1 - ranges.append([latitude, min_longitude, max_longitude]) - return ranges - - -def load_country_geometries(source_path, codes): - features = json.loads(source_path.read_text(encoding="utf-8"))["features"] - - geometries = {} - for code in codes: - field, value = COUNTRY_SELECTORS.get(code, ("ISO_A2", code)) - matches = [feature for feature in features if feature.get("properties", {}).get(field) == value] - if len(matches) != 1: - raise UpdateError(f"nation.{code} matched {len(matches)} source features") - geometries[code] = shape(matches[0]["geometry"]) - return geometries - - -def load_state_geometries(source_path, codes): - with zipfile.ZipFile(source_path) as source_zip: - source_names = source_zip.namelist() - shape_name = next(name for name in source_names if name.lower().endswith(".shp")) - database_name = next(name for name in source_names if name.lower().endswith(".dbf")) - reader = shapefile.Reader( - shp=io.BytesIO(source_zip.read(shape_name)), - dbf=io.BytesIO(source_zip.read(database_name)), - ) - source_geometries = {} - for record in reader.iterShapeRecords(): - source_geometries[record.record.as_dict()["STUSPS"]] = shape(record.shape.__geo_interface__) - - geometries = {} - for menu_code in codes: - source_code = STATE_CODE_ALIASES.get(menu_code, menu_code) - if source_code not in source_geometries: - raise UpdateError(f"us_state.{menu_code} has no source feature") - geometries[menu_code] = source_geometries[source_code] - return geometries - - -def update_menu(menu, geometries): - summary = {"legacy": 0, "locations": 0, "ranged": 0, "selected": 0} - - for section_name, section_geometries in geometries.items(): - entries = menu.get(section_name, {}) - for code, entry in entries.items(): - path = f"{section_name}.{code}" - bounds = entry["bounding_box"] - legacy_coordinates = archives_for_bounds(bounds) - geometry = scoped_geometry(section_geometries[code], bounds, path) - selected_coordinates = archives_for_geometry(geometry) - ranges = compact_ranges(selected_coordinates) - uses_archive_ranges = selected_coordinates != legacy_coordinates - if uses_archive_ranges: - entry["archive_ranges"] = ranges - else: - entry.pop("archive_ranges", None) - - summary["legacy"] += len(legacy_coordinates) - summary["locations"] += 1 - summary["ranged"] += int(uses_archive_ranges) - summary["selected"] += len(selected_coordinates) - return menu, summary - - -def render_menu(menu, newline): - rendered_menu = copy.deepcopy(menu) - replacements = {} - replacement_index = 0 - for entries in rendered_menu.values(): - for entry in entries.values(): - if "archive_ranges" not in entry: - continue - token = f"__MAPD_ARCHIVE_RANGES_{replacement_index}__" - replacements[token] = entry["archive_ranges"] - entry["archive_ranges"] = token - replacement_index += 1 - - rendered = json.dumps(rendered_menu, ensure_ascii=False, indent=2) + "\n" - for token, ranges in replacements.items(): - rendered = rendered.replace(json.dumps(token), json.dumps(ranges)) - return rendered.replace("\n", newline) - - -def parse_arguments(): - parser = argparse.ArgumentParser(description=DESCRIPTION, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("--menu", type=Path, default=DEFAULT_MENU_PATH) - parser.add_argument("--cache-dir", type=Path, default=DEFAULT_CACHE_DIRECTORY) - - mode = parser.add_mutually_exclusive_group() - mode.add_argument("--check", action="store_true", help="check without changing the menu (default)") - mode.add_argument("--write", action="store_true", help="update the menu") - - return parser.parse_args() - - -def run(arguments): - raw_menu = arguments.menu.read_bytes() - newline = "\r\n" if b"\r\n" in raw_menu else "\n" - menu = json.loads(raw_menu.decode("utf-8")) - - country_source = fetch_source(COUNTRY_SOURCE, arguments.cache_dir) - state_source = fetch_source(STATE_SOURCE, arguments.cache_dir) - - country_geometries = load_country_geometries(country_source, menu.get("nation", {})) - state_geometries = load_state_geometries(state_source, menu.get("us_state", {})) - - updated_menu, summary = update_menu(menu, {"nation": country_geometries, "us_state": state_geometries}) - expected = render_menu(updated_menu, newline).encode("utf-8") - - message = ("{locations} regions, {ranged} with explicit ranges; {legacy} legacy archive occurrences -> {selected} selected").format(**summary) - if not arguments.write: - if raw_menu != expected: - print(f"download menu is stale ({message}); run with --write", file=sys.stderr) - return 1 - print(f"download menu is up to date ({message})") - return 0 - - if raw_menu == expected: - print(f"download menu already up to date ({message})") - return 0 - arguments.menu.write_bytes(expected) - print(f"updated {arguments.menu} ({message})") - return 0 - - -def main(): - try: - return run(parse_arguments()) - except (UpdateError, OSError, ValueError, zipfile.BadZipFile) as error: - print(f"error: {error}", file=sys.stderr) - return 2 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/settings/download.go b/settings/download.go index 0249862..e737a06 100644 --- a/settings/download.go +++ b/settings/download.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "io" + "iter" "log/slog" "math" "net/http" @@ -27,8 +28,22 @@ type LocationData struct { // ArchiveRange is [minimum latitude, inclusive minimum longitude, exclusive maximum longitude] for a 2-degree latitude band. type ArchiveRange [3]int -func (r ArchiveRange) coordinates() (latitude, minLongitude, maxLongitude int) { - return r[0], r[1], r[2] +func (r *ArchiveRange) UnmarshalJSON(data []byte) error { + var coordinates []int + if err := json.Unmarshal(data, &coordinates); err != nil { + return err + } + if len(coordinates) != len(ArchiveRange{}) { + return fmt.Errorf("archive range must contain three coordinates") + } + + latitude, minLongitude, maxLongitude := coordinates[0], coordinates[1], coordinates[2] + if latitude < -90 || latitude >= 90 || minLongitude < -180 || maxLongitude > 180 || minLongitude >= maxLongitude || + latitude%GROUP_AREA_BOX_DEGREES != 0 || minLongitude%GROUP_AREA_BOX_DEGREES != 0 || maxLongitude%GROUP_AREA_BOX_DEGREES != 0 { + return fmt.Errorf("invalid archive range %v", coordinates) + } + *r = ArchiveRange{latitude, minLongitude, maxLongitude} + return nil } type DownloadMenu map[string]map[string]LocationData @@ -171,24 +186,31 @@ func adjustedBounds(bounds Bounds) (int, int, int, int) { return minLat, minLon, maxLat, maxLon } -func archiveRangesForLocation(location LocationData) []ArchiveRange { - if len(location.ArchiveRanges) > 0 { - return location.ArchiveRanges - } +func archiveRangesForLocation(location LocationData) iter.Seq[ArchiveRange] { + return func(yield func(ArchiveRange) bool) { + if len(location.ArchiveRanges) > 0 { + for _, archiveRange := range location.ArchiveRanges { + if !yield(archiveRange) { + return + } + } + return + } - minLat, minLon, maxLat, maxLon := adjustedBounds(location.BoundingBox) - var archiveRanges []ArchiveRange - for lat := minLat; lat < maxLat; lat += GROUP_AREA_BOX_DEGREES { - archiveRanges = append(archiveRanges, ArchiveRange{lat, minLon, maxLon}) + minLat, minLon, maxLat, maxLon := adjustedBounds(location.BoundingBox) + for latitude := minLat; latitude < maxLat; latitude += GROUP_AREA_BOX_DEGREES { + if !yield(ArchiveRange{latitude, minLon, maxLon}) { + return + } + } } - return archiveRanges } func (d *download) downloadLocation(location LocationData, locationName string) (err error, cancel bool) { slog.Info("Downloading Location", "location", locationName) - for _, archiveRange := range archiveRangesForLocation(location) { - latitude, minLongitude, maxLongitude := archiveRange.coordinates() + for archiveRange := range archiveRangesForLocation(location) { + latitude, minLongitude, maxLongitude := archiveRange[0], archiveRange[1], archiveRange[2] for longitude := minLongitude; longitude < maxLongitude; longitude += GROUP_AREA_BOX_DEGREES { select { // nonblocking update of progress case d.progressChan <- d.progress: @@ -293,13 +315,19 @@ func (d *download) downloadLocation(location LocationData, locationName string) return nil, false } +func countFilesForBounds(bounds Bounds) int { + minLat, minLon, maxLat, maxLon := adjustedBounds(bounds) + return ((maxLat - minLat) / GROUP_AREA_BOX_DEGREES) * ((maxLon - minLon) / GROUP_AREA_BOX_DEGREES) +} + func countFilesForLocation(location LocationData) int { + if len(location.ArchiveRanges) == 0 { + return countFilesForBounds(location.BoundingBox) + } + totalFiles := 0 - for _, archiveRange := range archiveRangesForLocation(location) { - _, minLongitude, maxLongitude := archiveRange.coordinates() - for longitude := minLongitude; longitude < maxLongitude; longitude += GROUP_AREA_BOX_DEGREES { - totalFiles++ - } + for _, archiveRange := range location.ArchiveRanges { + totalFiles += (archiveRange[2] - archiveRange[1]) / GROUP_AREA_BOX_DEGREES } return totalFiles }