Repository navigation
Expand file tree
/
Copy pathplanner_grid_maps.py
More file actions
186 lines (162 loc) · 8.36 KB
/
Copy pathplanner_grid_maps.py
File metadata and controls
186 lines (162 loc) · 8.36 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
"""Partition a planner map into immutable PMTiles grid archives."""
from collections import OrderedDict
from itertools import islice
import argparse
import json
from pathlib import Path
import struct
import tempfile
from . import planner_offline as offline, planner_runtime as runtime, step_pool, step_request
from .planner_map_archive import empty_mbtiles, lossless
# The largest pack file: specs/planner-offline.md#grid-publication-and-selection gives the reason.
PACK_BYTES = 15 * 1024 * 1024
# Only TrailView reads terrain of this zoom: its own archives keep it out of the planner and the offline areas.
FINE_ZOOM = 13
def recoded(tiles):
"""Terrain tiles encoded again by libwebp: the same pixels in fewer bytes than `planner-dem` writes. Pillow holds
the GIL while it encodes, so the tiles go to processes."""
workers = step_pool.worker_count()
with step_pool.pool(workers) as pool:
while batch := list(islice(tiles, 4 * workers)):
yield from zip((zxy for zxy, _ in batch), pool.map(smaller, (data for _, data in batch)))
def smaller(data):
return lossless(data)[0]
def staged(path):
"""The (tile ID, bytes) records of a staging file, in the order they were written."""
with path.open("rb") as records:
while record := records.read(12):
tile_id, length = struct.unpack("<QI", record)
data = records.read(length)
if len(data) != length: raise ValueError("Incomplete tile staging file")
yield tile_id, data
def stage(stream, tile_id, data):
stream.write(struct.pack("<QI", tile_id, len(data)))
stream.write(data)
def split(path):
"""Moves the tiles deeper than the zoom of the pack staged at `path` into the staging files of its four
squares one zoom deeper, and keeps its one tile at its own zoom, if it has one, at `path`."""
from pmtiles.tile import tileid_to_zxy
level = int(path.name.split("-")[0])
own, children = None, {}
try:
for count, (tile_id, data) in enumerate(staged(path), 1):
z, x, y = tileid_to_zxy(tile_id)
if z == level:
own = tile_id, data
continue
child = path.with_name(f"{level + 1}-{x >> (z - level - 1)}-{y >> (z - level - 1)}")
if child not in children: children[child] = child.open("ab")
stage(children[child], tile_id, data)
finally:
for stream in children.values(): stream.close()
if count == 1: raise ValueError(f"The pack {path.name} holds one tile of more than {PACK_BYTES} bytes")
path.unlink()
if own:
with path.open("ab") as stream: stage(stream, *own)
return [path, *children]
return list(children)
def map_tiles(source, output, kinds):
"""Writes the tiles of each archive into packs: a pack holds the tiles of one square from its zoom down to
the end of the level range of its base level. A pack file of more than `PACK_BYTES` splits into its one
tile at its own zoom and the packs of its four squares one zoom deeper."""
from pmtiles.reader import Reader, MmapSource, all_tiles
from pmtiles.tile import zxy_to_tileid
from pmtiles.writer import Writer
output.mkdir(parents=True, exist_ok=True)
for kind in kinds:
target = output / kind
target.mkdir(exist_ok=True)
with tempfile.TemporaryDirectory(prefix=".tiles-", dir=output) as temporary:
cache = OrderedDict()
try:
with (source / f"{kind}.pmtiles").open("rb") as file:
read = MmapSource(file)
reader = Reader(read)
header, metadata = reader.header(), reader.metadata()
terrain = kind == "terrain"
for (z, x, y), data in recoded(all_tiles(read)) if terrain else all_tiles(read):
level = runtime.base_level(z)
name = f"{level}-{x >> (z-level)}-{y >> (z-level)}"
if terrain and z >= FINE_ZOOM: name = f"{FINE_ZOOM}/{name}"
stream = cache.pop(name, None)
if stream is None:
(Path(temporary) / name).parent.mkdir(exist_ok=True)
stream = (Path(temporary) / name).open("ab")
stage(stream, zxy_to_tileid(z, x, y), data)
cache[name] = stream
if len(cache) > 16: cache.popitem(last=False)[1].close()
finally:
for stream in cache.values(): stream.close()
pending = [path for path in Path(temporary).rglob("*") if path.is_file()]
while pending:
path = pending.pop()
archive = target / f"{path.relative_to(temporary)}.pmtiles"
archive.parent.mkdir(exist_ok=True)
with archive.open("wb") as destination:
writer = Writer(destination)
try:
for tile_id, data in staged(path): writer.write_tile(tile_id, data)
writer.finalize(dict(header), dict(metadata))
finally:
writer.tile_f.close()
if archive.stat().st_size > PACK_BYTES:
archive.unlink()
pending += split(path)
else:
path.unlink()
def step(request):
from pmtiles.convert import mbtiles_to_pmtiles
from pmtiles.reader import Reader, MmapSource
kind = request["options"]["kind"]
output = Path(request["output"])
# A step that writes more than one archive names itself; its grid selects one archive.
layer = request["layers"][request["options"].get("step", f"planner/{kind}")]
if len(layer) != 1:
raise ValueError("A map grid reads one archive")
name, source = next(iter(layer.items()))
with tempfile.TemporaryDirectory(dir=output.parent) as directory:
work = Path(directory)
archives = work / "archives"
archives.mkdir()
archive = archives / f"{kind}.pmtiles"
# The PMTiles writer requires at least one tile; sea-only terrain has none.
empty = empty_mbtiles(source) if name.endswith(".mbtiles") else None
if empty:
header, metadata = empty
elif name.endswith(".mbtiles"):
mbtiles_to_pmtiles(source, archive, None)
else:
archive.symlink_to(Path(source).absolute())
if not empty:
map_tiles(archives, work / "tiles", [kind])
with archive.open("rb") as stream:
reader = Reader(MmapSource(stream))
header, metadata = reader.header(), reader.metadata()
objects = output / "objects"
tiles = work / "tiles" / kind
files = {f"maps/tiles/{kind}/{path.relative_to(tiles).as_posix()}": offline.pack_file(path, objects)
for path in sorted(tiles.rglob("*.pmtiles"))}
metadata = {**metadata, "tilejson": "3.0.0", "minzoom": header["min_zoom"],
"maxzoom": header["max_zoom"], "bounds": request["options"]["bounds"]}
if any(name.startswith(f"maps/tiles/terrain/{FINE_ZOOM}/") for name in files):
# The tile service and an offline area stop at zoom 12; TrailView reads the zoom 13 packs.
metadata.update(maxzoom=FINE_ZOOM - 1, trailview_maxzoom=FINE_ZOOM)
if kind == "overlays":
routing = json.loads(Path(request["layers"]["planner/routing/grid"]["index.json"]).read_bytes())
if metadata.get("routing_package") != routing["graph"]["source"]:
raise ValueError("Overlay tiles use another routing package")
metadata["routing_package"] = routing["files"]["routing/blocks.json"]["sha256"]
tilejson = work / f"{kind}.json"
tilejson.write_bytes(runtime.encoded(metadata))
files[f"maps/{kind}.json"] = offline.pack_file(tilejson, objects)
index = {"format": 1, "kind": kind, "files": files,
"source": offline.item(Path(source) if empty else archive), "metadata": metadata}
(output / "index.json").write_bytes(runtime.encoded(index))
step_request.metrics(request, {"archives": len(files) - 1})
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--step", action="store_true", required=True)
parser.parse_args()
step(step_request.read())
if __name__ == "__main__":
main()