Repository navigation
Expand file tree
/
Copy pathplanner_verify.py
More file actions
266 lines (243 loc) · 13.2 KB
/
Copy pathplanner_verify.py
File metadata and controls
266 lines (243 loc) · 13.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
"""Verify stored planner grid artifacts without preparing data or runtimes."""
import argparse
import base64
from contextlib import contextmanager, nullcontext
import gzip
from itertools import accumulate
import json
from pathlib import Path
import struct
import subprocess
from . import (planner_grid_index as index, planner_grid_maps as grid_maps, planner_grid_offline as grid_offline,
planner_offline as offline, planner_runtime as runtime)
from .planner_grid_search import search_metadata
from .planner_map_archive import pixels
GEOMETRIES = ('point', 'line', 'area')
def features(data):
"""The features of a TrailView feature tile (specs/trailview-features.md): `kind`, `tags`, `geometry` and
`parts`. A part has `outer`, `xy` and, when the feature has heights, `z` in metres with None where a height is
not solved, and the codes `along`, `tangent`, `grade`, `bank`, `widen`, `side` and `walk` of its alignment when
it has one."""
position = 0
def read():
nonlocal position
value = shift = 0
while True:
byte = data[position]
position += 1
value |= (byte & 0x7F) << shift
shift += 7
if byte < 0x80:
return value
signed = lambda value: value >> 1 ^ -(value & 1)
strings = []
for _ in range(read()):
length = read()
strings.append(bytes(data[position:position + length]).decode())
position += length
heads = [(strings[read()], read()) for _ in range(read())]
decoded = []
for kind, header in heads:
tags = {}
for _ in range(read()):
key = strings[read()]
tags[key] = strings[read()]
decoded.append({'kind': kind, 'tags': tags, 'geometry': GEOMETRIES[header & 3], 'parts': []})
counts = []
for feature, (_, header) in zip(decoded, heads):
sizes = []
for _ in range(header >> 4):
count = read()
sizes.append(count >> 1)
feature['parts'].append({'outer': bool(count & 1), 'xy': [], **({'z': []} if header & 4 else {})})
counts.append(sizes)
for feature, sizes, (_, header) in zip(decoded, counts, heads):
x, y, height = 0, 0, 0
for part, size in zip(feature['parts'], sizes):
for _ in range(size):
x, y = x + signed(read()), y + signed(read())
part['xy'].append([x, y])
if header & 4:
code = read()
height += signed(code - 1) if code else 0
part['z'].append(height / 100 if code else None)
aligned = []
for feature, sizes, (_, header) in zip(decoded, counts, heads):
for part, size in zip(feature['parts'], sizes):
along = read() if header & 8 else 0
if along:
part['along'] = along - 1
aligned.append((part, size))
column = lambda: [[read() for _ in range(size)] for _, size in aligned]
tangents, grades, banks, words = column(), column(), column(), column()
for (part, _), tangent, grade, bank, word in zip(aligned, tangents, grades, banks, words):
part['tangent'] = [turn % 1024 for turn in accumulate(map(signed, tangent))]
part['grade'] = list(accumulate(map(signed, grade)))
part['bank'] = [signed(value) for value in bank]
part['widen'] = [value >> 4 for value in word]
part['side'] = [value & 3 for value in word]
part['walk'] = [value >> 2 & 3 for value in word]
if position != len(data):
raise ValueError('A feature tile has bytes after its last feature')
return decoded
@contextmanager
def vector_reader():
try:
process = subprocess.Popen(['node', str(Path(__file__).with_name('planner_mvt_verify.mjs'))],
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
except OSError as error:
raise ValueError('Prepare Node and the locked builder reader before MVT verification') from error
try:
if process.stdout.readline().strip() != 'ready':
raise ValueError(process.stderr.read().strip() or 'Prepared MVT reader failed')
def read(data):
process.stdin.write(json.dumps(base64.b64encode(data).decode()) + '\n')
process.stdin.flush()
if process.stdout.readline().strip() != 'ok':
raise ValueError(process.stderr.read().strip() or 'Prepared MVT reader failed')
yield read
finally:
process.stdin.close()
process.stdout.close()
process.stderr.close()
process.wait()
def size(data):
return struct.unpack('<II', pixels(data)[:8])
def images(data):
"""The images of a tile body, one after the other. A WebP image is 8 + the RIFF size in its bytes 4 to 7 long;
an image of another format takes the rest of the body."""
found = []
while data:
end = 8 + int.from_bytes(data[4:8], 'little') if data[:4] == b'RIFF' else len(data)
found.append(data[:end])
data = data[end:]
return found
def archive(path, kind, vector=None):
from pmtiles.reader import Reader, MmapSource, all_tiles
from pmtiles.tile import Compression, TileType
with path.open('rb') as stream:
read = MmapSource(stream)
reader = Reader(read)
header, metadata = reader.header(), reader.metadata()
colour = kind.startswith('trailview-colour-q')
expected = (TileType.UNKNOWN if kind in {'snow-seasons', 'climate', 'trailview-features', 'trailview-stamps'} or colour
else TileType.WEBP if kind in {'terrain', 'snow', 'snow-levels'} or kind.startswith('trailview-')
else TileType.MVT)
if header['tile_type'] != expected:
raise ValueError(f'Unexpected {kind} tile format')
if expected == TileType.MVT and vector is None:
with vector_reader() as vector:
return archive(path, kind, vector)
# The compression and the zooms of the snow archives: specs/planner-snow-tiles.md.
if kind in {'snow', 'snow-levels'} and header['tile_compression'] != Compression.NONE:
raise ValueError(f'Unexpected {kind} tile compression')
zooms = {'snow': (0, 11), 'snow-levels': (0, 11), 'snow-seasons': (2, 14)}.get(kind)
if zooms and not zooms[0] <= header['min_zoom'] <= header['max_zoom'] <= zooms[1]:
raise ValueError(f'Unexpected {kind} zoom')
count = 0
for (z, _x, _y), data in all_tiles(read):
if not header['min_zoom'] <= z <= header['max_zoom'] or not data:
raise ValueError('Invalid tile or zoom')
if header['tile_compression'] == Compression.GZIP:
data = gzip.decompress(data)
elif header['tile_compression'] != Compression.NONE:
raise ValueError('Unsupported tile compression')
if kind in {'snow', 'snow-levels'}:
# One run image, or four: specs/planner-snow-tiles.md.
if [size(image) for image in images(data)] != [(512, 512)] * (1 if kind == 'snow' else 4):
raise ValueError('Snow tile size differs')
elif expected == TileType.MVT:
vector(data)
elif kind == 'trailview-features':
features(data)
elif kind == 'snow-seasons':
seasons = metadata.get('seasons')
if type(seasons) is not int or seasons <= 0 or len(data) != 2 * seasons * 64 * 64:
raise ValueError('Snow tile size differs')
elif colour:
# The NDVI tile, then the colour: specs/trailview-tiles.md#colour.
if [size(image) for image in images(data)] != [(256, 256), (512, 512)]:
raise ValueError('Colour tile size differs')
elif kind == 'climate':
# The two fixed layouts of specs/planner-climate-tiles.md.
sizes = {8: 24 * 16 * (2 + 2 * 12 + 12 * 16 + 5 * 52),
9: 12 * 8 * (2 + 2 * 12 + 5 * 10 * 52)}
if len(data) != sizes.get(z):
raise ValueError('Climate tile size differs')
count += 1
if not count or count != header['addressed_tiles_count']:
raise ValueError('Archive tile count differs')
def verify(source):
document = json.loads((source / 'release.json').read_bytes())
indexes, named = {}, {}
for step in document['indexes']:
data = (source / 'indexes' / step / 'index.json').read_bytes()
step_index = json.loads(data)
if step_index['kind'] in indexes:
raise ValueError('Duplicate grid index')
indexes[step_index['kind']], named[step] = step_index, index.described(data, step_index)
found = {path.parent.relative_to(source / 'indexes').as_posix() for path in (source / 'indexes').rglob('index.json')}
if found != named.keys():
raise ValueError('The planner descriptor does not name each grid index')
options = {key: document[key] for key in ('region', 'name', 'bounds', 'attribution', 'landcover_attribution')}
files, release, _, search = index.compose(indexes, options)
grid = f"search/{document['region']}.grid.json"
if set(document['files']) != {grid} or document != index.document(release, named, document['files']):
raise ValueError('Planner descriptor differs from its grid indexes')
files = {**files, **document['files']}
previous = json.loads((source / 'previous.json').read_bytes())['files'] if (source / 'previous.json').exists() else None
changed = lambda name: previous is None or previous.get(name) != files[name]
routing_changed = any(changed(name) for name in files if name.startswith('routing/'))
selected = {name: entry for name, entry in files.items()
if changed(name) or name.endswith('.json') or routing_changed and name.startswith('routing/')}
if any(name.startswith('maps/tiles/') and name.endswith('.pmtiles') and entry['bytes'] > grid_maps.PACK_BYTES
for name, entry in files.items()):
raise ValueError(f'A map pack is larger than {grid_maps.PACK_BYTES} bytes')
public = runtime.public_metadata(files)
if {path.relative_to(source).as_posix() for path in (source / 'public').rglob('*') if path.is_file()} != public.keys():
raise ValueError('Public pointers differ from the planner descriptor')
for name, data in public.items():
if (source / name).read_bytes() != data:
raise ValueError('Public pointer identity differs')
output = source / 'runtime'
offline.materialize(source, output, {'files': selected}, ('',))
catalog, cells = grid_offline.compose({kind: item for kind, item in indexes.items() if kind != 'offline'}, options)
documents = {'offline/catalog.json': catalog, **{f'offline/cells/{name}.json': cell for name, cell in cells.items()}}
if (set(indexes['offline']['files']) != documents.keys()
or any(json.loads((output / name).read_bytes()) != item for name, item in documents.items())
or json.loads((output / grid).read_bytes()) != search):
raise ValueError('Offline or search catalog differs from its grid indexes')
graph = json.loads((output / 'routing/blocks.json').read_bytes())
if graph != indexes['routing']['graph'] or runtime.digest(output / 'routing/blocks.json') != document['routing_package']:
raise ValueError('Routing graph differs from the planner descriptor')
for cell in indexes['routing']['cells']:
path = output / 'offline' / cell['manifest']
descriptor = json.loads(path.read_bytes())
if (runtime.digest(path) != cell['sha256'] or descriptor['source'] != graph['source']
or descriptor['data']['bounds'] != cell['bounds'] or descriptor['archives'] != cell['archives']):
raise ValueError('Routing cell source or coverage differs')
for cell in search['cells']:
for name in cell['files']:
if not changed('search/' + name): continue
path = output / 'search' / name
metadata = search_metadata(path, full=True)
component = Path(name).parts[1]
if (metadata['component'] != component or metadata['bounds'] != cell['bounds']
or metadata['osm_sha256'] != document['osm_sha256']
or metadata['time_zone'] != search['metadata']['time_zone']):
raise ValueError('Search shard source, component or coverage differs')
needs_vector = any(name.endswith('.pmtiles') and changed(name) and Path(name).parts[2] in {'basemap', 'places', 'overlays'} for name in files)
with vector_reader() if needs_vector else nullcontext() as vector:
for name in files:
if name.startswith('maps/') and name.endswith('.json') and Path(name).stem in index.MAP_KINDS:
kind = Path(name).stem
if json.loads((output / name).read_bytes()) != indexes[kind]['metadata']:
raise ValueError('Map metadata differs from its grid index')
if name.endswith('.pmtiles') and changed(name):
archive(output / name, Path(name).parts[2], vector)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('source', type=Path)
verify(parser.parse_args().source)
if __name__ == '__main__':
main()