Repository navigation
Expand file tree
/
Copy pathplanner_offline.py
More file actions
79 lines (67 loc) · 3.26 KB
/
Copy pathplanner_offline.py
File metadata and controls
79 lines (67 loc) · 3.26 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
"""Publish immutable planner objects and materialize runtime files from the object pool."""
import gzip
import os
from pathlib import Path
import shutil
import tempfile
from . import planner_runtime as runtime
COMPRESSED = {".bin", ".pmtiles", ".webp", ".png"}
CHUNK = 1024 * 1024
def verify(path, item):
if path.stat().st_size != item["bytes"] or runtime.digest(path) != item["sha256"]:
raise ValueError(f"Checksum mismatch: {path.name}")
def item(path):
return {"bytes": path.stat().st_size, "sha256": runtime.digest(path)}
def pack_file(source, objects):
"""Publish one immutable object. Compression runs only during publication."""
original = item(source)
objects.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(prefix=".pack-", dir=objects.parent) as temporary:
compressed = Path(temporary) / "object"
if source.suffix not in COMPRESSED:
with source.open("rb") as reader, compressed.open("wb") as target:
with gzip.GzipFile(filename="", fileobj=target, mode="wb", compresslevel=6, mtime=0) as encoder:
shutil.copyfileobj(reader, encoder, CHUNK)
if compressed.exists() and compressed.stat().st_size < original["bytes"]:
transport = {**item(compressed), "encoding": "gzip"}
target = objects / transport["sha256"]
if not target.exists(): compressed.rename(target)
else:
transport = {**original, "encoding": "identity"}
target = objects / original["sha256"]
if not target.exists():
try: os.link(source, target)
except OSError: shutil.copyfile(source, target)
verify(target, transport)
with target.open("rb") as stream: os.fsync(stream.fileno())
return {**original, "transport": transport}
def materialize(source, destination, document, prefixes):
"""Install the runtime files under `prefixes` from the object pool of a verified grid release."""
for name, entry in document["files"].items():
if not name.startswith(prefixes): continue
path = destination / name
if path.exists():
verify(path, entry)
continue
path.parent.mkdir(parents=True, exist_ok=True)
transport = entry["transport"]
original = source / "objects" / transport["sha256"]
verify(original, transport)
temporary = path.with_suffix(path.suffix + ".partial")
try:
if transport["encoding"] == "identity":
try: os.link(original, temporary)
except OSError: shutil.copyfile(original, temporary)
elif transport["encoding"] == "gzip":
with gzip.open(original, "rb") as reader, temporary.open("wb") as writer:
remaining = entry["bytes"]
while chunk := reader.read(min(CHUNK, remaining + 1)):
remaining -= len(chunk)
if remaining < 0: raise ValueError("Decoded object exceeds the release size")
writer.write(chunk)
else:
raise ValueError("Unsupported object encoding")
verify(temporary, entry)
temporary.replace(path)
finally:
temporary.unlink(missing_ok=True)