From a3349f924238e990b1f0c3d6962b5ac1a6b89841 Mon Sep 17 00:00:00 2001 From: Caglar Pir Date: Fri, 18 Sep 2026 16:23:44 +0200 Subject: [PATCH] Read a video's start time from its own clock, not creation time minus duration A video's start time decides where its sampled frames land on a GPS track. We derived it as creation_time - duration, which pushed every video one full duration into the past unless its camera happened to stamp the creation time at the end of the recording. That subtraction came from 644701c (2019), whose removed comment says why: "Blackvue actually reports endtime in the created_at exif field". The workaround was later generalised to every video. ISO/IEC 14496-12 defines creation_time as the creation time of the presentation, which for a recording is the moment it started, and that is what cameras and ffmpeg write. Take the start time from the video's own telemetry when it has some. That clock is absolute UTC, so it is right for BlackVue, whose container time is the end of the recording, and for GoPro, whose container time is in local time (#819). Fall back to creation_time for the plain MP4s that get geotagged from a GPX, which are exactly the videos that have no telemetry to sync against. Reported for a 10:49 dashcam video paired with a GPX starting at the video's creation time: every sampled frame came out 648.88s early, so the documented GPX workflow failed with MapillaryOutsideGPXTrackError on all of them. Where the track does reach back far enough -- a GPX covering a recording split into several files -- the frames were silently placed up to a kilometre off instead. With this change all 289 sampled frames land on the track exactly. --- README.md | 4 +- mapillary_tools/ffmpeg.py | 33 +++++----- mapillary_tools/sample_video.py | 54 +++++++++++++++- tests/integration/test_gopro.py | 16 +++-- tests/unit/test_ffmpeg.py | 10 +-- tests/unit/test_sample_video.py | 106 +++++++++++++++++++++++++++++--- 6 files changed, 182 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 6a439630..aa7bd509 100644 --- a/README.md +++ b/README.md @@ -284,7 +284,9 @@ It is used to locate the images along the GPS tracks. mapillary_tools process MY_IMAGE_DIR --geotag_source "gpx" --geotag_source_path MY_EXTERNAL_GPS.gpx ``` -To geotag videos with a GPX file, video start time (video creation time minus video duration) is required to locate the sample images along the GPS tracks. +To geotag videos with a GPX file, video start time is required to locate the sample images along the GPS tracks. +It is read from the video's own GPS track when it has one, and otherwise from the video creation time. +Use `--video_start_time` to override it for cameras that write neither correctly. ```sh # Geotagging with GPX works with interval-based sampling only, diff --git a/mapillary_tools/ffmpeg.py b/mapillary_tools/ffmpeg.py index 0264e2b2..e34d3fc0 100644 --- a/mapillary_tools/ffmpeg.py +++ b/mapillary_tools/ffmpeg.py @@ -609,9 +609,8 @@ def probe_video_start_time(self) -> datetime.datetime | None: """ Determine the start time of the video by analyzing stream metadata. - Searches for creation time and duration information in video streams first, - then falls back to other stream types. Calculates start time as: - creation_time - duration + Searches for a creation time in video streams first, then falls back to + other stream types. Returns: Video start time as datetime object, or None if cannot be determined @@ -673,34 +672,34 @@ def probe_video_with_max_resolution(self) -> Stream | None: @classmethod def extract_stream_start_time(cls, stream: Stream) -> datetime.datetime | None: """ - Calculate the start time of a specific stream. + Read the start time of a specific stream from its creation time. - Determines start time by subtracting stream duration from creation time: - start_time = creation_time - duration + ISO/IEC 14496-12 defines creation_time as the creation time of the + presentation, which for a recording is the moment it started, and that + is what cameras and ffmpeg write. We used to subtract the duration from + it, a workaround for BlackVue dashcams that really do stamp the time the + recording ended. That workaround put every other camera's video one full + duration into the past, so sampled frames landed that far back along the + GPS track. Cameras that stamp the end time embed GPS with absolute + timestamps, so callers sync against that clock instead (see + sample_video._extract_video_start_time). Args: - stream: Stream dictionary containing metadata including tags and duration + stream: Stream dictionary containing metadata including tags Returns: - Stream start time as datetime object, or None if required metadata is missing + Stream start time as datetime object, or None if the creation time is missing Note: Handles multiple datetime formats including ISO format and custom patterns. """ - duration_str = stream.get("duration") - LOG.debug("Extracted video duration: %s", duration_str) - if duration_str is None: - return None - duration = float(duration_str) - creation_time_str = stream.get("tags", {}).get("creation_time") LOG.debug("Extracted video creation time: %s", creation_time_str) if creation_time_str is None: return None try: - creation_time = datetime.datetime.fromisoformat(creation_time_str) + return datetime.datetime.fromisoformat(creation_time_str) except ValueError: - creation_time = datetime.datetime.strptime( + return datetime.datetime.strptime( creation_time_str, "%Y-%m-%dT%H:%M:%S.%f%z" ) - return creation_time - datetime.timedelta(seconds=duration) diff --git a/mapillary_tools/sample_video.py b/mapillary_tools/sample_video.py index 1e3e7764..db92a910 100644 --- a/mapillary_tools/sample_video.py +++ b/mapillary_tools/sample_video.py @@ -17,6 +17,7 @@ from . import constants, exceptions, ffmpeg as ffmpeglib, geo, types, utils from .exif_write import ExifEdit from .geotag import geotag_videos_from_video +from .geotag.video_extractors.native import NativeVideoExtractor from .mp4 import mp4_sample_parser from .serializer.description import parse_capture_time @@ -179,6 +180,49 @@ def wip_sample_dir(sample_dir: Path) -> Path: ) +def _gps_clock_start_time( + points: T.Sequence[geo.Point], +) -> datetime.datetime | None: + """ + Map the first absolute GPS timestamp in a track back to the video's time 0. + + Point times are relative to the start of the video, so subtracting one from + its own absolute timestamp gives the wall clock at which the video started. + """ + for point in points: + unix_time = point.get_unix_time() + if unix_time is not None: + return datetime.datetime.fromtimestamp( + unix_time - point.time, tz=datetime.timezone.utc + ) + + return None + + +def _extract_video_start_time( + video_path: Path, probe: ffmpeglib.Probe +) -> datetime.datetime | None: + """ + Determine the wall clock time at which a video started recording. + + A video's own telemetry is the better clock: it is absolute UTC, so it is + immune both to cameras that stamp the container's creation time at the end + of the recording (BlackVue) and to cameras that stamp it in local time + (GoPro). Fall back to the creation time when there is no telemetry to sync + against, which is the case for the plain MP4s that get geotagged from a GPX. + """ + try: + video_metadata = NativeVideoExtractor(video_path).extract() + except exceptions.MapillaryDescriptionError as ex: + LOG.debug("No video telemetry to read the start time from: %s", ex) + else: + start_time = _gps_clock_start_time(video_metadata.points) + if start_time is not None: + return start_time + + return probe.probe_video_start_time() + + def _sample_single_video_by_interval( video_path: Path, sample_dir: Path, @@ -189,9 +233,9 @@ def _sample_single_video_by_interval( ffmpeg = ffmpeglib.FFMPEG(constants.FFMPEG_PATH, constants.FFPROBE_PATH) if start_time is None: - start_time = ffmpeglib.Probe( - ffmpeg.probe_format_and_streams(video_path) - ).probe_video_start_time() + start_time = _extract_video_start_time( + video_path, ffmpeglib.Probe(ffmpeg.probe_format_and_streams(video_path)) + ) if start_time is None: raise exceptions.MapillaryVideoError( f"Unable to extract video start time from {video_path}" @@ -287,6 +331,10 @@ def _sample_single_video_by_distance( probe = ffmpeglib.Probe(ffmpeg.probe_format_and_streams(video_path)) if start_time is None: + # Unlike interval sampling, this is only a fallback for tracks whose + # points carry no absolute timestamp: the ones that do are timestamped + # from their own GPS clock below, so there is nothing to be gained by + # parsing the telemetry twice just to read that clock here start_time = probe.probe_video_start_time() if start_time is None: raise exceptions.MapillaryVideoError( diff --git a/tests/integration/test_gopro.py b/tests/integration/test_gopro.py index 199b780d..cc101aa9 100644 --- a/tests/integration/test_gopro.py +++ b/tests/integration/test_gopro.py @@ -31,12 +31,16 @@ "MAPILLARY_TOOLS_GOPRO_GPS_PRECISION": "10000000", "MAPILLARY_TOOLS_MAX_CAPTURE_SPEED_KMH": "2000000", # km/h } +# The capture times come from the GPMF GPS clock (first fix at +# 2019-11-18T23:42:08.645Z), not from the container's creation time. This camera +# writes the creation time in local time, so reading the start time from there +# would timestamp every sample 8 hours off. EXPECTED_DESCS: T.List[T.Any] = [ { "filename": "hero8.mp4/hero8_v_000001.jpg", "filetype": "image", "MAPAltitude": 9540.24, - "MAPCaptureTime": "2019_11_18_15_41_12_354", + "MAPCaptureTime": "2019_11_18_23_42_08_645", "MAPCompassHeading": { "TrueHeading": 123.93587938690177, "MagneticHeading": 123.93587938690177, @@ -51,7 +55,7 @@ "filename": "hero8.mp4/hero8_v_000002.jpg", "filetype": "image", "MAPAltitude": 7112.573717404068, - "MAPCaptureTime": "2019_11_18_15_41_14_354", + "MAPCaptureTime": "2019_11_18_23_42_10_645", "MAPCompassHeading": { "TrueHeading": 140.8665026186285, "MagneticHeading": 140.8665026186285, @@ -66,7 +70,7 @@ "filename": "hero8.mp4/hero8_v_000003.jpg", "filetype": "image", "MAPAltitude": 7463.642846094319, - "MAPCaptureTime": "2019_11_18_15_41_16_354", + "MAPCaptureTime": "2019_11_18_23_42_12_645", "MAPCompassHeading": { "TrueHeading": 138.44255851085705, "MagneticHeading": 138.44255851085705, @@ -81,7 +85,7 @@ "filename": "hero8.mp4/hero8_v_000004.jpg", "filetype": "image", "MAPAltitude": 6909.8168472111465, - "MAPCaptureTime": "2019_11_18_15_41_18_354", + "MAPCaptureTime": "2019_11_18_23_42_14_645", "MAPCompassHeading": { "TrueHeading": 142.23462669862568, "MagneticHeading": 142.23462669862568, @@ -96,7 +100,7 @@ "filename": "hero8.mp4/hero8_v_000005.jpg", "filetype": "image", "MAPAltitude": 7212.594480737465, - "MAPCaptureTime": "2019_11_18_15_41_20_354", + "MAPCaptureTime": "2019_11_18_23_42_16_645", "MAPCompassHeading": { "TrueHeading": 164.70819093235514, "MagneticHeading": 164.70819093235514, @@ -111,7 +115,7 @@ "filename": "hero8.mp4/hero8_v_000006.jpg", "filetype": "image", "MAPAltitude": 7274.361994963208, - "MAPCaptureTime": "2019_11_18_15_41_22_354", + "MAPCaptureTime": "2019_11_18_23_42_18_645", "MAPCompassHeading": { "TrueHeading": 139.71549328876722, "MagneticHeading": 139.71549328876722, diff --git a/tests/unit/test_ffmpeg.py b/tests/unit/test_ffmpeg.py index c79fc8ce..7a5d9cc6 100644 --- a/tests/unit/test_ffmpeg.py +++ b/tests/unit/test_ffmpeg.py @@ -158,7 +158,7 @@ def test_probe_format_and_streams_gopro_ok(setup_data: py.path.local): start_time = probe.probe_video_start_time() assert start_time is not None - assert datetime.datetime.isoformat(start_time) == "2019-11-18T15:41:12.354033+00:00" + assert datetime.datetime.isoformat(start_time) == "2019-11-18T15:41:25+00:00" max_stream = probe.probe_video_with_max_resolution() assert max_stream is not None assert max_stream["index"] == 0 @@ -248,18 +248,20 @@ def test_creation_time(expected, probe_creation_time, probe_duration): creation_time = probe.probe_video_start_time() assert expected == creation_time + # The creation time is the start of the recording, so the duration is not + # subtracted from it test_creation_time( - datetime.datetime(2023, 3, 7, 1, 35, 29, 190123, tzinfo=datetime.timezone.utc), + datetime.datetime(2023, 3, 7, 1, 35, 34, 123456, tzinfo=datetime.timezone.utc), "2023-03-07T01:35:34.123456Z", "4.933333", ) test_creation_time( - datetime.datetime(2023, 3, 7, 1, 35, 29, 66667, tzinfo=datetime.timezone.utc), + datetime.datetime(2023, 3, 7, 1, 35, 34, tzinfo=datetime.timezone.utc), "2023-03-07T01:35:34.000000Z", "4.933333", ) test_creation_time( - datetime.datetime(2023, 3, 7, 1, 35, 29, 66667), + datetime.datetime(2023, 3, 7, 1, 35, 34), "2023-03-07 01:35:34", "4.933333", ) diff --git a/tests/unit/test_sample_video.py b/tests/unit/test_sample_video.py index 0743eeb4..838744a5 100644 --- a/tests/unit/test_sample_video.py +++ b/tests/unit/test_sample_video.py @@ -21,6 +21,7 @@ ffmpeg as ffmpeglib, geo, sample_video, + telemetry, ) from mapillary_tools.mp4 import mp4_sample_parser from mapillary_tools.serializer import description @@ -28,6 +29,12 @@ _PWD = Path(os.path.dirname(os.path.abspath(__file__))) +# The creation time of the hello.mp4 probe fixture, which is where videos +# without their own GPS clock get their start time from +PROBE_START_TIME = datetime.datetime( + 2021, 8, 10, 14, 38, 6, tzinfo=datetime.timezone.utc +) + # --------------------------------------------------------------------------- # Interval-based sampling tests (using MOCK_FFMPEG) @@ -85,8 +92,7 @@ def test_sample_video(tmpdir: py.path.local, setup_mock): rerun=True, ) samples = sample_dir.join("hello.mp4").listdir() - video_start_time = description.parse_capture_time("2021_08_10_14_37_05_023") - _validate_interval([Path(s) for s in samples], video_start_time) + _validate_interval([Path(s) for s in samples], PROBE_START_TIME) def test_sample_single_video(tmpdir: py.path.local, setup_mock): @@ -101,8 +107,7 @@ def test_sample_single_video(tmpdir: py.path.local, setup_mock): rerun=True, ) samples = sample_dir.join("hello.mp4").listdir() - video_start_time = description.parse_capture_time("2021_08_10_14_37_05_023") - _validate_interval([Path(s) for s in samples], video_start_time) + _validate_interval([Path(s) for s in samples], PROBE_START_TIME) def test_sample_video_with_start_time(tmpdir: py.path.local, setup_mock): @@ -123,6 +128,93 @@ def test_sample_video_with_start_time(tmpdir: py.path.local, setup_mock): _validate_interval([Path(s) for s in samples], video_start_time) +def test_sample_video_from_gps_clock(tmpdir: py.path.local, setup_mock, monkeypatch): + """A video's own GPS clock wins over the container's creation time.""" + root = _PWD.joinpath("data/mock_sample_video") + video_dir = root.joinpath("videos") + sample_dir = tmpdir.mkdir("sampled_video_frames") + + # A camera that stamps the creation time at the end of the recording, or in + # local time, still has a correct absolute clock in its telemetry + gps_start_time = datetime.datetime( + 2021, 8, 10, 6, 38, 6, tzinfo=datetime.timezone.utc + ) + points = [ + telemetry.GPSPoint( + time=float(i), + lat=40.0 + i * 0.001, + lon=-74.0, + alt=None, + angle=None, + epoch_time=gps_start_time.timestamp() + i, + fix=None, + precision=None, + ground_speed=None, + ) + for i in range(3) + ] + monkeypatch.setattr( + sample_video, + "NativeVideoExtractor", + lambda video_path: mock.Mock( + extract=lambda: VideoMetadata( + filename=video_path, + filetype=FileType.BLACKVUE, + points=T.cast(T.List[geo.Point], points), + ) + ), + ) + + sample_video.sample_video( + video_dir, + Path(sample_dir), + video_sample_distance=-1, + video_sample_interval=2, + rerun=True, + ) + + samples = sample_dir.join("hello.mp4").listdir() + _validate_interval([Path(s) for s in samples], gps_start_time) + + +class TestGPSClockStartTime: + """Tests for _gps_clock_start_time.""" + + @staticmethod + def _gps_point(time: float, epoch_time: float | None) -> telemetry.GPSPoint: + return telemetry.GPSPoint( + time=time, + lat=40.0, + lon=-74.0, + alt=None, + angle=None, + epoch_time=epoch_time, + fix=None, + precision=None, + ground_speed=None, + ) + + def test_maps_first_timestamp_back_to_video_start(self) -> None: + # The first point is 2.5s into the video, so the video started 2.5s + # before that point was recorded + points = [self._gps_point(2.5, 1628599086.0)] + assert sample_video._gps_clock_start_time(points) == datetime.datetime( + 2021, 8, 10, 12, 38, 3, 500000, tzinfo=datetime.timezone.utc + ) + + def test_skips_points_without_a_timestamp(self) -> None: + points = [self._gps_point(0.0, None), self._gps_point(1.0, 1628599086.0)] + assert sample_video._gps_clock_start_time(points) == datetime.datetime( + 2021, 8, 10, 12, 38, 5, tzinfo=datetime.timezone.utc + ) + + def test_no_absolute_timestamps(self) -> None: + assert sample_video._gps_clock_start_time(_make_gps_points(3)) is None + + def test_no_points(self) -> None: + assert sample_video._gps_clock_start_time([]) is None + + # --------------------------------------------------------------------------- # Helpers for distance-based sampling tests # --------------------------------------------------------------------------- @@ -130,12 +222,6 @@ def test_sample_video_with_start_time(tmpdir: py.path.local, setup_mock): MOCK_PROBE_JSON = _PWD / "data" / "mock_sample_video" / "videos" / "hello.mp4" TEST_EXIF_JPG = _PWD / "data" / "test_exif.jpg" -# Start time derived from the hello.mp4 probe fixture: -# creation_time "2021-08-10T14:38:06.000000Z" - duration "60.977000" -PROBE_START_TIME = datetime.datetime( - 2021, 8, 10, 14, 36, 55, 23000, tzinfo=datetime.timezone.utc -) - def _load_probe_output() -> ffmpeglib.ProbeOutput: with open(MOCK_PROBE_JSON) as fp: