diff --git a/README.md b/README.md index 6a439630..aa7bd509 100644 --- a/README.md +++ b/README.md @@ -284,7 +284,9 @@ It is used to locate the images along the GPS tracks. mapillary_tools process MY_IMAGE_DIR --geotag_source "gpx" --geotag_source_path MY_EXTERNAL_GPS.gpx ``` -To geotag videos with a GPX file, video start time (video creation time minus video duration) is required to locate the sample images along the GPS tracks. +To geotag videos with a GPX file, video start time is required to locate the sample images along the GPS tracks. +It is read from the video's own GPS track when it has one, and otherwise from the video creation time. +Use `--video_start_time` to override it for cameras that write neither correctly. ```sh # Geotagging with GPX works with interval-based sampling only, diff --git a/mapillary_tools/ffmpeg.py b/mapillary_tools/ffmpeg.py index 0264e2b2..e34d3fc0 100644 --- a/mapillary_tools/ffmpeg.py +++ b/mapillary_tools/ffmpeg.py @@ -609,9 +609,8 @@ def probe_video_start_time(self) -> datetime.datetime | None: """ Determine the start time of the video by analyzing stream metadata. - Searches for creation time and duration information in video streams first, - then falls back to other stream types. Calculates start time as: - creation_time - duration + Searches for a creation time in video streams first, then falls back to + other stream types. Returns: Video start time as datetime object, or None if cannot be determined @@ -673,34 +672,34 @@ def probe_video_with_max_resolution(self) -> Stream | None: @classmethod def extract_stream_start_time(cls, stream: Stream) -> datetime.datetime | None: """ - Calculate the start time of a specific stream. + Read the start time of a specific stream from its creation time. - Determines start time by subtracting stream duration from creation time: - start_time = creation_time - duration + ISO/IEC 14496-12 defines creation_time as the creation time of the + presentation, which for a recording is the moment it started, and that + is what cameras and ffmpeg write. We used to subtract the duration from + it, a workaround for BlackVue dashcams that really do stamp the time the + recording ended. That workaround put every other camera's video one full + duration into the past, so sampled frames landed that far back along the + GPS track. Cameras that stamp the end time embed GPS with absolute + timestamps, so callers sync against that clock instead (see + sample_video._extract_video_start_time). Args: - stream: Stream dictionary containing metadata including tags and duration + stream: Stream dictionary containing metadata including tags Returns: - Stream start time as datetime object, or None if required metadata is missing + Stream start time as datetime object, or None if the creation time is missing Note: Handles multiple datetime formats including ISO format and custom patterns. """ - duration_str = stream.get("duration") - LOG.debug("Extracted video duration: %s", duration_str) - if duration_str is None: - return None - duration = float(duration_str) - creation_time_str = stream.get("tags", {}).get("creation_time") LOG.debug("Extracted video creation time: %s", creation_time_str) if creation_time_str is None: return None try: - creation_time = datetime.datetime.fromisoformat(creation_time_str) + return datetime.datetime.fromisoformat(creation_time_str) except ValueError: - creation_time = datetime.datetime.strptime( + return datetime.datetime.strptime( creation_time_str, "%Y-%m-%dT%H:%M:%S.%f%z" ) - return creation_time - datetime.timedelta(seconds=duration) diff --git a/mapillary_tools/sample_video.py b/mapillary_tools/sample_video.py index 1e3e7764..db92a910 100644 --- a/mapillary_tools/sample_video.py +++ b/mapillary_tools/sample_video.py @@ -17,6 +17,7 @@ from . import constants, exceptions, ffmpeg as ffmpeglib, geo, types, utils from .exif_write import ExifEdit from .geotag import geotag_videos_from_video +from .geotag.video_extractors.native import NativeVideoExtractor from .mp4 import mp4_sample_parser from .serializer.description import parse_capture_time @@ -179,6 +180,49 @@ def wip_sample_dir(sample_dir: Path) -> Path: ) +def _gps_clock_start_time( + points: T.Sequence[geo.Point], +) -> datetime.datetime | None: + """ + Map the first absolute GPS timestamp in a track back to the video's time 0. + + Point times are relative to the start of the video, so subtracting one from + its own absolute timestamp gives the wall clock at which the video started. + """ + for point in points: + unix_time = point.get_unix_time() + if unix_time is not None: + return datetime.datetime.fromtimestamp( + unix_time - point.time, tz=datetime.timezone.utc + ) + + return None + + +def _extract_video_start_time( + video_path: Path, probe: ffmpeglib.Probe +) -> datetime.datetime | None: + """ + Determine the wall clock time at which a video started recording. + + A video's own telemetry is the better clock: it is absolute UTC, so it is + immune both to cameras that stamp the container's creation time at the end + of the recording (BlackVue) and to cameras that stamp it in local time + (GoPro). Fall back to the creation time when there is no telemetry to sync + against, which is the case for the plain MP4s that get geotagged from a GPX. + """ + try: + video_metadata = NativeVideoExtractor(video_path).extract() + except exceptions.MapillaryDescriptionError as ex: + LOG.debug("No video telemetry to read the start time from: %s", ex) + else: + start_time = _gps_clock_start_time(video_metadata.points) + if start_time is not None: + return start_time + + return probe.probe_video_start_time() + + def _sample_single_video_by_interval( video_path: Path, sample_dir: Path, @@ -189,9 +233,9 @@ def _sample_single_video_by_interval( ffmpeg = ffmpeglib.FFMPEG(constants.FFMPEG_PATH, constants.FFPROBE_PATH) if start_time is None: - start_time = ffmpeglib.Probe( - ffmpeg.probe_format_and_streams(video_path) - ).probe_video_start_time() + start_time = _extract_video_start_time( + video_path, ffmpeglib.Probe(ffmpeg.probe_format_and_streams(video_path)) + ) if start_time is None: raise exceptions.MapillaryVideoError( f"Unable to extract video start time from {video_path}" @@ -287,6 +331,10 @@ def _sample_single_video_by_distance( probe = ffmpeglib.Probe(ffmpeg.probe_format_and_streams(video_path)) if start_time is None: + # Unlike interval sampling, this is only a fallback for tracks whose + # points carry no absolute timestamp: the ones that do are timestamped + # from their own GPS clock below, so there is nothing to be gained by + # parsing the telemetry twice just to read that clock here start_time = probe.probe_video_start_time() if start_time is None: raise exceptions.MapillaryVideoError( diff --git a/tests/integration/test_gopro.py b/tests/integration/test_gopro.py index 199b780d..cc101aa9 100644 --- a/tests/integration/test_gopro.py +++ b/tests/integration/test_gopro.py @@ -31,12 +31,16 @@ "MAPILLARY_TOOLS_GOPRO_GPS_PRECISION": "10000000", "MAPILLARY_TOOLS_MAX_CAPTURE_SPEED_KMH": "2000000", # km/h } +# The capture times come from the GPMF GPS clock (first fix at +# 2019-11-18T23:42:08.645Z), not from the container's creation time. This camera +# writes the creation time in local time, so reading the start time from there +# would timestamp every sample 8 hours off. EXPECTED_DESCS: T.List[T.Any] = [ { "filename": "hero8.mp4/hero8_v_000001.jpg", "filetype": "image", "MAPAltitude": 9540.24, - "MAPCaptureTime": "2019_11_18_15_41_12_354", + "MAPCaptureTime": "2019_11_18_23_42_08_645", "MAPCompassHeading": { "TrueHeading": 123.93587938690177, "MagneticHeading": 123.93587938690177, @@ -51,7 +55,7 @@ "filename": "hero8.mp4/hero8_v_000002.jpg", "filetype": "image", "MAPAltitude": 7112.573717404068, - "MAPCaptureTime": "2019_11_18_15_41_14_354", + "MAPCaptureTime": "2019_11_18_23_42_10_645", "MAPCompassHeading": { "TrueHeading": 140.8665026186285, "MagneticHeading": 140.8665026186285, @@ -66,7 +70,7 @@ "filename": "hero8.mp4/hero8_v_000003.jpg", "filetype": "image", "MAPAltitude": 7463.642846094319, - "MAPCaptureTime": "2019_11_18_15_41_16_354", + "MAPCaptureTime": "2019_11_18_23_42_12_645", "MAPCompassHeading": { "TrueHeading": 138.44255851085705, "MagneticHeading": 138.44255851085705, @@ -81,7 +85,7 @@ "filename": "hero8.mp4/hero8_v_000004.jpg", "filetype": "image", "MAPAltitude": 6909.8168472111465, - "MAPCaptureTime": "2019_11_18_15_41_18_354", + "MAPCaptureTime": "2019_11_18_23_42_14_645", "MAPCompassHeading": { "TrueHeading": 142.23462669862568, "MagneticHeading": 142.23462669862568, @@ -96,7 +100,7 @@ "filename": "hero8.mp4/hero8_v_000005.jpg", "filetype": "image", "MAPAltitude": 7212.594480737465, - "MAPCaptureTime": "2019_11_18_15_41_20_354", + "MAPCaptureTime": "2019_11_18_23_42_16_645", "MAPCompassHeading": { "TrueHeading": 164.70819093235514, "MagneticHeading": 164.70819093235514, @@ -111,7 +115,7 @@ "filename": "hero8.mp4/hero8_v_000006.jpg", "filetype": "image", "MAPAltitude": 7274.361994963208, - "MAPCaptureTime": "2019_11_18_15_41_22_354", + "MAPCaptureTime": "2019_11_18_23_42_18_645", "MAPCompassHeading": { "TrueHeading": 139.71549328876722, "MagneticHeading": 139.71549328876722, diff --git a/tests/unit/test_ffmpeg.py b/tests/unit/test_ffmpeg.py index c79fc8ce..7a5d9cc6 100644 --- a/tests/unit/test_ffmpeg.py +++ b/tests/unit/test_ffmpeg.py @@ -158,7 +158,7 @@ def test_probe_format_and_streams_gopro_ok(setup_data: py.path.local): start_time = probe.probe_video_start_time() assert start_time is not None - assert datetime.datetime.isoformat(start_time) == "2019-11-18T15:41:12.354033+00:00" + assert datetime.datetime.isoformat(start_time) == "2019-11-18T15:41:25+00:00" max_stream = probe.probe_video_with_max_resolution() assert max_stream is not None assert max_stream["index"] == 0 @@ -248,18 +248,20 @@ def test_creation_time(expected, probe_creation_time, probe_duration): creation_time = probe.probe_video_start_time() assert expected == creation_time + # The creation time is the start of the recording, so the duration is not + # subtracted from it test_creation_time( - datetime.datetime(2023, 3, 7, 1, 35, 29, 190123, tzinfo=datetime.timezone.utc), + datetime.datetime(2023, 3, 7, 1, 35, 34, 123456, tzinfo=datetime.timezone.utc), "2023-03-07T01:35:34.123456Z", "4.933333", ) test_creation_time( - datetime.datetime(2023, 3, 7, 1, 35, 29, 66667, tzinfo=datetime.timezone.utc), + datetime.datetime(2023, 3, 7, 1, 35, 34, tzinfo=datetime.timezone.utc), "2023-03-07T01:35:34.000000Z", "4.933333", ) test_creation_time( - datetime.datetime(2023, 3, 7, 1, 35, 29, 66667), + datetime.datetime(2023, 3, 7, 1, 35, 34), "2023-03-07 01:35:34", "4.933333", ) diff --git a/tests/unit/test_sample_video.py b/tests/unit/test_sample_video.py index 0743eeb4..838744a5 100644 --- a/tests/unit/test_sample_video.py +++ b/tests/unit/test_sample_video.py @@ -21,6 +21,7 @@ ffmpeg as ffmpeglib, geo, sample_video, + telemetry, ) from mapillary_tools.mp4 import mp4_sample_parser from mapillary_tools.serializer import description @@ -28,6 +29,12 @@ _PWD = Path(os.path.dirname(os.path.abspath(__file__))) +# The creation time of the hello.mp4 probe fixture, which is where videos +# without their own GPS clock get their start time from +PROBE_START_TIME = datetime.datetime( + 2021, 8, 10, 14, 38, 6, tzinfo=datetime.timezone.utc +) + # --------------------------------------------------------------------------- # Interval-based sampling tests (using MOCK_FFMPEG) @@ -85,8 +92,7 @@ def test_sample_video(tmpdir: py.path.local, setup_mock): rerun=True, ) samples = sample_dir.join("hello.mp4").listdir() - video_start_time = description.parse_capture_time("2021_08_10_14_37_05_023") - _validate_interval([Path(s) for s in samples], video_start_time) + _validate_interval([Path(s) for s in samples], PROBE_START_TIME) def test_sample_single_video(tmpdir: py.path.local, setup_mock): @@ -101,8 +107,7 @@ def test_sample_single_video(tmpdir: py.path.local, setup_mock): rerun=True, ) samples = sample_dir.join("hello.mp4").listdir() - video_start_time = description.parse_capture_time("2021_08_10_14_37_05_023") - _validate_interval([Path(s) for s in samples], video_start_time) + _validate_interval([Path(s) for s in samples], PROBE_START_TIME) def test_sample_video_with_start_time(tmpdir: py.path.local, setup_mock): @@ -123,6 +128,93 @@ def test_sample_video_with_start_time(tmpdir: py.path.local, setup_mock): _validate_interval([Path(s) for s in samples], video_start_time) +def test_sample_video_from_gps_clock(tmpdir: py.path.local, setup_mock, monkeypatch): + """A video's own GPS clock wins over the container's creation time.""" + root = _PWD.joinpath("data/mock_sample_video") + video_dir = root.joinpath("videos") + sample_dir = tmpdir.mkdir("sampled_video_frames") + + # A camera that stamps the creation time at the end of the recording, or in + # local time, still has a correct absolute clock in its telemetry + gps_start_time = datetime.datetime( + 2021, 8, 10, 6, 38, 6, tzinfo=datetime.timezone.utc + ) + points = [ + telemetry.GPSPoint( + time=float(i), + lat=40.0 + i * 0.001, + lon=-74.0, + alt=None, + angle=None, + epoch_time=gps_start_time.timestamp() + i, + fix=None, + precision=None, + ground_speed=None, + ) + for i in range(3) + ] + monkeypatch.setattr( + sample_video, + "NativeVideoExtractor", + lambda video_path: mock.Mock( + extract=lambda: VideoMetadata( + filename=video_path, + filetype=FileType.BLACKVUE, + points=T.cast(T.List[geo.Point], points), + ) + ), + ) + + sample_video.sample_video( + video_dir, + Path(sample_dir), + video_sample_distance=-1, + video_sample_interval=2, + rerun=True, + ) + + samples = sample_dir.join("hello.mp4").listdir() + _validate_interval([Path(s) for s in samples], gps_start_time) + + +class TestGPSClockStartTime: + """Tests for _gps_clock_start_time.""" + + @staticmethod + def _gps_point(time: float, epoch_time: float | None) -> telemetry.GPSPoint: + return telemetry.GPSPoint( + time=time, + lat=40.0, + lon=-74.0, + alt=None, + angle=None, + epoch_time=epoch_time, + fix=None, + precision=None, + ground_speed=None, + ) + + def test_maps_first_timestamp_back_to_video_start(self) -> None: + # The first point is 2.5s into the video, so the video started 2.5s + # before that point was recorded + points = [self._gps_point(2.5, 1628599086.0)] + assert sample_video._gps_clock_start_time(points) == datetime.datetime( + 2021, 8, 10, 12, 38, 3, 500000, tzinfo=datetime.timezone.utc + ) + + def test_skips_points_without_a_timestamp(self) -> None: + points = [self._gps_point(0.0, None), self._gps_point(1.0, 1628599086.0)] + assert sample_video._gps_clock_start_time(points) == datetime.datetime( + 2021, 8, 10, 12, 38, 5, tzinfo=datetime.timezone.utc + ) + + def test_no_absolute_timestamps(self) -> None: + assert sample_video._gps_clock_start_time(_make_gps_points(3)) is None + + def test_no_points(self) -> None: + assert sample_video._gps_clock_start_time([]) is None + + # --------------------------------------------------------------------------- # Helpers for distance-based sampling tests # --------------------------------------------------------------------------- @@ -130,12 +222,6 @@ def test_sample_video_with_start_time(tmpdir: py.path.local, setup_mock): MOCK_PROBE_JSON = _PWD / "data" / "mock_sample_video" / "videos" / "hello.mp4" TEST_EXIF_JPG = _PWD / "data" / "test_exif.jpg" -# Start time derived from the hello.mp4 probe fixture: -# creation_time "2021-08-10T14:38:06.000000Z" - duration "60.977000" -PROBE_START_TIME = datetime.datetime( - 2021, 8, 10, 14, 36, 55, 23000, tzinfo=datetime.timezone.utc -) - def _load_probe_output() -> ffmpeglib.ProbeOutput: with open(MOCK_PROBE_JSON) as fp: