From 1da3d67ff3bf0caa3850764c453825133380665c Mon Sep 17 00:00:00 2001 From: Dichen Zhang Date: Tue, 6 Oct 2026 18:17:03 +0000 Subject: [PATCH 1/2] fix(extractor): decouple Motion Photo extractor from MotionPhotoChecker --- .../motion_photo_checker_extractor_common.cc | 33 +++++++---- .../motion_photo_checker_extractor_common.h | 10 ++++ .../src/motion_photo_extractor/function.cc | 43 +++++++-------- .../tests/motion_photo_extractor_test.cc | 55 +++++++++++++++++++ 4 files changed, 106 insertions(+), 35 deletions(-) diff --git a/lib/core/motion_photo/src/motion_photo_checker_extractor_common/motion_photo_checker_extractor_common.cc b/lib/core/motion_photo/src/motion_photo_checker_extractor_common/motion_photo_checker_extractor_common.cc index e7d7546..ac28ae1 100644 --- a/lib/core/motion_photo/src/motion_photo_checker_extractor_common/motion_photo_checker_extractor_common.cc +++ b/lib/core/motion_photo/src/motion_photo_checker_extractor_common/motion_photo_checker_extractor_common.cc @@ -250,15 +250,12 @@ tuple GetImageSizeAndXmpRange( return make_tuple(image_size, primary_xmp_range, mpvd_box); } -bool ParseAndCheckXmpMetadata(DataSource* data_source, size_t file_size, - FileType file_type, - const string& motion_photo_file_name, - const DataRange& xmp_range, size_t image_size, - const std::string& metadata_file_name_output, - MotionPhoto* motion_photo, - MotionPhotoReader* motion_photo_reader, - MotionPhotoChecker* motion_photo_checker, - const StringOutputter& outputter) { +bool ParseXmpMetadata(DataSource* data_source, + const string& motion_photo_file_name, + const DataRange& xmp_range, + const std::string& metadata_file_name_output, + MotionPhotoReader* motion_photo_reader, + const StringOutputter& outputter) { outputter("\nParsing XMP metadata\n"); const size_t kMaxXmpLineLength = 100; DataLineMap data_line_map; @@ -271,8 +268,22 @@ bool ParseAndCheckXmpMetadata(DataSource* data_source, size_t file_size, WriteToOutput(motion_photo_file_name, metadata_file_name_output, xmp_range.GetBegin(), xmp_range.GetEnd()); } - if (!motion_photo_reader->ReadImageMetadata({xmp_range}, data_source, - &bytes_parsed)) { + return motion_photo_reader->ReadImageMetadata({xmp_range}, data_source, + &bytes_parsed); +} + +bool ParseAndCheckXmpMetadata(DataSource* data_source, size_t file_size, + FileType file_type, + const string& motion_photo_file_name, + const DataRange& xmp_range, size_t image_size, + const std::string& metadata_file_name_output, + MotionPhoto* motion_photo, + MotionPhotoReader* motion_photo_reader, + MotionPhotoChecker* motion_photo_checker, + const StringOutputter& outputter) { + if (!ParseXmpMetadata(data_source, motion_photo_file_name, xmp_range, + metadata_file_name_output, motion_photo_reader, + outputter)) { return false; } diff --git a/lib/core/motion_photo/src/motion_photo_checker_extractor_common/motion_photo_checker_extractor_common.h b/lib/core/motion_photo/src/motion_photo_checker_extractor_common/motion_photo_checker_extractor_common.h index 444755f..2ebc2c2 100644 --- a/lib/core/motion_photo/src/motion_photo_checker_extractor_common/motion_photo_checker_extractor_common.h +++ b/lib/core/motion_photo/src/motion_photo_checker_extractor_common/motion_photo_checker_extractor_common.h @@ -53,6 +53,16 @@ std::tuple GetImageSizeAndXmpRange( const image_io::StringOutputter& outputter, image_io::MessageHandler* message_handler); +// Parses the XMP metadata without running MotionPhotoChecker validations. +// Optionally writes the metadata to metadata_file_name_output. +bool ParseXmpMetadata( + image_io::DataSource* data_source, + const std::string& motion_photo_file_name, + const image_io::DataRange& xmp_range, + const std::string& metadata_file_name_output, + MotionPhotoReader* motion_photo_reader, + const image_io::StringOutputter& outputter); + // Parses and checks the XMP metadata. // Optionally writes the metadata to metadata_file_name_output. bool ParseAndCheckXmpMetadata( diff --git a/lib/core/motion_photo/src/motion_photo_extractor/function.cc b/lib/core/motion_photo/src/motion_photo_extractor/function.cc index 301e975..380d082 100644 --- a/lib/core/motion_photo/src/motion_photo_extractor/function.cc +++ b/lib/core/motion_photo/src/motion_photo_extractor/function.cc @@ -30,7 +30,6 @@ #include "image_io/utils/string_outputter_message_writer.h" #include "metadata_collection.pb.h" #include "motion_photo/metadata_engine.h" -#include "motion_photo/motion_photo_checker.h" #include "motion_photo/motion_photo_reader.h" #include "motion_photo/motion_photo_utils.h" #include "motion_photo/motion_photo_writer.h" @@ -81,7 +80,7 @@ string GetProgramName(std::string_view arg0) { program_name = arg0.substr(last_slash + 1); } if (program_name.empty()) { - program_name = "motion_photo_checker"; + program_name = "motion_photo_extractor"; } return program_name; } @@ -255,7 +254,6 @@ int ExtractMotionPhoto(const MotionPhotoExtractorParams& params, MotionPhoto motion_photo(file_type); MotionPhotoReader motion_photo_reader(&motion_photo, &message_handler); - MotionPhotoChecker motion_photo_checker(motion_photo, &message_handler); unique_ptr data_source; tie(data_source, file_size) = @@ -272,12 +270,14 @@ int ExtractMotionPhoto(const MotionPhotoExtractorParams& params, int64_t video_length = -1; int64_t image_size = -1; - for (const auto& block : collection.blocks()) { - if (block.format_identifier() == "container.trailer") { - image_size = block.offset(); - if (block.has_video_offset_in_block() && block.has_video_length()) { - video_offset = block.offset() + block.video_offset_in_block(); - video_length = block.video_length(); + if (engine.IsMotionPhoto(collection)) { + for (const auto& block : collection.blocks()) { + if (block.format_identifier() == "container.trailer") { + image_size = block.offset(); + if (block.has_video_offset_in_block() && block.has_video_length()) { + video_offset = block.offset() + block.video_offset_in_block(); + video_length = block.video_length(); + } } } } @@ -337,26 +337,21 @@ int ExtractMotionPhoto(const MotionPhotoExtractorParams& params, } motion_photo.SetMpvdBox(mpvd_box); - // Parse the XMP metadata string from the data source and check the values. - if (!ParseAndCheckXmpMetadata(data_source.get(), file_size, file_type, - params.motion_photo_file_name, xmp_range, - std_image_size, params.metadata_file_name_output, - &motion_photo, &motion_photo_reader, - &motion_photo_checker, outputter)) { + // Parse the XMP metadata string without running strict MotionPhotoChecker rules. + if (!ParseXmpMetadata(data_source.get(), params.motion_photo_file_name, + xmp_range, params.metadata_file_name_output, + &motion_photo_reader, outputter)) { + return 1; + } + if (!motion_photo.IsMotionPhoto() && !mpvd_box.IsValid()) { + message_handler.ReportMessage(Message::kValueError, + "Input file is not a motion photo"); return 1; } - // Read and decode and check the MP4 track data and metadata. int64_t offset = motion_photo.GetMp4FileOffset(file_size); - if (offset < file_size) { - if (!ReadDecodeAndCheckVideoMetadata(params.motion_photo_file_name, offset, - &motion_photo, &motion_photo_reader, - &motion_photo_checker, outputter, - &message_handler)) { - return 0; - } - + if (offset >= 0 && offset < static_cast(file_size)) { if (!params.video_file_name_output.empty()) { WriteToOutput(params.motion_photo_file_name, params.video_file_name_output, offset, file_size); diff --git a/lib/core/motion_photo/tests/motion_photo_extractor_test.cc b/lib/core/motion_photo/tests/motion_photo_extractor_test.cc index 52c0d24..6c299c1 100644 --- a/lib/core/motion_photo/tests/motion_photo_extractor_test.cc +++ b/lib/core/motion_photo/tests/motion_photo_extractor_test.cc @@ -15,8 +15,11 @@ #include #include +#include #include #include +#include +#include #include #include @@ -219,5 +222,57 @@ TEST(MotionPhotoExtractorFunctionTest, StillOnlyImageRejection) { fs::remove_all(kTestDir); } +// Ensures extractor succeeds directly without running strict MotionPhotoChecker +// validation rules (e.g. when GCamera:MotionPhotoVersion is "2" instead of "1", +// which MotionPhotoChecker rejects with an error). +TEST(MotionPhotoExtractorFunctionTest, ExtractionBypassesCheckerValidation) { + const string kTestDir = CreateTestDir("ExtractionBypassesCheckerValidation"); + const string kSrcJpg = + "lib/core/motion_photo/testdata/motion_photo_single_video_track.MP.jpg"; + const string kModifiedJpg = + (fs::path(kTestDir) / "non_conforming_version.jpg").string(); + const string kOutputFileImage = + (fs::path(kTestDir) / "extracted_still.jpg").string(); + const string kOutputFileVideo = + (fs::path(kTestDir) / "extracted_video.mp4").string(); + + std::ifstream in(kSrcJpg, std::ios::binary); + string bytes((std::istreambuf_iterator(in)), + std::istreambuf_iterator()); + const string kTarget = "MotionPhotoVersion=\"1\""; + const string kReplacement = "MotionPhotoVersion=\"2\""; + size_t pos = bytes.find(kTarget); + EXPECT_NE(pos, string::npos); + if (pos != string::npos) { + bytes.replace(pos, kTarget.size(), kReplacement); + } + { + std::ofstream out(kModifiedJpg, std::ios::binary); + out.write(bytes.data(), bytes.size()); + } + + const char* argv[] = {"MotionPhotoExtractor", "-mi", + kModifiedJpg.c_str(), "-po", + kOutputFileImage.c_str(), "-vo", + kOutputFileVideo.c_str()}; + + bool has_zero_errors = false; + int status = MotionPhotoExtractorFunction(7, argv, [&](absl::string_view str) { + if (absl::StrContains(str, "0 errors")) { + has_zero_errors = true; + } + }); + + EXPECT_EQ(status, 0); + EXPECT_TRUE(has_zero_errors); + EXPECT_TRUE(fs::exists(kOutputFileImage)); + EXPECT_TRUE(fs::exists(kOutputFileVideo)); + + MotionPhotoMp4FileReader mp4_reader; + EXPECT_TRUE(mp4_reader.Read(kOutputFileVideo, 0)); + + fs::remove_all(kTestDir); +} + } // namespace motion_photo } // namespace libmotionphoto From f10dbca9b80fb080ddc3200d5ea8adf6b5061bbc Mon Sep 17 00:00:00 2001 From: Dichen Zhang Date: Tue, 6 Oct 2026 19:11:13 +0000 Subject: [PATCH 2/2] fix(heif): detect HEIC and AVIF motion photos without libheif --- .../motionphoto/MetadataEngineTest.java | 26 + lib/core/CMakeLists.txt | 1 + .../motion_photo/container_splitter.h | 10 +- .../motion_photo/heif_container_splitter.cc | 614 ++++++++++++++++++ .../motion_photo/jpeg_container_splitter.cc | 9 +- .../src/motion_photo/metadata_engine.cc | 7 +- .../src/motion_photo_jni/motion_photo_jni.cc | 30 +- .../tests/heif_container_splitter_test.cc | 437 +++++++++++++ .../tests/libmotionphoto_api_test.cc | 25 + 9 files changed, 1139 insertions(+), 20 deletions(-) create mode 100644 lib/core/motion_photo/src/motion_photo/heif_container_splitter.cc create mode 100644 lib/core/motion_photo/tests/heif_container_splitter_test.cc diff --git a/javatests/com/google/libmotionphoto/motionphoto/MetadataEngineTest.java b/javatests/com/google/libmotionphoto/motionphoto/MetadataEngineTest.java index 277d606..00ad292 100644 --- a/javatests/com/google/libmotionphoto/motionphoto/MetadataEngineTest.java +++ b/javatests/com/google/libmotionphoto/motionphoto/MetadataEngineTest.java @@ -85,6 +85,32 @@ public void isMotionPhoto_withOptions_evaluatesCorrectly() { } } + @Test + public void parseMetadata_heicMotionPhoto_isMotionPhoto() throws Exception { + String heicPath = TestUtil.getTestDataPath("motion_photo_single_video_track.MP.heic"); + + byte[] pathBytes = MetadataEngine.parseMetadata(heicPath); + assertThat(pathBytes).isNotNull(); + assertThat(MetadataEngine.isMotionPhoto(pathBytes)).isTrue(); + + File file = new File(heicPath); + try (ParcelFileDescriptor pfd = + ParcelFileDescriptor.open(file, ParcelFileDescriptor.MODE_READ_ONLY)) { + byte[] fdBytes = MetadataEngine.parseMetadataFd(pfd.getFd(), 0, file.length()); + assertThat(fdBytes).isEqualTo(pathBytes); + } + } + + @Test + public void parseMetadata_heicWithoutVideo_isNotMotionPhoto() { + // The XMP says motion photo, but the file has no 'mpvd' video box. + byte[] protoBytes = + MetadataEngine.parseMetadata(TestUtil.getTestDataPath("motion_photo_photo_only.heic")); + + assertThat(protoBytes).isNotNull(); + assertThat(MetadataEngine.isMotionPhoto(protoBytes)).isFalse(); + } + @Test public void extractAgtm_viaPathAndMemory_returnsValidJson() throws Exception { String agtmJsonPath = MetadataEngine.extractAgtm(agtmPhotoPath); diff --git a/lib/core/CMakeLists.txt b/lib/core/CMakeLists.txt index 7ffde36..f676d8d 100644 --- a/lib/core/CMakeLists.txt +++ b/lib/core/CMakeLists.txt @@ -208,6 +208,7 @@ if (NOT ANDROID) motion_photo/tests/motion_photo_checker_test.cc motion_photo/tests/motion_photo_extractor_test.cc motion_photo/tests/mp4_box_reader_test.cc + motion_photo/tests/heif_container_splitter_test.cc motion_photo/tests/libmotionphoto_api_test.cc ) target_include_directories(motion_photo_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/.. motion_photo/includes ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/lib/core/motion_photo/includes/motion_photo/container_splitter.h b/lib/core/motion_photo/includes/motion_photo/container_splitter.h index e11f0fe..5318cbc 100644 --- a/lib/core/motion_photo/includes/motion_photo/container_splitter.h +++ b/lib/core/motion_photo/includes/motion_photo/container_splitter.h @@ -38,8 +38,14 @@ class ContainerSplitter { image_io::DataSource* data_source, size_t file_size) = 0; }; -std::unique_ptr CreateContainerSplitter( - FileType type, image_io::MessageHandler* message_handler); +// Returns a splitter for JPEG images. +std::unique_ptr CreateJpegContainerSplitter( + image_io::MessageHandler* message_handler); + +// Returns a splitter for HEIF-based images (HEIC and AVIF) that does not need +// libheif. It emits the primary image's XMP item as a "standard.xmp" block, and +// only when the file also has a top-level 'mpvd' (motion photo video) box. +std::unique_ptr CreateHeifContainerSplitter(); } // namespace motion_photo } // namespace libmotionphoto diff --git a/lib/core/motion_photo/src/motion_photo/heif_container_splitter.cc b/lib/core/motion_photo/src/motion_photo/heif_container_splitter.cc new file mode 100644 index 0000000..fcd1a3e --- /dev/null +++ b/lib/core/motion_photo/src/motion_photo/heif_container_splitter.cc @@ -0,0 +1,614 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// A ContainerSplitter for HEIF-based images (HEIC and AVIF). It reads the ISO +// BMFF item boxes directly instead of going through libheif, so motion photo +// detection also works in builds without libheif, such as Android. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "image_io/base/data_segment.h" +#include "image_io/base/data_source.h" +#include "motion_photo/container_splitter.h" +#include "motion_photo/metadata_block.h" +#include "motion_photo/mpvd_box.h" + +namespace libmotionphoto { +namespace motion_photo { + +namespace { + +// Limits that keep malformed or hostile input from driving large allocations +// or long loops. +constexpr size_t kMaxTopLevelBoxes = 1024; +constexpr size_t kMaxChildBoxes = 4096; +constexpr uint64_t kMaxMetaPayloadSize = 16 * 1024 * 1024; +constexpr uint64_t kMaxXmpItemSize = 16 * 1024 * 1024; +constexpr size_t kMaxXmpItems = 16; +constexpr uint64_t kMaxItemExtents = 256; + +// ISO/IEC 23008-12 stores XMP as a 'mime' item with this content type. +constexpr std::string_view kXmpContentType = "application/rdf+xml"; + +// Copies `length` bytes at file offset `start` into `out`. Returns false if the +// data source cannot supply the whole range. +bool ReadFromDataSource(image_io::DataSource* data_source, size_t start, + size_t length, uint8_t* out) { + size_t copied = 0; + while (copied < length) { + const size_t chunk_start = start + copied; + std::shared_ptr segment = + data_source->GetDataSegment(chunk_start, length - copied); + if (segment == nullptr) { + return false; + } + // A non-null buffer means `chunk_start` lies inside the segment, so each + // iteration copies at least one byte. + const uint8_t* buffer = segment->GetBuffer(chunk_start); + if (buffer == nullptr) { + return false; + } + const size_t chunk_size = + std::min(length - copied, segment->GetEnd() - chunk_start); + std::copy(buffer, buffer + chunk_size, out + copied); + copied += chunk_size; + } + return true; +} + +// Appends `length` bytes at file offset `start` to `out`. +bool AppendFromDataSource(image_io::DataSource* data_source, size_t start, + size_t length, std::vector* out) { + const size_t old_size = out->size(); + out->resize(old_size + length); + return ReadFromDataSource(data_source, start, length, out->data() + old_size); +} + +// A bounds-checked big-endian reader over an in-memory byte range. Reading past +// the end puts the reader into a sticky failed state in which every read +// returns zero or an empty value. +class BoxReader { + public: + BoxReader(const uint8_t* data, size_t size) : data_(data), size_(size) {} + + bool ok() const { return ok_; } + + // Returns the number of unread bytes, or 0 after a failed read. + size_t remaining() const { return ok_ ? size_ - position_ : 0; } + + // Returns a pointer to the first unread byte. + const uint8_t* current() const { return data_ + position_; } + + // Reads an unsigned big-endian integer that is `num_bytes` (0 to 8) long. + uint64_t ReadUInt(size_t num_bytes) { + if (!ok_ || num_bytes > sizeof(uint64_t) || num_bytes > size_ - position_) { + ok_ = false; + return 0; + } + uint64_t value = 0; + for (size_t i = 0; i < num_bytes; ++i) { + value = (value << 8) | data_[position_ + i]; + } + position_ += num_bytes; + return value; + } + + // Reads the version of a FullBox header and skips its 24-bit flags. + uint64_t ReadFullBoxVersion() { + const uint64_t version = ReadUInt(1); + Skip(3); + return version; + } + + // Reads a four-character code such as a box type or an item type. + std::string_view ReadFourCc() { + if (!ok_ || size_ - position_ < 4) { + ok_ = false; + return {}; + } + std::string_view four_cc(reinterpret_cast(data_ + position_), + 4); + position_ += 4; + return four_cc; + } + + // Reads a NUL-terminated string. A string that runs to the end of the range + // is accepted without a terminator, because some writers omit the last NUL. + std::string_view ReadCString() { + const size_t available = remaining(); + if (available == 0) { + return {}; + } + const char* begin = reinterpret_cast(data_ + position_); + const void* terminator = std::memchr(begin, '\0', available); + if (terminator == nullptr) { + position_ = size_; + return std::string_view(begin, available); + } + const size_t length = static_cast(terminator) - begin; + position_ += length + 1; + return std::string_view(begin, length); + } + + void Skip(size_t num_bytes) { + if (!ok_ || num_bytes > size_ - position_) { + ok_ = false; + return; + } + position_ += num_bytes; + } + + private: + const uint8_t* data_; + size_t size_; + size_t position_ = 0; + bool ok_ = true; +}; + +struct BoxHeader { + std::string_view type; + size_t header_size = 0; + // The size of the whole box, including the header. + size_t box_size = 0; +}; + +// Reads the box header at the reader's position. `available` is the number of +// bytes from the start of the box to the end of its parent. Returns nullopt if +// the header is malformed or the box overruns its parent. +std::optional ReadBoxHeader(BoxReader* reader, size_t available) { + BoxHeader header; + uint64_t box_size = reader->ReadUInt(4); + header.type = reader->ReadFourCc(); + header.header_size = 8; + if (box_size == 1) { + box_size = reader->ReadUInt(8); + header.header_size = 16; + } else if (box_size == 0) { + // The box extends to the end of its parent. + box_size = available; + } + if (!reader->ok() || box_size < header.header_size || box_size > available) { + return std::nullopt; + } + header.box_size = static_cast(box_size); + return header; +} + +// Calls `visitor(type, payload, payload_size)` for each box in the in-memory +// range [data, data + size), in order. Stops when the visitor returns false or +// at the first malformed box. +template +void ForEachBox(const uint8_t* data, size_t size, Visitor visitor) { + BoxReader reader(data, size); + for (size_t i = 0; i < kMaxChildBoxes && reader.remaining() >= 8; ++i) { + const uint8_t* box_start = reader.current(); + std::optional header = + ReadBoxHeader(&reader, reader.remaining()); + if (!header.has_value()) { + return; + } + const size_t payload_size = header->box_size - header->header_size; + if (!visitor(header->type, box_start + header->header_size, payload_size)) { + return; + } + reader.Skip(payload_size); + } +} + +// The location of a top-level box payload in the file. +struct BoxLocation { + size_t payload_offset = 0; + size_t payload_size = 0; +}; + +// Finds the first top-level box of type `box_type`. Returns nullopt if there +// is none, or if a malformed box precedes it. +std::optional FindTopLevelBox(image_io::DataSource* data_source, + size_t file_size, + std::string_view box_type) { + size_t position = 0; + for (size_t i = 0; i < kMaxTopLevelBoxes && file_size - position >= 8; ++i) { + std::array header_bytes; + const size_t header_length = + std::min(header_bytes.size(), file_size - position); + if (!ReadFromDataSource(data_source, position, header_length, + header_bytes.data())) { + return std::nullopt; + } + BoxReader reader(header_bytes.data(), header_length); + std::optional header = + ReadBoxHeader(&reader, file_size - position); + if (!header.has_value()) { + return std::nullopt; + } + if (header->type == box_type) { + return BoxLocation{position + header->header_size, + header->box_size - header->header_size}; + } + position += header->box_size; + } + return std::nullopt; +} + +// Returns true if the file has a valid top-level 'mpvd' box, which holds the +// motion photo video in HEIF motion photos. This uses the same check as +// MotionPhotoHeifInfoBuilder. +bool HasMpvdBox(image_io::DataSource* data_source) { + MpvdBox mpvd_box; + MpvdBoxScanReceiver receiver(mpvd_box); + MpvdBox::Scan(data_source, 0, receiver); + return mpvd_box.IsValid(); +} + +// A byte range inside the in-memory copy of the 'meta' box payload. +struct ByteRange { + const uint8_t* data = nullptr; + size_t size = 0; + + bool found() const { return data != nullptr; } +}; + +// The 'meta' child boxes used to locate the XMP item. +struct MetaBoxes { + ByteRange pitm; + ByteRange iinf; + ByteRange iref; + ByteRange iloc; + ByteRange idat; +}; + +// Returns the payloads of the first 'pitm', 'iinf', 'iref', 'iloc', and 'idat' +// boxes in a 'meta' box payload. +MetaBoxes FindMetaBoxes(const std::vector& meta_payload) { + MetaBoxes boxes; + BoxReader reader(meta_payload.data(), meta_payload.size()); + // 'meta' is a FullBox: skip its version and flags. + reader.ReadFullBoxVersion(); + ForEachBox(reader.current(), reader.remaining(), + [&boxes](std::string_view type, const uint8_t* payload, + size_t payload_size) { + ByteRange* box = nullptr; + if (type == "pitm") { + box = &boxes.pitm; + } else if (type == "iinf") { + box = &boxes.iinf; + } else if (type == "iref") { + box = &boxes.iref; + } else if (type == "iloc") { + box = &boxes.iloc; + } else if (type == "idat") { + box = &boxes.idat; + } + if (box != nullptr && !box->found()) { + *box = ByteRange{payload, payload_size}; + } + return true; + }); + return boxes; +} + +// Returns the primary item ID from a 'pitm' payload. +std::optional ParsePrimaryItemId(ByteRange pitm) { + BoxReader reader(pitm.data, pitm.size); + const uint64_t version = reader.ReadFullBoxVersion(); + const uint64_t item_id = reader.ReadUInt(version == 0 ? 2 : 4); + if (!reader.ok()) { + return std::nullopt; + } + return static_cast(item_id); +} + +// Returns the item ID from an 'infe' payload if the item holds XMP. +std::optional ParseXmpItemId(const uint8_t* data, size_t size) { + BoxReader reader(data, size); + const uint64_t version = reader.ReadFullBoxVersion(); + if (version > 3) { + return std::nullopt; + } + uint64_t item_id = 0; + if (version >= 2) { + item_id = reader.ReadUInt(version == 2 ? 2 : 4); + reader.Skip(2); // item_protection_index + if (reader.ReadFourCc() != "mime") { + return std::nullopt; + } + } else { + item_id = reader.ReadUInt(2); + reader.Skip(2); // item_protection_index + } + reader.ReadCString(); // item_name + const std::string_view content_type = reader.ReadCString(); + if (!reader.ok() || content_type != kXmpContentType) { + return std::nullopt; + } + return static_cast(item_id); +} + +// Returns the IDs of the XMP items listed in an 'iinf' payload, in order. +std::vector FindXmpItemIds(ByteRange iinf) { + std::vector xmp_item_ids; + BoxReader reader(iinf.data, iinf.size); + const uint64_t version = reader.ReadFullBoxVersion(); + // Skip entry_count and walk the 'infe' boxes directly. + reader.Skip(version == 0 ? 2 : 4); + if (!reader.ok()) { + return xmp_item_ids; + } + ForEachBox(reader.current(), reader.remaining(), + [&xmp_item_ids](std::string_view type, const uint8_t* payload, + size_t payload_size) { + if (type == "infe") { + std::optional item_id = + ParseXmpItemId(payload, payload_size); + if (item_id.has_value()) { + xmp_item_ids.push_back(*item_id); + } + } + return xmp_item_ids.size() < kMaxXmpItems; + }); + return xmp_item_ids; +} + +// Returns the IDs of the items that have a 'cdsc' (content describes) reference +// to `described_item_id` in an 'iref' payload. +std::vector FindItemsDescribing(ByteRange iref, + uint32_t described_item_id) { + std::vector item_ids; + BoxReader reader(iref.data, iref.size); + const size_t item_id_size = reader.ReadFullBoxVersion() == 0 ? 2 : 4; + if (!reader.ok()) { + return item_ids; + } + ForEachBox( + reader.current(), reader.remaining(), + [&](std::string_view type, const uint8_t* payload, size_t payload_size) { + if (type != "cdsc") { + return true; + } + BoxReader reference(payload, payload_size); + const uint64_t from_item_id = reference.ReadUInt(item_id_size); + const uint64_t reference_count = reference.ReadUInt(2); + for (uint64_t i = 0; i < reference_count; ++i) { + const uint64_t to_item_id = reference.ReadUInt(item_id_size); + if (!reference.ok()) { + break; + } + if (to_item_id == described_item_id) { + item_ids.push_back(static_cast(from_item_id)); + break; + } + } + return true; + }); + return item_ids; +} + +// Returns the XMP item that describes the primary item, falling back to the +// first XMP item when no 'cdsc' reference links one to the primary item. +uint32_t ChooseXmpItemId(const MetaBoxes& boxes, + const std::vector& xmp_item_ids) { + if (boxes.pitm.found() && boxes.iref.found()) { + std::optional primary_item_id = ParsePrimaryItemId(boxes.pitm); + if (primary_item_id.has_value()) { + const std::vector primary_descriptors = + FindItemsDescribing(boxes.iref, *primary_item_id); + for (uint32_t item_id : xmp_item_ids) { + if (std::find(primary_descriptors.begin(), primary_descriptors.end(), + item_id) != primary_descriptors.end()) { + return item_id; + } + } + } + } + return xmp_item_ids.front(); +} + +// Where the data of an item is stored, as described by its 'iloc' entry. +struct ItemLocation { + uint64_t construction_method = 0; + uint64_t data_reference_index = 0; + uint64_t base_offset = 0; + // {offset, length} pairs relative to `base_offset`. + std::vector> extents; +}; + +bool IsValidIlocFieldSize(size_t num_bytes) { + return num_bytes == 0 || num_bytes == 4 || num_bytes == 8; +} + +// Returns the location of `item_id` from an 'iloc' payload. +std::optional FindItemLocation(ByteRange iloc, uint32_t item_id) { + BoxReader reader(iloc.data, iloc.size); + const uint64_t version = reader.ReadFullBoxVersion(); + if (version > 2) { + return std::nullopt; + } + const uint64_t field_sizes = reader.ReadUInt(2); + const size_t offset_size = (field_sizes >> 12) & 0xF; + const size_t length_size = (field_sizes >> 8) & 0xF; + const size_t base_offset_size = (field_sizes >> 4) & 0xF; + const size_t index_size = version == 0 ? 0 : field_sizes & 0xF; + if (!IsValidIlocFieldSize(offset_size) || + !IsValidIlocFieldSize(length_size) || + !IsValidIlocFieldSize(base_offset_size) || + !IsValidIlocFieldSize(index_size)) { + return std::nullopt; + } + const size_t extent_size = index_size + offset_size + length_size; + const size_t item_id_size = version < 2 ? 2 : 4; + const uint64_t item_count = reader.ReadUInt(item_id_size); + for (uint64_t i = 0; i < item_count && reader.ok(); ++i) { + const uint64_t current_item_id = reader.ReadUInt(item_id_size); + ItemLocation location; + if (version > 0) { + location.construction_method = reader.ReadUInt(2) & 0xF; + } + location.data_reference_index = reader.ReadUInt(2); + location.base_offset = reader.ReadUInt(base_offset_size); + const uint64_t extent_count = reader.ReadUInt(2); + if (current_item_id != item_id) { + reader.Skip(static_cast(extent_count) * extent_size); + continue; + } + if (extent_count > kMaxItemExtents) { + return std::nullopt; + } + for (uint64_t e = 0; e < extent_count; ++e) { + reader.Skip(index_size); + const uint64_t extent_offset = reader.ReadUInt(offset_size); + const uint64_t extent_length = reader.ReadUInt(length_size); + location.extents.emplace_back(extent_offset, extent_length); + } + if (!reader.ok()) { + return std::nullopt; + } + return location; + } + return std::nullopt; +} + +// Copies the data of an item into `out` by following its extents. Extents are +// file offsets for construction method 0 and offsets into the 'idat' payload +// for construction method 1. Returns the file offset of the first extent, or +// nullopt if the item is stored in an unsupported way, points outside the +// file, or is larger than kMaxXmpItemSize. +std::optional ReadItemData(image_io::DataSource* data_source, + size_t file_size, + const ItemLocation& location, ByteRange idat, + size_t idat_file_offset, + std::vector* out) { + // A non-zero data_reference_index points at data in another file. + if (location.data_reference_index != 0 || location.construction_method > 1 || + location.extents.empty()) { + return std::nullopt; + } + std::optional first_extent_offset; + for (const auto& [extent_offset, extent_length] : location.extents) { + // A zero length means "to the end of the data", which XMP never needs. + if (extent_length == 0 || extent_length > kMaxXmpItemSize - out->size() || + extent_offset > + std::numeric_limits::max() - location.base_offset) { + return std::nullopt; + } + const uint64_t start = location.base_offset + extent_offset; + size_t extent_file_offset = 0; + if (location.construction_method == 0) { + if (start > file_size || extent_length > file_size - start) { + return std::nullopt; + } + extent_file_offset = static_cast(start); + if (!AppendFromDataSource(data_source, extent_file_offset, + static_cast(extent_length), out)) { + return std::nullopt; + } + } else { + if (!idat.found() || start > idat.size || + extent_length > idat.size - start) { + return std::nullopt; + } + const uint8_t* extent_data = idat.data + static_cast(start); + out->insert(out->end(), extent_data, + extent_data + static_cast(extent_length)); + extent_file_offset = idat_file_offset + static_cast(start); + } + if (!first_extent_offset.has_value()) { + first_extent_offset = extent_file_offset; + } + } + return first_extent_offset; +} + +// Returns the XMP of the primary image as a "standard.xmp" block. +std::optional ExtractPrimaryXmp( + image_io::DataSource* data_source, size_t file_size) { + std::optional meta = + FindTopLevelBox(data_source, file_size, "meta"); + if (!meta.has_value() || meta->payload_size > kMaxMetaPayloadSize) { + return std::nullopt; + } + std::vector meta_payload; + if (!AppendFromDataSource(data_source, meta->payload_offset, + meta->payload_size, &meta_payload)) { + return std::nullopt; + } + const MetaBoxes boxes = FindMetaBoxes(meta_payload); + if (!boxes.iinf.found() || !boxes.iloc.found()) { + return std::nullopt; + } + const std::vector xmp_item_ids = FindXmpItemIds(boxes.iinf); + if (xmp_item_ids.empty()) { + return std::nullopt; + } + std::optional location = + FindItemLocation(boxes.iloc, ChooseXmpItemId(boxes, xmp_item_ids)); + if (!location.has_value()) { + return std::nullopt; + } + const size_t idat_file_offset = + boxes.idat.found() + ? meta->payload_offset + + static_cast(boxes.idat.data - meta_payload.data()) + : 0; + + RawMetadataBlock block; + block.type = "XMP"; + block.format_identifier = "standard.xmp"; + std::optional offset = + ReadItemData(data_source, file_size, *location, boxes.idat, + idat_file_offset, &block.bytes); + if (!offset.has_value()) { + return std::nullopt; + } + block.offset = *offset; + return block; +} + +class HeifContainerSplitter : public ContainerSplitter { + public: + // Emits the primary image's XMP only when the file also has an 'mpvd' box: + // editors often drop the unknown 'mpvd' box but keep the XMP, and such a file + // no longer contains a video. The video itself is not copied. + std::vector Split(image_io::DataSource* data_source, + size_t file_size) override { + std::vector blocks; + if (data_source == nullptr || file_size == 0 || !HasMpvdBox(data_source)) { + return blocks; + } + std::optional xmp_block = + ExtractPrimaryXmp(data_source, file_size); + if (xmp_block.has_value()) { + blocks.push_back(*std::move(xmp_block)); + } + return blocks; + } +}; + +} // namespace + +std::unique_ptr CreateHeifContainerSplitter() { + return std::make_unique(); +} + +} // namespace motion_photo +} // namespace libmotionphoto diff --git a/lib/core/motion_photo/src/motion_photo/jpeg_container_splitter.cc b/lib/core/motion_photo/src/motion_photo/jpeg_container_splitter.cc index 9fed8ae..22c76df 100644 --- a/lib/core/motion_photo/src/motion_photo/jpeg_container_splitter.cc +++ b/lib/core/motion_photo/src/motion_photo/jpeg_container_splitter.cc @@ -308,12 +308,9 @@ class JpegContainerSplitter : public ContainerSplitter, } // namespace -std::unique_ptr CreateContainerSplitter( - FileType type, image_io::MessageHandler* message_handler) { - if (type == FileType::kJpeg) { - return std::make_unique(message_handler); - } - return nullptr; +std::unique_ptr CreateJpegContainerSplitter( + image_io::MessageHandler* message_handler) { + return std::make_unique(message_handler); } } // namespace motion_photo diff --git a/lib/core/motion_photo/src/motion_photo/metadata_engine.cc b/lib/core/motion_photo/src/motion_photo/metadata_engine.cc index 63fe6f4..98d264e 100644 --- a/lib/core/motion_photo/src/motion_photo/metadata_engine.cc +++ b/lib/core/motion_photo/src/motion_photo/metadata_engine.cc @@ -60,7 +60,12 @@ MetadataCollection MetadataEngine::Parse(image_io::DataSource* data_source, size_t file_size, FileType file_type) { MetadataCollection collection; - auto splitter = CreateContainerSplitter(file_type, message_handler_); + std::unique_ptr splitter; + if (file_type == FileType::kJpeg) { + splitter = CreateJpegContainerSplitter(message_handler_); + } else if (IsHeif(file_type)) { + splitter = CreateHeifContainerSplitter(); + } if (!splitter) { return collection; } diff --git a/lib/core/motion_photo/src/motion_photo_jni/motion_photo_jni.cc b/lib/core/motion_photo/src/motion_photo_jni/motion_photo_jni.cc index 9f8f07b..df9653a 100644 --- a/lib/core/motion_photo/src/motion_photo_jni/motion_photo_jni.cc +++ b/lib/core/motion_photo/src/motion_photo_jni/motion_photo_jni.cc @@ -63,6 +63,17 @@ using libmotionphoto::jni::ScopedUtfChars; using libmotionphoto::jni::ThrowOutOfMemoryError; using libmotionphoto::jni::ThrowRuntimeException; +// Returns kHeic for ISO BMFF images, which start with an 'ftyp' box, and kJpeg +// otherwise. The metadata engine parses HEIC and AVIF the same way. +libmotionphoto::motion_photo::FileType SniffFileType(const uint8_t* data, + size_t size) { + if (data != nullptr && size >= 8 && data[4] == 'f' && data[5] == 't' && + data[6] == 'y' && data[7] == 'p') { + return libmotionphoto::motion_photo::FileType::kHeic; + } + return libmotionphoto::motion_photo::FileType::kJpeg; +} + // Lazy random-access DataSource backed by pread() on an open file descriptor. // Reads 64 KB chunks on demand over the full [0, total_length_) range without // buffering the entire file in memory. @@ -218,6 +229,11 @@ jbyteArray ParseMetadataFromFd(JNIEnv* env, int fd, int64_t offset, engine.RegisterProvider(std::make_unique()); mp::FileType file_type = mp::FileType::kJpeg; + if (std::shared_ptr header_segment = + data_source.GetDataSegment(0, 8)) { + file_type = SniffFileType(header_segment->GetBuffer(0), + header_segment->GetLength()); + } mp::MetadataCollection collection = engine.Parse(&data_source, static_cast(slice_length), file_type); @@ -713,11 +729,8 @@ Java_com_google_libmotionphoto_motionphoto_MetadataEngineJni_extractAgtmFromMemo mp::MetadataEngine engine(&message_handler); int64_t target_ts = 0; - mp::FileType file_type = mp::FileType::kJpeg; - if (len >= 8 && buf.get()[4] == 'f' && buf.get()[5] == 't' && - buf.get()[6] == 'y' && buf.get()[7] == 'p') { - file_type = mp::FileType::kHeic; - } + const mp::FileType file_type = + SniffFileType(data, static_cast(len)); auto data_segment = ii::DataSegment::Create( ii::DataRange(0, len), data, @@ -804,13 +817,8 @@ Java_com_google_libmotionphoto_motionphoto_MetadataEngineJni_extractAgtmFd__IJJ( mp::MetadataEngine engine(&message_handler); int64_t target_ts = 0; - mp::FileType file_type = mp::FileType::kJpeg; - if (bytes_read >= 8 && buffer[4] == 'f' && buffer[5] == 't' && - buffer[6] == 'y' && buffer[7] == 'p') { - file_type = mp::FileType::kHeic; - } - const uint8_t* data = reinterpret_cast(buffer.data()); + const mp::FileType file_type = SniffFileType(data, bytes_read); auto data_segment = ii::DataSegment::Create( ii::DataRange(0, bytes_read), data, ii::DataSegment::BufferDispositionPolicy::kDontDelete); diff --git a/lib/core/motion_photo/tests/heif_container_splitter_test.cc b/lib/core/motion_photo/tests/heif_container_splitter_test.cc new file mode 100644 index 0000000..23fb9d7 --- /dev/null +++ b/lib/core/motion_photo/tests/heif_container_splitter_test.cc @@ -0,0 +1,437 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "image_io/base/data_range.h" +#include "image_io/base/data_segment.h" +#include "image_io/base/data_segment_data_source.h" +#include "metadata_collection.pb.h" +#include "motion_photo/container_splitter.h" +#include "motion_photo/metadata_block.h" +#include "motion_photo/metadata_engine.h" +#include "motion_photo/motion_photo.h" +#include "motion_photo/tests/test_framework.h" + +namespace libmotionphoto { +namespace motion_photo { +namespace { + +using Bytes = std::vector; + +constexpr char kTestDataDir[] = "lib/core/motion_photo/testdata/"; +constexpr char kHeicMotionPhoto[] = "motion_photo_single_video_track.MP.heic"; +// Motion photo XMP without the 'mpvd' box, i.e. the video was removed. +constexpr char kHeicWithoutVideo[] = "motion_photo_photo_only.heic"; +constexpr std::string_view kXmpContentType = "application/rdf+xml"; + +Bytes ReadTestFile(const std::string& name) { + std::ifstream file(kTestDataDir + name, std::ios::binary); + return Bytes(std::istreambuf_iterator(file), + std::istreambuf_iterator()); +} + +std::string AsString(const Bytes& bytes) { + return std::string(bytes.begin(), bytes.end()); +} + +// Splits the first `size` bytes of `data` with the HEIF splitter. +std::vector SplitHeif(const uint8_t* data, size_t size) { + std::shared_ptr segment = + image_io::DataSegment::Create(image_io::DataRange(0, size), data, + image_io::DataSegment::kDontDelete); + image_io::DataSegmentDataSource data_source(segment); + return CreateHeifContainerSplitter()->Split(&data_source, size); +} + +std::vector SplitHeif(const Bytes& file) { + return SplitHeif(file.data(), file.size()); +} + +// Runs the full MetadataEngine pipeline and returns its motion photo verdict. +bool IsMotionPhotoPerEngine(const Bytes& file, FileType file_type) { + std::shared_ptr segment = + image_io::DataSegment::Create(image_io::DataRange(0, file.size()), + file.data(), + image_io::DataSegment::kDontDelete); + image_io::DataSegmentDataSource data_source(segment); + MetadataEngine engine(/*message_handler=*/nullptr); + return MetadataEngine::IsMotionPhoto( + engine.Parse(&data_source, file.size(), file_type)); +} + +// Helpers that write ISO BMFF boxes for synthetic HEIF files. + +void AppendUInt(uint64_t value, size_t num_bytes, Bytes* out) { + for (size_t i = num_bytes; i > 0; --i) { + out->push_back(static_cast(value >> (8 * (i - 1)))); + } +} + +void AppendString(std::string_view text, Bytes* out) { + out->insert(out->end(), text.begin(), text.end()); +} + +void AppendCString(std::string_view text, Bytes* out) { + AppendString(text, out); + out->push_back('\0'); +} + +Bytes Concat(std::initializer_list parts) { + Bytes out; + for (const Bytes& part : parts) { + out.insert(out.end(), part.begin(), part.end()); + } + return out; +} + +Bytes MakeBox(std::string_view type, const Bytes& payload) { + Bytes box; + AppendUInt(8 + payload.size(), 4, &box); + AppendString(type, &box); + return Concat({box, payload}); +} + +Bytes MakeFullBox(std::string_view type, uint8_t version, + const Bytes& payload) { + Bytes header; + AppendUInt(version, 1, &header); + AppendUInt(0, 3, &header); // flags + return MakeBox(type, Concat({header, payload})); +} + +Bytes MakeFtypBox() { + Bytes payload; + AppendString("heic", &payload); + AppendUInt(0, 4, &payload); // minor_version + AppendString("mif1heic", &payload); + return MakeBox("ftyp", payload); +} + +Bytes MakeMpvdBox() { + Bytes video; + AppendString("not really a video", &video); + return MakeBox("mpvd", video); +} + +Bytes MakePitmV0(uint16_t item_id) { + Bytes payload; + AppendUInt(item_id, 2, &payload); + return MakeFullBox("pitm", 0, payload); +} + +// An 'infe' version 2 box. `content_type` is only written for 'mime' items. +Bytes MakeInfeV2(uint16_t item_id, std::string_view item_type, + std::string_view content_type) { + Bytes payload; + AppendUInt(item_id, 2, &payload); + AppendUInt(0, 2, &payload); // item_protection_index + AppendString(item_type, &payload); + AppendCString("", &payload); // item_name + if (item_type == "mime") { + AppendCString(content_type, &payload); + } + return MakeFullBox("infe", 2, payload); +} + +Bytes MakeIinfV0(std::initializer_list entries) { + Bytes payload; + AppendUInt(entries.size(), 2, &payload); + return MakeFullBox("iinf", 0, Concat({payload, Concat(entries)})); +} + +// An 'iref' version 0 box with one 'cdsc' reference per {from, to} item pair. +Bytes MakeCdscIrefV0( + std::initializer_list> references) { + Bytes payload; + for (const auto& [from_item_id, to_item_id] : references) { + Bytes cdsc; + AppendUInt(from_item_id, 2, &cdsc); + AppendUInt(1, 2, &cdsc); // reference_count + AppendUInt(to_item_id, 2, &cdsc); + payload = Concat({payload, MakeBox("cdsc", cdsc)}); + } + return MakeFullBox("iref", 0, payload); +} + +struct IdatItem { + uint16_t item_id = 0; + uint32_t offset = 0; + uint32_t length = 0; + uint16_t data_reference_index = 0; +}; + +// An 'iloc' version 1 box in which every item is one extent of the 'idat' box +// (construction method 1), with 4-byte offsets and lengths. +Bytes MakeIdatIlocV1(std::initializer_list items) { + Bytes payload; + AppendUInt(0x4400, 2, &payload); // offset_size 4, length_size 4, others 0. + AppendUInt(items.size(), 2, &payload); + for (const IdatItem& item : items) { + AppendUInt(item.item_id, 2, &payload); + AppendUInt(1, 2, &payload); // construction_method + AppendUInt(item.data_reference_index, 2, &payload); + AppendUInt(1, 2, &payload); // extent_count + AppendUInt(item.offset, 4, &payload); + AppendUInt(item.length, 4, &payload); + } + return MakeFullBox("iloc", 1, payload); +} + +// Builds a HEIF file whose primary item 1 is described by item 2, a 'mime' item +// stored in 'idat' after `idat_prefix`. +Bytes MakeHeifWithIdatItem(std::string_view content_type, + std::string_view idat_prefix, + std::string_view item_data, + uint16_t data_reference_index) { + Bytes idat; + AppendString(idat_prefix, &idat); + AppendString(item_data, &idat); + const IdatItem item = {2, static_cast(idat_prefix.size()), + static_cast(item_data.size()), + data_reference_index}; + return Concat( + {MakeFtypBox(), + MakeFullBox("meta", 0, + Concat({MakePitmV0(1), + MakeIinfV0({MakeInfeV2(1, "hvc1", ""), + MakeInfeV2(2, "mime", content_type)}), + MakeCdscIrefV0({{2, 1}}), MakeIdatIlocV1({item}), + MakeBox("idat", idat)})), + MakeMpvdBox()}); +} + +TEST(HeifContainerSplitterTest, ExtractsXmpFromHeicMotionPhoto) { + const Bytes file = ReadTestFile(kHeicMotionPhoto); + ASSERT_FALSE(file.empty()); + + std::vector blocks = SplitHeif(file); + + ASSERT_EQ(blocks.size(), 1u); + EXPECT_EQ(blocks[0].type, "XMP"); + EXPECT_EQ(blocks[0].format_identifier, "standard.xmp"); + // The XMP of this file is item 2, a single 1111-byte extent at 27616. + EXPECT_EQ(blocks[0].offset, 27616u); + EXPECT_TRUE(blocks[0].bytes == + Bytes(file.begin() + 27616, file.begin() + 27616 + 1111)); +} + +TEST(HeifContainerSplitterTest, IgnoresHeicWithoutMpvdBox) { + const Bytes file = ReadTestFile(kHeicWithoutVideo); + ASSERT_FALSE(file.empty()); + + EXPECT_TRUE(SplitHeif(file).empty()); +} + +TEST(HeifContainerSplitterTest, IgnoresJpeg) { + const Bytes file = ReadTestFile("motion_photo_single_video_track.MP.jpg"); + ASSERT_FALSE(file.empty()); + + EXPECT_TRUE(SplitHeif(file).empty()); +} + +TEST(HeifContainerSplitterTest, IgnoresNullDataSource) { + EXPECT_TRUE(CreateHeifContainerSplitter()->Split(nullptr, 1024).empty()); +} + +TEST(HeifContainerSplitterTest, ExtractsXmpStoredInIdat) { + const std::string_view prefix = "pad"; + const std::string_view xmp = "stored in idat"; + const Bytes file = MakeHeifWithIdatItem(kXmpContentType, prefix, xmp, + /*data_reference_index=*/0); + + std::vector blocks = SplitHeif(file); + + ASSERT_EQ(blocks.size(), 1u); + EXPECT_EQ(AsString(blocks[0].bytes), std::string(xmp)); + // 'idat' is the last box before 'mpvd', so its payload ends there. + const size_t idat_payload_offset = + file.size() - MakeMpvdBox().size() - prefix.size() - xmp.size(); + EXPECT_EQ(blocks[0].offset, idat_payload_offset + prefix.size()); +} + +TEST(HeifContainerSplitterTest, IgnoresNonXmpMimeItem) { + const Bytes file = + MakeHeifWithIdatItem("application/octet-stream", "", "", + /*data_reference_index=*/0); + + EXPECT_TRUE(SplitHeif(file).empty()); +} + +TEST(HeifContainerSplitterTest, IgnoresXmpStoredInAnotherFile) { + const Bytes file = MakeHeifWithIdatItem(kXmpContentType, "", "", + /*data_reference_index=*/1); + + EXPECT_TRUE(SplitHeif(file).empty()); +} + +TEST(HeifContainerSplitterTest, PrefersXmpThatDescribesPrimaryItem) { + const std::string_view other_xmp = "other"; + const std::string_view primary_xmp = "primary"; + Bytes idat; + AppendString(other_xmp, &idat); + AppendString(primary_xmp, &idat); + const IdatItem other_item = {2, 0, static_cast(other_xmp.size())}; + const IdatItem primary_item = {3, static_cast(other_xmp.size()), + static_cast(primary_xmp.size())}; + // XMP item 2 comes first but describes item 4; XMP item 3 describes the + // primary item 1. + const Bytes file = Concat( + {MakeFtypBox(), + MakeFullBox("meta", 0, + Concat({MakePitmV0(1), + MakeIinfV0({MakeInfeV2(1, "hvc1", ""), + MakeInfeV2(2, "mime", kXmpContentType), + MakeInfeV2(3, "mime", kXmpContentType)}), + MakeCdscIrefV0({{2, 4}, {3, 1}}), + MakeIdatIlocV1({other_item, primary_item}), + MakeBox("idat", idat)})), + MakeMpvdBox()}); + + std::vector blocks = SplitHeif(file); + + ASSERT_EQ(blocks.size(), 1u); + EXPECT_EQ(AsString(blocks[0].bytes), std::string(primary_xmp)); +} + +// Covers 32-bit item IDs ('pitm' v1, 'infe' v3, 'iref' v1), 'iloc' v2 with +// 8-byte fields, base offsets and extent indices, and multi-extent items. +TEST(HeifContainerSplitterTest, ReadsMultiExtentItemWithWideFields) { + constexpr uint32_t kPrimaryItemId = 0x10001; + constexpr uint32_t kXmpItemId = 0x10002; + const std::string_view first_part = "first|"; + const std::string_view gap = "not part of the item"; + const std::string_view second_part = "second"; + Bytes mdat; + AppendString(first_part, &mdat); + AppendString(gap, &mdat); + AppendString(second_part, &mdat); + + Bytes pitm; + AppendUInt(kPrimaryItemId, 4, &pitm); + Bytes image_infe; + AppendUInt(kPrimaryItemId, 4, &image_infe); + AppendUInt(0, 2, &image_infe); // item_protection_index + AppendString("hvc1", &image_infe); + AppendCString("", &image_infe); + Bytes xmp_infe; + AppendUInt(kXmpItemId, 4, &xmp_infe); + AppendUInt(0, 2, &xmp_infe); // item_protection_index + AppendString("mime", &xmp_infe); + AppendCString("XMP", &xmp_infe); + AppendCString(kXmpContentType, &xmp_infe); + Bytes iinf; + AppendUInt(2, 4, &iinf); // entry_count + iinf = Concat({iinf, MakeFullBox("infe", 3, image_infe), + MakeFullBox("infe", 3, xmp_infe)}); + Bytes cdsc; + AppendUInt(kXmpItemId, 4, &cdsc); + AppendUInt(1, 2, &cdsc); // reference_count + AppendUInt(kPrimaryItemId, 4, &cdsc); + // The size of 'meta' does not depend on the base offset it records. + auto make_meta = [&](uint32_t base_offset) { + Bytes iloc; + AppendUInt(0x8844, 2, &iloc); // offset 8, length 8, base 4, index 4. + AppendUInt(1, 4, &iloc); // item_count + AppendUInt(kXmpItemId, 4, &iloc); + AppendUInt(0, 2, &iloc); // construction_method: file offsets. + AppendUInt(0, 2, &iloc); // data_reference_index + AppendUInt(base_offset, 4, &iloc); + AppendUInt(2, 2, &iloc); // extent_count + AppendUInt(0, 4, &iloc); // extent_index + AppendUInt(0, 8, &iloc); + AppendUInt(first_part.size(), 8, &iloc); + AppendUInt(0, 4, &iloc); // extent_index + AppendUInt(first_part.size() + gap.size(), 8, &iloc); + AppendUInt(second_part.size(), 8, &iloc); + return MakeFullBox( + "meta", 0, + Concat({MakeFullBox("pitm", 1, pitm), MakeFullBox("iinf", 1, iinf), + MakeFullBox("iref", 1, MakeBox("cdsc", cdsc)), + MakeFullBox("iloc", 2, iloc)})); + }; + const Bytes ftyp = MakeFtypBox(); + const uint32_t mdat_payload_offset = + static_cast(ftyp.size() + make_meta(0).size() + 8); + const Bytes file = Concat({ftyp, make_meta(mdat_payload_offset), + MakeBox("mdat", mdat), MakeMpvdBox()}); + + std::vector blocks = SplitHeif(file); + + ASSERT_EQ(blocks.size(), 1u); + EXPECT_EQ(AsString(blocks[0].bytes), + std::string(first_part) + std::string(second_part)); + EXPECT_EQ(blocks[0].offset, static_cast(mdat_payload_offset)); +} + +TEST(HeifContainerSplitterTest, SurvivesTruncatedInput) { + const Bytes file = ReadTestFile(kHeicMotionPhoto); + ASSERT_FALSE(file.empty()); + + for (size_t size = 1; size < file.size(); size += 7) { + std::vector blocks = SplitHeif(file.data(), size); + EXPECT_TRUE(blocks.size() <= 1u); + for (const RawMetadataBlock& block : blocks) { + EXPECT_TRUE(block.offset + block.bytes.size() <= size); + } + } +} + +TEST(HeifContainerSplitterTest, SurvivesCorruptedBoxes) { + Bytes file = ReadTestFile(kHeicMotionPhoto); + ASSERT_FALSE(file.empty()); + + // Corrupts each byte of the 'ftyp' and 'meta' boxes, which end at 434. + for (size_t i = 0; i < 434; ++i) { + const uint8_t original = file[i]; + for (uint8_t value : {uint8_t{0x00}, uint8_t{0xFF}, + static_cast(original ^ 0x80)}) { + file[i] = value; + std::vector blocks = SplitHeif(file); + EXPECT_TRUE(blocks.size() <= 1u); + for (const RawMetadataBlock& block : blocks) { + EXPECT_TRUE(block.offset + block.bytes.size() <= file.size()); + } + } + file[i] = original; + } +} + +TEST(MetadataEngineHeifTest, DetectsHeifMotionPhoto) { + const Bytes file = ReadTestFile(kHeicMotionPhoto); + ASSERT_FALSE(file.empty()); + + EXPECT_TRUE(IsMotionPhotoPerEngine(file, FileType::kHeic)); + EXPECT_TRUE(IsMotionPhotoPerEngine(file, FileType::kAvif)); +} + +TEST(MetadataEngineHeifTest, RejectsHeicWithoutVideo) { + const Bytes file = ReadTestFile(kHeicWithoutVideo); + ASSERT_FALSE(file.empty()); + + EXPECT_FALSE(IsMotionPhotoPerEngine(file, FileType::kHeic)); +} + +} // namespace +} // namespace motion_photo +} // namespace libmotionphoto diff --git a/lib/core/motion_photo/tests/libmotionphoto_api_test.cc b/lib/core/motion_photo/tests/libmotionphoto_api_test.cc index f312bd2..f853a3e 100644 --- a/lib/core/motion_photo/tests/libmotionphoto_api_test.cc +++ b/lib/core/motion_photo/tests/libmotionphoto_api_test.cc @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -60,6 +61,30 @@ TEST(LibMotionPhotoApiTest, ParseMotionPhotoFromFileHeic) { EXPECT_TRUE(metadata.video_length > 0); } +TEST(LibMotionPhotoApiTest, ParseMotionPhotoHeicWithoutOutputMetadata) { + std::string file_path = + "lib/core/motion_photo/testdata/motion_photo_single_video_track.MP.heic"; + EXPECT_TRUE(ParseMotionPhotoFromFile(file_path, nullptr)); + + std::ifstream file(file_path, std::ios::binary); + std::vector buffer((std::istreambuf_iterator(file)), + std::istreambuf_iterator()); + ASSERT_FALSE(buffer.empty()); + EXPECT_TRUE( + ParseMotionPhotoFromMemory(buffer.data(), buffer.size(), nullptr)); +} + +TEST(LibMotionPhotoApiTest, ParseMotionPhotoHeicWithoutVideo) { + // Motion photo XMP, but no 'mpvd' box: the video was removed. + std::string file_path = + "lib/core/motion_photo/testdata/motion_photo_photo_only.heic"; + EXPECT_FALSE(ParseMotionPhotoFromFile(file_path, nullptr)); + + MotionPhotoMetadata metadata; + EXPECT_FALSE(ParseMotionPhotoFromFile(file_path, &metadata)); + EXPECT_FALSE(metadata.is_motion_photo); +} + // 2. Test ParseMotionPhotoFromFd TEST(LibMotionPhotoApiTest, ParseMotionPhotoFromFd) { std::string file_path = "lib/core/motion_photo/testdata/motion_photo_single_video_track.MP.jpg";