From bcc0f1da2bb16858f22eb1cc0da12fd9e7a35530 Mon Sep 17 00:00:00 2001 From: Junyi Ou Date: Fri, 14 Aug 2026 11:30:34 -0400 Subject: [PATCH 1/2] feat(ebml-iterable): add incremental decoder --- Cargo.lock | 7 +- Cargo.toml | 5 + crates/ebml-iterable/Cargo.toml | 23 + crates/ebml-iterable/LICENSE | 21 + crates/ebml-iterable/README.md | 89 ++ .../specification-derive/Cargo.toml | 18 + .../specification-derive/src/ast.rs | 167 ++++ .../specification-derive/src/attr.rs | 488 +++++++++++ .../specification-derive/src/easy_ebml.rs | 124 +++ .../specification-derive/src/lib.rs | 143 ++++ .../specification-derive/src/pathing.rs | 86 ++ crates/ebml-iterable/specification/Cargo.toml | 11 + .../specification/src/empty_spec.rs | 104 +++ crates/ebml-iterable/specification/src/lib.rs | 233 ++++++ crates/ebml-iterable/src/errors.rs | 363 +++++++++ crates/ebml-iterable/src/lib.rs | 58 ++ crates/ebml-iterable/src/nonblocking.rs | 59 ++ crates/ebml-iterable/src/spec_util.rs | 89 ++ crates/ebml-iterable/src/specs.rs | 11 + crates/ebml-iterable/src/tag_decoder.rs | 406 ++++++++++ crates/ebml-iterable/src/tag_iterator.rs | 597 ++++++++++++++ .../ebml-iterable/src/tag_iterator_async.rs | 191 +++++ crates/ebml-iterable/src/tag_iterator_util.rs | 125 +++ crates/ebml-iterable/src/tag_parse.rs | 110 +++ crates/ebml-iterable/src/tag_writer.rs | 760 ++++++++++++++++++ crates/ebml-iterable/src/tools.rs | 523 ++++++++++++ .../ebml-iterable/tests/corrupt_data_tests.rs | 239 ++++++ .../tests/derive_spec_compile.rs | 45 ++ .../derive_spec_compile_with_hierarchy.rs | 51 ++ crates/ebml-iterable/tests/spec_write_read.rs | 375 +++++++++ crates/ebml-iterable/tests/test_spec.rs | 214 +++++ 31 files changed, 5729 insertions(+), 6 deletions(-) create mode 100644 crates/ebml-iterable/Cargo.toml create mode 100644 crates/ebml-iterable/LICENSE create mode 100644 crates/ebml-iterable/README.md create mode 100644 crates/ebml-iterable/specification-derive/Cargo.toml create mode 100644 crates/ebml-iterable/specification-derive/src/ast.rs create mode 100644 crates/ebml-iterable/specification-derive/src/attr.rs create mode 100644 crates/ebml-iterable/specification-derive/src/easy_ebml.rs create mode 100644 crates/ebml-iterable/specification-derive/src/lib.rs create mode 100644 crates/ebml-iterable/specification-derive/src/pathing.rs create mode 100644 crates/ebml-iterable/specification/Cargo.toml create mode 100644 crates/ebml-iterable/specification/src/empty_spec.rs create mode 100644 crates/ebml-iterable/specification/src/lib.rs create mode 100644 crates/ebml-iterable/src/errors.rs create mode 100644 crates/ebml-iterable/src/lib.rs create mode 100644 crates/ebml-iterable/src/nonblocking.rs create mode 100644 crates/ebml-iterable/src/spec_util.rs create mode 100644 crates/ebml-iterable/src/specs.rs create mode 100644 crates/ebml-iterable/src/tag_decoder.rs create mode 100644 crates/ebml-iterable/src/tag_iterator.rs create mode 100644 crates/ebml-iterable/src/tag_iterator_async.rs create mode 100644 crates/ebml-iterable/src/tag_iterator_util.rs create mode 100644 crates/ebml-iterable/src/tag_parse.rs create mode 100644 crates/ebml-iterable/src/tag_writer.rs create mode 100644 crates/ebml-iterable/src/tools.rs create mode 100644 crates/ebml-iterable/tests/corrupt_data_tests.rs create mode 100644 crates/ebml-iterable/tests/derive_spec_compile.rs create mode 100644 crates/ebml-iterable/tests/derive_spec_compile_with_hierarchy.rs create mode 100644 crates/ebml-iterable/tests/spec_write_read.rs create mode 100644 crates/ebml-iterable/tests/test_spec.rs diff --git a/Cargo.lock b/Cargo.lock index 7adf897e5..477034ee5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2198,9 +2198,8 @@ dependencies = [ [[package]] name = "ebml-iterable" version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5173ac3752f08b526a6991509615e1a345b221ec3c58c7633433e8c9582312" dependencies = [ + "bytes 1.12.1", "ebml-iterable-specification", "ebml-iterable-specification-derive", "futures", @@ -2209,14 +2208,10 @@ dependencies = [ [[package]] name = "ebml-iterable-specification" version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f56467af159a98735d44231f53eaa505e919e6003266f103b99649a93f106784" [[package]] name = "ebml-iterable-specification-derive" version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b066b81018300fdce40f71c4db355a102699324af96fad28f25ab1b5f87de066" dependencies = [ "ebml-iterable-specification", "proc-macro2 1.0.106", diff --git a/Cargo.toml b/Cargo.toml index e5eb35315..5a0751b03 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,10 @@ members = [ "testsuite", "tools/generate-openapi", ] +exclude = [ + "crates/ebml-iterable/specification", + "crates/ebml-iterable/specification-derive", +] default-members = [ "devolutions-agent", "devolutions-agent-updater", @@ -32,6 +36,7 @@ codegen-units = 1 lto = true [patch.crates-io] +ebml-iterable = { path = "crates/ebml-iterable" } tracing-appender = { git = "https://github.com/CBenoit/tracing.git", rev = "42097daf92e683cf18da7639ddccb056721a796c" } [workspace.lints.rust] diff --git a/crates/ebml-iterable/Cargo.toml b/crates/ebml-iterable/Cargo.toml new file mode 100644 index 000000000..1fe3e469f --- /dev/null +++ b/crates/ebml-iterable/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "ebml-iterable" +version = "0.6.3" +authors = ["Austin Blake "] +edition = "2018" +description = "This crate provides an iterator over EBML encoded data. The items provided by the iterator are Tags as defined in EBML. The iterator is spec-agnostic and requires a specification implementing specific traits to read files. Typically, you would only use this crate to implement a custom specification - most often you would prefer a crate providing an existing specification, like `webm-iterable`." +readme = "README.md" +license = "MIT" +keywords = ["ebml", "encoding", "parser", "serialization"] +categories = ["parsing", "data-structures", "encoding", "multimedia::encoding"] +homepage = "https://github.com/austinleroy/ebml-iterable" +repository = "https://github.com/austinleroy/ebml-iterable" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +bytes = "1" +ebml-iterable-specification = { version = "=0.4.0", path = "specification" } +ebml-iterable-specification-derive = { version = "=0.4.0", path = "specification-derive", optional = true } +futures = { version = "0.3.28", optional = true } + +[features] +derive-spec = ["ebml-iterable-specification-derive"] diff --git a/crates/ebml-iterable/LICENSE b/crates/ebml-iterable/LICENSE new file mode 100644 index 000000000..4e97feb24 --- /dev/null +++ b/crates/ebml-iterable/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Austin Blake + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/ebml-iterable/README.md b/crates/ebml-iterable/README.md new file mode 100644 index 000000000..2b78e03a8 --- /dev/null +++ b/crates/ebml-iterable/README.md @@ -0,0 +1,89 @@ +[EBML][EBML] stands for Extensible Binary Meta-Language and is somewhat of a +binary version of XML. It's used for container formats like [WebM][webm] or +[MKV][mkv]. + +# Devolutions fork + +Devolutions Gateway vendors this crate from Austin Blake's [`ebml-iterable`](https://github.com/austinleroy/ebml-iterable) commit `25303fa5a9107d8d28fa8458b0b982946c319135`. +The original work and this fork use the MIT license in [`LICENSE`](LICENSE). +This fork adds `TagDecoder`, which keeps incomplete EBML elements pending while a recording grows. + +> IMPORTANT: The iterator contained in this crate is spec-agnostic and requires a specification implementing the `EbmlSpecification` and `EbmlTag` traits to read files. Typically, you would only use this crate to implement a custom specification - most often you would prefer a crate providing an existing specification, like [webm-iterable][webm-iterable]. + +```Cargo.toml +[dependencies] +ebml-iterable = "0.6.3" +``` + +# Usage + +The `TagIterator` struct implements Rust's standard [Iterator][rust-iterator] trait. +This struct can be created with the `new` function on any source that implements the standard [Read][rust-read] trait. The iterator outputs `TSpec` objects based on the defined specification and the tag data. + +> Note: The `with_capacity` method can be used to construct a `TagIterator` with a specified default buffer size. This is only useful as a microoptimization to memory management if you know the maximum tag size of the file you're reading. + +The data in the tag can then be modified as desired (encryption, compression, etc.) and reencoded using the `TagWriter` struct. This struct can be created with the `new` function on any source that implements the standard [Write][rust-write] trait. Once created, this struct can encode EBML using the `write` method on any objects that implement `EbmlSpecification` and `EbmlTag` regardless of whether they came from a `TagIterator`. This will emit binary EBML to the underlying `Write` destination. + +## Master Enum + +Most tag types contain their data directly, but there is a category of tag in EBML called `Master` which contains other tags. This crate contains an enumeration of three different classifications of master tags: + + * `Start` is a marker for the beginning of a "master" tag. + * `End` is a marker for the end of a "master" tag. + * `Full(children)` is a complete tag that includes all child tags of the `Master` tag. This is only emitted by the `TagIterator` for tag types passed in via `tags_to_buffer`. + +## TagDataType + +```rs +pub enum TagDataType { + Master, + UnsignedInt, + Integer, + Utf8, + Binary, + Float, +} +``` + +TagDataType is an enum containing the possible data types stored within a tag. The relationship between the tag variant and the type of data contained in the tag directly corresponds is defined by whichever specification is in use. Because EBML is binary, the correct specification is required to parse tag content. + + * Master: A complete master tag containing any number of child tags. + * UnsignedInt: An unsigned integer. + * Integer: A signed integer. + * Utf8: A Unicode text string. Note that the [EBML spec][rfc8794] includes a separate element type for ASCII. Given that ASCII is a subset of Utf8, this library currently parses and encodes both types using the same Utf8 logic. + * Binary: Binary data, otherwise uninterpreted. + * Float: IEEE-754 floating point number. + +> Note: This library made a conscious decision to not parse "Date" elements from EBML due to lack of built-in support for dates in Rust. Specification implementations should treat Date elements as Binary so that consumers have the option of parsing the unaltered data using their library of choice, if needed. + +# Specification Implementation + +Any specification based on EBML can use this library to parse or write binary data. Writing needs nothing special (if you use the `write_raw()` method), but parsing requires a struct implementing the `EbmlSpecification` and `EbmlTag` traits. These traits currently have a large number of methods to implement and need consistent implementations to avoid errors, so any implementation attempt is recommended to use the `"derive-spec"` feature flag in this crate and using the provided macro. Custom specification implementations can refer to [webm-iterable][webm-iterable] as an example. + +# Features + +There is currently only one optional feature in this crate, but that may change over time as needs arise. + +* **derive-spec** - + When enabled, this provides a macro to simplify implementations of the `EbmlSpecification` and `EbmlTag` traits. This introduces dependencies on [`syn`](https://crates.io/crates/syn), [`quote`](https://crates.io/crates/quote), and [`proc-macro2`](https://crates.io/crates/proc-macro2), so expect compile times to increase a little. + + +# State of this project + +Parsing and writing complete files should both work. Streaming (using tags of unknown size) should now also be supported, as of version 0.4.0. If something is broken, please create [an issue][new-issue]. + +Any additional feature requests can also be submitted as [an issue][new-issue]. + +# Author + +[Austin Blake](https://github.com/austinleroy) + +[EBML]: http://ebml.sourceforge.net/ +[webm]: https://www.webmproject.org/ +[mkv]: http://www.matroska.org/technical/specs/index.html +[rfc8794]: https://datatracker.ietf.org/doc/rfc8794/ +[rust-iterator]: https://doc.rust-lang.org/std/iter/trait.Iterator.html +[rust-read]: https://doc.rust-lang.org/std/io/trait.Read.html +[rust-write]: https://doc.rust-lang.org/std/io/trait.Write.html +[new-issue]: https://github.com/austinleroy/ebml-iterable/issues +[webm-iterable]: https://github.com/austinleroy/webm-iterable diff --git a/crates/ebml-iterable/specification-derive/Cargo.toml b/crates/ebml-iterable/specification-derive/Cargo.toml new file mode 100644 index 000000000..d2a84389b --- /dev/null +++ b/crates/ebml-iterable/specification-derive/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "ebml-iterable-specification-derive" +version = "0.4.0" +authors = ["Austin Blake "] +edition = "2018" +description = "Provides macros for implementing `EbmlSpecification` for the `ebml-iterable` crate." +license = "MIT" +homepage = "https://github.com/austinleroy/ebml-iterable" +repository = "https://github.com/austinleroy/ebml-iterable" + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = "1.0" +quote = "1.0" +syn = { version = "1.0", features = ["full"] } +ebml-iterable-specification = { version = "=0.4.0", path = "../specification" } \ No newline at end of file diff --git a/crates/ebml-iterable/specification-derive/src/ast.rs b/crates/ebml-iterable/specification-derive/src/ast.rs new file mode 100644 index 000000000..f05af7b31 --- /dev/null +++ b/crates/ebml-iterable/specification-derive/src/ast.rs @@ -0,0 +1,167 @@ +use std::collections::HashSet; + +use ebml_iterable_specification::TagDataType; +use quote::ToTokens; +use syn::spanned::Spanned; +use syn::{Error, Generics, Ident, ItemEnum, LitInt, Path, Result}; + +use crate::pathing::{EBMLPath, PathPart}; + +pub struct Enum<'a> { + pub original: &'a ItemEnum, + pub ident: Ident, + pub variants: Vec>, + pub generics: &'a Generics, +} + +pub struct Variant<'a> { + pub original: &'a syn::Variant, + pub ident: Ident, + pub id_attr: (u64, Attribute<'a>), + pub data_type_attr: (TagDataType, Path, Attribute<'a>), + pub path_attr: Option<(EBMLPath, Attribute<'a>)>, +} + +pub struct Attribute<'a> { + pub original: &'a syn::Attribute, +} + +impl<'a> Enum<'a> { + pub fn from_syn(node: &'a ItemEnum) -> Result { + let variant_names: HashSet<_> = node.variants.iter().map(|var| var.ident.clone()).collect(); + let variants = node + .variants + .iter() + .map(|node| Variant::from_syn(node, &variant_names)) + .collect::>()?; + + Ok(Enum { + original: node, + ident: node.ident.clone(), + variants, + generics: &node.generics, + }) + } +} + +impl<'a> Variant<'a> { + fn from_syn(node: &'a syn::Variant, variant_names: &HashSet) -> Result { + let mut id_attr: Option<(u64, Attribute<'a>)> = None; + let mut data_type_attr: Option<(TagDataType, Path, Attribute<'a>)> = None; + let mut path_attr: Option<(EBMLPath, Attribute<'a>)> = None; + + for attr in &node.attrs { + if attr.path.is_ident("id") { + if id_attr.is_some() { + return Err(Error::new_spanned( + node, + format!("duplicate {} attribute", attr.to_token_stream()), + )); + } + let val = attr.parse_args::()?.base10_parse::()?; + id_attr = Some((val, Attribute { original: attr })); + } else if attr.path.is_ident("data_type") { + if data_type_attr.is_some() { + return Err(Error::new_spanned( + node, + format!("duplicate {} attribute", attr.to_token_stream()), + )); + } + + let val = attr.parse_args::().map_err(|err| { + Error::new( + err.span(), + format!("{} requires `ebml_iterable::TagDataType`", attr.to_token_stream()), + ) + })?; + let data_type_name = val.segments.iter().last(); + if data_type_name.is_none() { + return Err(Error::new_spanned( + val, + format!("{} requires `ebml_iterable::TagDataType`", attr.to_token_stream()), + )); + } + let data_type_name = data_type_name.unwrap().ident.to_string(); + let data_type_val = if data_type_name == "UnsignedInt" { + TagDataType::UnsignedInt + } else if data_type_name == "Integer" { + TagDataType::Integer + } else if data_type_name == "Utf8" { + TagDataType::Utf8 + } else if data_type_name == "Binary" { + TagDataType::Binary + } else if data_type_name == "Float" { + TagDataType::Float + } else if data_type_name == "Master" { + TagDataType::Master + } else { + return Err(Error::new_spanned( + val, + format!("unrecognized `ebml_iterable::TagDataType` value: {data_type_name}"), + )); + }; + data_type_attr = Some((data_type_val, val, Attribute { original: attr })); + } else if attr.path.is_ident("doc_path") { + if path_attr.is_some() { + return Err(Error::new_spanned( + node, + format!("duplicate {} attribute", attr.to_token_stream()), + )); + } + let path = attr.parse_args::().map_err(|err| { + Error::new(err.span(), format!("{} must be a path string", attr.to_token_stream())) + })?; + let mut last_was_global = false; + for path_part in &path.parts { + match path_part { + PathPart::Ident(id) => { + last_was_global = false; + variant_names + .get(id) + .ok_or(Error::new(id.span(), format!("Unknown variant [{id}] in path")))?; + } + PathPart::Global((_, max)) => { + if matches!(max, Some(0)) { + return Err(Error::new(attr.span(), "Global maximum cannot be 0".to_string())); + } + if last_was_global { + return Err(Error::new( + attr.span(), + "Cannot use GlobalPlaceholders back-to-back in path".to_string(), + )); + } + last_was_global = true; + } + } + } + path_attr = Some((path, Attribute { original: attr })) + } + } + + let id_attr = if let Some(id_attr) = id_attr { + id_attr + } else { + return Err(Error::new_spanned( + node, + "#[id] attribute is required when using #[ebml_specification] attribute", + )); + }; + + let data_type_attr = if let Some(data_type_attr) = data_type_attr { + data_type_attr + } else { + return Err(Error::new_spanned( + node, + "#[data_type] attribute is required when using #[ebml_specification] attribute", + )); + }; + + Ok(Variant { + original: node, + ident: node.ident.clone(), + id_attr, + data_type_attr, + path_attr, + }) + } +} diff --git a/crates/ebml-iterable/specification-derive/src/attr.rs b/crates/ebml-iterable/specification-derive/src/attr.rs new file mode 100644 index 000000000..41f658386 --- /dev/null +++ b/crates/ebml-iterable/specification-derive/src/attr.rs @@ -0,0 +1,488 @@ +use std::collections::HashMap; +use std::str::FromStr; + +use ebml_iterable_specification::TagDataType; +use ebml_iterable_specification::TagDataType::Master; +use proc_macro2::TokenStream; +use quote::{quote, quote_spanned, ToTokens}; +use syn::spanned::Spanned; +use syn::{Attribute, Error, Fields, FieldsUnnamed, Ident, ItemEnum, Path, Result, Variant, Visibility}; + +use super::ast::Enum; +use super::pathing::PathPart; + +pub fn impl_ebml_specification(original: &mut ItemEnum) -> Result { + let tag_data_type = spanned_tag_data_type(original); + original.variants.push(syn::parse2::(quote! { + #[id(0xbf)] + #[data_type(#tag_data_type::Binary)] + #[doc_path((1-))] + Crc32 + })?); + original.variants.push(syn::parse2::(quote! { + #[id(0xec)] + #[data_type(#tag_data_type::Binary)] + #[doc_path((-))] + Void + })?); + + let input = Enum::from_syn(original)?; + + let mut used_ids = HashMap::::new(); + for var in &input.variants { + if let Some(original) = used_ids.insert(var.id_attr.0, var.original) { + let mut err = Error::new_spanned( + var.original, + format!("duplicate {} detected", var.id_attr.1.original.to_token_stream()), + ); + err.combine(Error::new_spanned( + original, + format!("{} already used previously", var.id_attr.1.original.to_token_stream()), + )); + return Err(err); + } + } + + let map: HashMap<_, _> = input.variants.iter().map(|var| (&var.ident, var)).collect(); + for origin in &input.variants { + if !matches!(origin.data_type_attr.0, TagDataType::Master) && origin.path_attr.is_some() { + validate_path(origin, &map)?; + } + } + + let ebml_specification_impl = get_impl(input)?; + let modified_orig = modify_orig(original)?; + + Ok(quote!( + #modified_orig + + #ebml_specification_impl + )) +} + +// verify all parents are Master type elements and their path lines up with this item's path +fn validate_path(origin: &crate::ast::Variant, variants_map: &HashMap<&Ident, &crate::ast::Variant>) -> Result<()> { + // Only validate the element if it has a path attribute + if let Some(path_parts) = origin.path_attr.as_ref().map(|(path, _)| &path.parts) { + // Only validate if there is a specific parent element + if let Some(parent) = path_parts + .iter() + .rev() + .filter_map(|p| { + if let PathPart::Ident(ident) = p { + Some(ident) + } else { + None + } + }) + .next() + { + let parent = *variants_map.get(parent).unwrap(); + if parent.data_type_attr.0 != Master { + return Err(Error::new_spanned(parent.original, "Parents must be of Master type")); + } + + if let Some((parent_path, _)) = parent.path_attr.as_ref() { + for i in 0..parent_path.parts.len() { + if parent_path.parts[i] != path_parts[i] { + return Err(Error::new_spanned( + origin.original, + format!( + "Path segment [{}] did not align with parent [{}] path.", + path_parts[i], parent.ident + ), + )); + } + } + validate_path(parent, variants_map)?; + } + } + } + + Ok(()) +} + +fn modify_orig(original: &mut ItemEnum) -> Result { + let spanned_master_enum = spanned_master_enum(original); + for var in original.variants.iter_mut() { + let data_type_attribute: &Attribute = var + .attrs + .iter() + .find(|a| a.path.is_ident("data_type")) + .expect("#[data_type()] attribute required for variants under #[ebml_specification]"); + + let data_type_path = data_type_attribute.parse_args::().map_err(|err| { + Error::new( + err.span(), + format!( + "{} requires `ebml_iterable::TagDataType`", + data_type_attribute.to_token_stream() + ), + ) + })?; + let data_type = get_last_path_ident(&data_type_path).ok_or_else(|| { + Error::new_spanned( + data_type_attribute.clone(), + format!( + "{} requires `ebml_iterable::TagDataType`", + data_type_attribute.to_token_stream() + ), + ) + })?; + + let data_type = if data_type == "Master" { + let orig_ident = &original.ident; + quote!( (#spanned_master_enum<#orig_ident>) ) + } else if data_type == "UnsignedInt" { + quote!((u64)) + } else if data_type == "Integer" { + quote!((i64)) + } else if data_type == "Utf8" { + quote!((String)) + } else if data_type == "Binary" { + quote!((::std::vec::Vec)) + } else if data_type == "Float" { + quote!((f64)) + } else { + return Err(Error::new_spanned( + data_type_attribute.clone(), + format!("unknown data_type \"{data_type}\""), + )); + }; + + var.attrs + .retain(|a| !(a.path.is_ident("id") || a.path.is_ident("data_type") || a.path.is_ident("doc_path"))); + var.fields = Fields::Unnamed(syn::parse2::(data_type)?); + } + original + .variants + .push(syn::parse_str::("RawTag(u64, ::std::vec::Vec)")?); + + Ok(quote!(#original)) +} + +fn get_impl(input: Enum) -> Result { + let ty = &input.ident; + let spanned_master_enum = spanned_master_enum(input.original); + + let get_tag_data_type = input.variants.iter().map(|var: &crate::ast::Variant| { + let id = &var.id_attr.0; + let data_type = &var.data_type_attr.1; + + quote_spanned! { var.data_type_attr.2.original.span() => + #id => Some(#data_type), + } + }); + + let get_id = input.variants.iter().map(|var: &crate::ast::Variant| { + let name = &var.ident; + let id = &var.id_attr.0; + + quote_spanned! { var.id_attr.1.original.span() => + #ty::#name(_) => #id, + } + }); + + let get_tag = |ret_val: String| { + move |var: &crate::ast::Variant| { + let name = &var.ident; + let id = &var.id_attr.0; + let ret_val = TokenStream::from_str(&ret_val) + .expect("Misuse of get_tag function in ebml_iterable_specification_derive_attr"); + + quote_spanned! { var.original.span() => + #id => Some(#ty::#name(#ret_val)), + } + } + }; + + let path_part = spanned_path_part(input.original); + let variant_map: HashMap<_, _> = input.variants.iter().map(|var| (&var.ident, var)).collect(); + let get_path_by_id = input.variants.iter().filter_map(|v| match v.path_attr.as_ref() { + None => None, + Some(path) => { + let id = &v.id_attr.0; + let path_array: Vec = path + .0 + .parts + .iter() + .map(|p| match p { + PathPart::Ident(ident) => { + let id = variant_map.get(&ident).map(|v| v.id_attr.0).unwrap(); + quote_spanned! { path.1.original.span() => #path_part::Id(#id) } + } + PathPart::Global((min, max)) => { + let min_tokens = if let Some(min) = min { + quote! {Some(#min)} + } else { + quote! {None} + }; + let max_tokens = if let Some(max) = max { + quote! {Some(#max)} + } else { + quote! {None} + }; + quote_spanned! { path.1.original.span() => #path_part::Global((#min_tokens, #max_tokens)) } + } + }) + .collect(); + Some(quote_spanned! { v.original.span() => + #id => &[#(#path_array),*], + }) + } + }); + + let get_unsigned_int_tag = input + .variants + .iter() + .filter(|v| matches!(&v.data_type_attr.0, TagDataType::UnsignedInt)) + .map(get_tag(String::from("data"))); + + let get_signed_int_tag = input + .variants + .iter() + .filter(|v| matches!(&v.data_type_attr.0, TagDataType::Integer)) + .map(get_tag(String::from("data"))); + + let get_utf8_tag = input + .variants + .iter() + .filter(|v| matches!(&v.data_type_attr.0, TagDataType::Utf8)) + .map(get_tag(String::from("data"))); + + let get_binary_tag = input + .variants + .iter() + .filter(|v| matches!(&v.data_type_attr.0, TagDataType::Binary)) + .map(get_tag(String::from("data.to_vec()"))); + + let get_float_tag = input + .variants + .iter() + .filter(|v| matches!(&v.data_type_attr.0, TagDataType::Float)) + .map(get_tag(String::from("data"))); + + let get_master_tag = input + .variants + .iter() + .filter(|v| matches!(&v.data_type_attr.0, TagDataType::Master)) + .map(get_tag(String::from("data"))); + + let as_data = |var: &crate::ast::Variant| { + let name = &var.ident; + + quote! { + #ty::#name(val) => Some(val), + } + }; + + let as_unsigned_int = input + .variants + .iter() + .filter(|v| matches!(&v.data_type_attr.0, TagDataType::UnsignedInt)) + .map(as_data); + + let as_signed_int = input + .variants + .iter() + .filter(|v| matches!(&v.data_type_attr.0, TagDataType::Integer)) + .map(as_data); + + let as_utf8 = input + .variants + .iter() + .filter(|v| matches!(&v.data_type_attr.0, TagDataType::Utf8)) + .map(as_data); + + let as_binary = input + .variants + .iter() + .filter(|v| matches!(&v.data_type_attr.0, TagDataType::Binary)) + .map(as_data); + + let as_float = input + .variants + .iter() + .filter(|v| matches!(&v.data_type_attr.0, TagDataType::Float)) + .map(as_data); + + let as_master = input + .variants + .iter() + .filter(|v| matches!(&v.data_type_attr.0, TagDataType::Master)) + .map(as_data); + + let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl(); + let ebml_spec_trait = spanned_ebml_specification_trait(input.original); + let ebml_tag_trait = spanned_ebml_tag_trait(input.original); + let tag_data_type = spanned_tag_data_type(input.original); + + Ok(quote! { + impl #impl_generics #ebml_spec_trait <#ty> for #ty #ty_generics #where_clause { + fn get_tag_data_type(id: u64) -> Option<#tag_data_type> { + match id { + #(#get_tag_data_type)* + _ => None + } + } + + fn get_path_by_id(id: u64) -> &'static [#path_part] { + match id { + #(#get_path_by_id)* + _ => &[] + } + } + + fn get_unsigned_int_tag(id: u64, data: u64) -> Option<#ty> { + match id { + #(#get_unsigned_int_tag)* + _ => None + } + } + + fn get_signed_int_tag(id: u64, data: i64) -> Option<#ty> { + match id { + #(#get_signed_int_tag)* + _ => None + } + } + + fn get_utf8_tag(id: u64, data: String) -> Option<#ty> { + match id { + #(#get_utf8_tag)* + _ => None + } + } + + fn get_binary_tag(id: u64, data: &[u8]) -> Option<#ty> { + match id { + #(#get_binary_tag)* + _ => None + } + } + + fn get_float_tag(id: u64, data: f64) -> Option<#ty> { + match id { + #(#get_float_tag)* + _ => None + } + } + + fn get_master_tag(id: u64, data: #spanned_master_enum<#ty>) -> Option<#ty> { + match id { + #(#get_master_tag)* + _ => None + } + } + + fn get_raw_tag(id: u64, data: &[u8]) -> #ty { + #ty::RawTag(id, data.to_vec()) + } + } + + impl #impl_generics #ebml_tag_trait <#ty> for #ty #ty_generics #where_clause { + + fn get_id(&self) -> u64 { + match self { + #(#get_id)* + #ty::RawTag(id, _data) => *id, + } + } + + fn as_unsigned_int(&self) -> Option<&u64> { + match self { + #(#as_unsigned_int)* + _ => None, + } + } + + fn as_signed_int(&self) -> Option<&i64> { + match self { + #(#as_signed_int)* + _ => None, + } + } + + fn as_utf8(&self) -> Option<&str> { + match self { + #(#as_utf8)* + _ => None, + } + } + + fn as_binary(&self) -> Option<&[u8]> { + match self { + #(#as_binary)* + #ty::RawTag(_id, data) => Some(data), + _ => None, + } + } + + fn as_float(&self) -> Option<&f64> { + match self { + #(#as_float)* + _ => None, + } + } + + fn as_master(&self) -> Option<&#spanned_master_enum<#ty>> { + match self { + #(#as_master)* + _ => None, + } + } + } + }) +} + +fn spanned_ebml_iterable_specs(input: &ItemEnum) -> TokenStream { + let vis_span = match &input.vis { + Visibility::Public(vis) => Some(vis.pub_token.span()), + Visibility::Crate(vis) => Some(vis.crate_token.span()), + Visibility::Restricted(vis) => Some(vis.pub_token.span()), + Visibility::Inherited => None, + }; + let data_span = input.enum_token.span(); + let first_span = vis_span.unwrap_or(data_span); + quote_spanned!(first_span=> ebml_iterable::specs::) +} + +fn spanned_master_enum(input: &ItemEnum) -> TokenStream { + let path = spanned_ebml_iterable_specs(input); + let last_span = input.ident.span(); + let r#enum = quote_spanned!(last_span=> Master); + quote!(#path #r#enum) +} + +fn spanned_ebml_specification_trait(input: &ItemEnum) -> TokenStream { + let path = spanned_ebml_iterable_specs(input); + let last_span = input.ident.span(); + let spec = quote_spanned!(last_span=> EbmlSpecification); + quote!(#path #spec) +} + +fn spanned_ebml_tag_trait(input: &ItemEnum) -> TokenStream { + let path = spanned_ebml_iterable_specs(input); + let last_span = input.ident.span(); + let spec = quote_spanned!(last_span=> EbmlTag); + quote!(#path #spec) +} + +fn spanned_tag_data_type(input: &ItemEnum) -> TokenStream { + let path = spanned_ebml_iterable_specs(input); + let last_span = input.ident.span(); + let r#type = quote_spanned!(last_span=> TagDataType); + quote!(#path #r#type) +} + +fn spanned_path_part(input: &ItemEnum) -> TokenStream { + let path = spanned_ebml_iterable_specs(input); + let last_span = input.ident.span(); + let r#type = quote_spanned!(last_span=> PathPart); + quote!(#path #r#type) +} + +fn get_last_path_ident(path: &Path) -> Option<&Ident> { + let seg = path.segments.iter().last(); + seg.map(|seg| &seg.ident) +} diff --git a/crates/ebml-iterable/specification-derive/src/easy_ebml.rs b/crates/ebml-iterable/specification-derive/src/easy_ebml.rs new file mode 100644 index 000000000..6bb9159bf --- /dev/null +++ b/crates/ebml-iterable/specification-derive/src/easy_ebml.rs @@ -0,0 +1,124 @@ +use proc_macro2::TokenStream; +use quote::quote; +use syn::parse::{Parse, ParseBuffer, ParseStream}; +use syn::punctuated::Punctuated; +use syn::{AttrStyle, Attribute, Error, Ident, LitInt, Result, Token, Variant, Visibility}; + +use crate::pathing::{EBMLPath, PathPart}; + +pub struct EasyEBML { + attrs: Vec, + visibility: Visibility, + ident: Ident, + variants: Punctuated, +} + +impl Parse for EasyEBML { + fn parse(input: ParseStream) -> Result { + let attrs = input.call(Attribute::parse_outer)?; + let visibility: Visibility = input.parse()?; + input.parse::()?; + let ident = input.parse::()?; + let content: ParseBuffer; + syn::braced!(content in input); + let variants = content.parse_terminated(EasyEBMLVariant::parse)?; + Ok(Self { + attrs, + visibility, + ident, + variants, + }) + } +} + +impl EasyEBML { + pub fn implement(self) -> Result { + let EasyEBML { + attrs, + visibility, + ident, + variants, + } = self; + + let variants: Vec<_> = variants + .into_iter() + .map(EasyEBMLVariant::into_variant) + .collect::>()?; + + Ok(quote!( + #[ebml_iterable::specs::ebml_specification] + #(#attrs)* + #visibility enum #ident { + #(#variants),* + } + )) + } +} + +pub struct EasyEBMLVariant { + path: EBMLPath, + ty: Ident, + id: LitInt, +} + +impl EasyEBMLVariant { + pub fn into_variant(self) -> Result { + let EasyEBMLVariant { path, ty, id } = self; + let span = path.span; + let mut path: Vec = path.parts.into_iter().collect(); + let ident: Ident = match path + .pop() + .ok_or_else(|| Error::new(span, "easy_ebml enum variant must be at least: `Name: Type = id`"))? + { + PathPart::Ident(id) => Ok(id), + PathPart::Global(_) => Err(Error::new(span, "easy_ebml enum variant cannot end in global path")), + }?; + let mut attrs = vec![]; + attrs.push(Attribute { + pound_token: Default::default(), + style: AttrStyle::Outer, + bracket_token: Default::default(), + path: Ident::new("id", proc_macro2::Span::call_site()).into(), + tokens: quote!((#id)), + }); + attrs.push(Attribute { + pound_token: Default::default(), + style: AttrStyle::Outer, + bracket_token: Default::default(), + path: Ident::new("data_type", proc_macro2::Span::call_site()).into(), + tokens: quote!((TagDataType::#ty)), + }); + + if !path.is_empty() { + let mut tokens: Punctuated = Punctuated::new(); + for part in path { + tokens.push(part); + } + attrs.push(Attribute { + pound_token: Default::default(), + style: AttrStyle::Outer, + bracket_token: Default::default(), + path: Ident::new("doc_path", proc_macro2::Span::call_site()).into(), + tokens: quote!((#tokens)), + }); + } + + Ok(Variant { + attrs, + ident, + fields: syn::Fields::Unit, + discriminant: None, + }) + } +} + +impl Parse for EasyEBMLVariant { + fn parse(input: ParseStream) -> syn::Result { + let path = input.parse()?; + input.parse::()?; + let ty: Ident = input.parse()?; + input.parse::()?; + let id: LitInt = input.parse()?; + Ok(Self { path, ty, id }) + } +} diff --git a/crates/ebml-iterable/specification-derive/src/lib.rs b/crates/ebml-iterable/specification-derive/src/lib.rs new file mode 100644 index 000000000..8c7449866 --- /dev/null +++ b/crates/ebml-iterable/specification-derive/src/lib.rs @@ -0,0 +1,143 @@ +extern crate proc_macro; + +mod ast; +mod attr; +mod easy_ebml; +mod pathing; + +use proc_macro::TokenStream; +use syn::{Error, ItemEnum}; + +use crate::easy_ebml::EasyEBML; + +/// +/// Attribute that derives implementations of [`EbmlSpecification`][spec] and [`EbmlTag`][tag] for an enum. +/// +/// This macro is intended to make implementing the traits in ebml-iterable-specification easier to manage. Rather than requiring handwritten implementations for [`EbmlSpecification`][spec] and [`EbmlTag`][tag] methods, this macro understands attributes assigned to enum members and generates an implementation accordingly. +/// +/// When deriving `EbmlSpecification` for an enum, the following attributes are required for each variant: +/// * __#[id(`u64`)]__ - This attribute specifies the "id" of the tag. e.g. `0x1a45dfa3` +/// * __#[data_type(`TagDataType`)]__ - This attribute specifies the type of data contained in the tag. e.g. `TagDataType::UnsignedInt` +/// +/// The following attribute is optional for each variant: +/// * __#[doc_path(Path/To/Element)]__ - This attribute specifies the document path of the current element. If this attribute is not present, the variant is treated as a Root element. Global elements can be defined with wildcard paths, e.g. #[doc_path(Segment/(1-)/)]. +/// +/// # Note +/// +/// This attribute modifies the variants in the enumeration by adding fields to them. It also will add the following variants to the enum: +/// - `Crc32(Vec)` - global tag defined in the EBML spec +/// - `Void(Vec)` - global tag defined in the EBML spec +/// - `RawTag(u64, Vec)` - used to support reading "unknown" tags that aren't in the spec +/// +/// [spec]: ebml_iterable_specification::EbmlSpecification +/// [tag]: ebml_iterable_specification::EbmlTag + +#[proc_macro_attribute] +pub fn ebml_specification(_args: TokenStream, input: TokenStream) -> TokenStream { + let mut input = match syn::parse::(input) { + Ok(syntax_tree) => syntax_tree, + Err(err) => { + return TokenStream::from( + Error::new( + err.span(), + "#[ebml_specification] attribute can only be applied to enums", + ) + .to_compile_error(), + ) + } + }; + + attr::impl_ebml_specification(&mut input) + .unwrap_or_else(|err| err.to_compile_error()) + .into() +} + +/// +/// Macro that makes writing an EBML spec easy. +/// +/// This provides an even easier alternative to create implementations of the [`EbmlSpecification`][spec] and [`EbmlTag`][tag] traits than using the [`[#ebml_specification]`][macro] attribute. As a bonus, your spec will be more legible and maintainable! +/// +/// As an example, compare the following equivalent definitions: +/// ``` +/// # use ebml_iterable_specification_derive::ebml_specification; +/// # use ebml_iterable_specification::TagDataType::{Master, UnsignedInt}; +/// # pub mod ebml_iterable { pub mod specs { +/// # pub use ebml_iterable_specification_derive::ebml_specification as ebml_specification; +/// # pub use ebml_iterable_specification::EbmlSpecification as EbmlSpecification; +/// # pub use ebml_iterable_specification::EbmlTag as EbmlTag; +/// # pub use ebml_iterable_specification::TagDataType as TagDataType; +/// # pub use ebml_iterable_specification::Master as Master; +/// # pub use ebml_iterable_specification::PathPart as PathPart; +/// # }} +/// #[ebml_specification] +/// #[derive(Clone)] +/// enum Example { +/// #[id(0x01)] +/// #[data_type(Master)] +/// Root, +/// +/// #[id(0x02)] +/// #[data_type(Master)] +/// #[doc_path(Root)] +/// Parent, +/// +/// #[id(0x100)] +/// #[data_type(UnsignedInt)] +/// #[doc_path(Root/Parent)] +/// Data, +/// } +/// ``` +/// vs +/// ``` +/// # use ebml_iterable_specification_derive::easy_ebml; +/// # use ebml_iterable_specification::TagDataType; +/// # use ebml_iterable_specification::TagDataType::{Master, UnsignedInt}; +/// # pub mod ebml_iterable { pub mod specs { +/// # pub use ebml_iterable_specification_derive::ebml_specification as ebml_specification; +/// # pub use ebml_iterable_specification::EbmlSpecification as EbmlSpecification; +/// # pub use ebml_iterable_specification::EbmlTag as EbmlTag; +/// # pub use ebml_iterable_specification::TagDataType as TagDataType; +/// # pub use ebml_iterable_specification::Master as Master; +/// # pub use ebml_iterable_specification::PathPart as PathPart; +/// # }} +/// easy_ebml! { +/// #[derive(Clone)] +/// enum Example { +/// Root : Master = 0x01, +/// Root/Parent : Master = 0x02, +/// Root/Parent/Data : UnsignedInt = 0x100, +/// } +/// } +/// ``` +/// +/// Behind the scenes `easy_ebml!` still uses the existing [`[#ebml_specification]`][macro] attribute macro, so the final output of this macro will remain identical. +/// +/// [spec]: ebml_iterable_specification::EbmlSpecification +/// [tag]: ebml_iterable_specification::EbmlTag +/// [macro]: macro@crate::ebml_specification + +#[proc_macro] +pub fn easy_ebml(input: TokenStream) -> TokenStream { + let input = match syn::parse::(input) { + Ok(syntax_tree) => syntax_tree, + Err(err) => { + return TokenStream::from( + Error::new( + err.span(), + "easy_ebml! {} content must be of format: enum Name {\ + Root: Type = id,\ + Path/Of/Component: Type = id,\ + // example\ + Ebml: Master = 0x1a45dfa3,\ + Ebml/EbmlVersion: UnsignedInt = 0x4286,\ + // global elements can be used in paths, example:\ + (1-)/Crc32: Binary = 0xbf,\ + }", + ) + .to_compile_error(), + ) + } + }; + + input.implement().unwrap_or_else(|err| err.to_compile_error()).into() +} diff --git a/crates/ebml-iterable/specification-derive/src/pathing.rs b/crates/ebml-iterable/specification-derive/src/pathing.rs new file mode 100644 index 000000000..201e703b6 --- /dev/null +++ b/crates/ebml-iterable/specification-derive/src/pathing.rs @@ -0,0 +1,86 @@ +use proc_macro2::Span; +use quote::{quote, TokenStreamExt}; +use syn::parse::{Parse, ParseStream}; +use syn::punctuated::Punctuated; +use syn::{Ident, LitInt, Result, Token}; + +pub struct EBMLPath { + pub span: Span, + pub parts: Punctuated, +} + +impl Parse for EBMLPath { + fn parse(input: ParseStream) -> Result { + let parts: Punctuated = Punctuated::parse_separated_nonempty(input)?; + Ok(Self { + parts, + span: input.span(), + }) + } +} + +#[derive(PartialEq)] +pub enum PathPart { + Ident(Ident), + Global((Option, Option)), +} + +impl Parse for PathPart { + fn parse(input: ParseStream) -> Result { + if input.lookahead1().peek(syn::token::Paren) { + let content; + syn::parenthesized!(content in input); + let min: Option = { + if content.lookahead1().peek(Token![-]) { + None + } else { + Some(content.parse::()?.base10_parse()?) + } + }; + content.parse::()?; + let max: Option = { + let val: Option = content.parse()?; + if let Some(val) = val { + Some(val.base10_parse()?) + } else { + None + } + }; + + Ok(PathPart::Global((min, max))) + } else { + let id: Ident = input.parse()?; + Ok(PathPart::Ident(id)) + } + } +} + +impl std::fmt::Display for PathPart { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + PathPart::Ident(id) => write!(f, "{id}"), + PathPart::Global((min, max)) => { + let min = if let Some(min) = min { + min.to_string() + } else { + String::new() + }; + let max = if let Some(max) = max { + max.to_string() + } else { + String::new() + }; + write!(f, "({min}-{max})") + } + } + } +} + +impl quote::ToTokens for PathPart { + fn to_tokens(&self, tokens: &mut proc_macro2::TokenStream) { + match self { + PathPart::Ident(id) => tokens.append(id.clone()), + PathPart::Global((min, max)) => tokens.extend(quote! {(#min-#max)}), + } + } +} diff --git a/crates/ebml-iterable/specification/Cargo.toml b/crates/ebml-iterable/specification/Cargo.toml new file mode 100644 index 000000000..db0b5bbbe --- /dev/null +++ b/crates/ebml-iterable/specification/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "ebml-iterable-specification" +version = "0.4.0" +authors = ["Austin Blake "] +edition = "2018" +description = "Provides the base `EbmlSpecification` used by the `ebml-iterable` and `ebml-iterable-specification-derive` crates." +license = "MIT" +homepage = "https://github.com/austinleroy/ebml-iterable" +repository = "https://github.com/austinleroy/ebml-iterable" + +[dependencies] \ No newline at end of file diff --git a/crates/ebml-iterable/specification/src/empty_spec.rs b/crates/ebml-iterable/specification/src/empty_spec.rs new file mode 100644 index 000000000..3f32b27f7 --- /dev/null +++ b/crates/ebml-iterable/specification/src/empty_spec.rs @@ -0,0 +1,104 @@ +use super::{EbmlSpecification, EbmlTag, Master, PathPart, TagDataType}; + +/// +/// An empty specification for use with examples or testing. +/// +/// This struct isn't intended for production use and should only be used for examples or PoCs. Use at your own risk - may change in the future without warning. +/// +/// # NOT SUITABLE FOR PRODUCTION +/// +#[derive(Debug, Clone, PartialEq, Eq, Ord, PartialOrd)] +pub struct EmptySpec { + id: u64, + children: Option>, + data: Option>, +} + +impl EmptySpec { + pub fn with_children(id: u64, children: Vec) -> Self { + EmptySpec::get_master_tag(id, Master::Full(children)).unwrap() + } + + pub fn with_data(id: u64, data: &[u8]) -> Self { + EmptySpec::get_binary_tag(id, data).unwrap() + } +} + +impl EbmlSpecification for EmptySpec { + fn get_tag_data_type(_id: u64) -> Option { + Some(TagDataType::Binary) + } + + fn get_path_by_id(_id: u64) -> &'static [PathPart] { + &[] + } + + fn get_unsigned_int_tag(_id: u64, _data: u64) -> Option { + None + } + + fn get_signed_int_tag(_id: u64, _data: i64) -> Option { + None + } + + fn get_utf8_tag(_id: u64, _data: String) -> Option { + None + } + + fn get_binary_tag(id: u64, data: &[u8]) -> Option { + Some(EmptySpec { + id, + children: None, + data: Some(data.to_vec()), + }) + } + + fn get_float_tag(_id: u64, _data: f64) -> Option { + None + } + + fn get_master_tag(id: u64, data: Master) -> Option { + Some(EmptySpec { + id, + children: Some(data), + data: None, + }) + } + + fn get_raw_tag(id: u64, data: &[u8]) -> EmptySpec { + EmptySpec::get_binary_tag(id, data).expect("get binary tag for EmptySpec should always return Some") + } +} + +impl EbmlTag for EmptySpec { + fn get_id(&self) -> u64 { + self.id + } + + fn as_unsigned_int(&self) -> Option<&u64> { + None + } + + fn as_signed_int(&self) -> Option<&i64> { + None + } + + fn as_utf8(&self) -> Option<&str> { + None + } + + fn as_binary(&self) -> Option<&[u8]> { + self.data.as_deref() + } + + fn as_float(&self) -> Option<&f64> { + None + } + + fn as_master(&self) -> Option<&Master> { + match &self.children { + Some(children) => Some(children), + None => None, + } + } +} diff --git a/crates/ebml-iterable/specification/src/lib.rs b/crates/ebml-iterable/specification/src/lib.rs new file mode 100644 index 000000000..7da368a93 --- /dev/null +++ b/crates/ebml-iterable/specification/src/lib.rs @@ -0,0 +1,233 @@ +//! This crate provides a core ebml specification that is used by the ebml-iterable crate. +//! +//! The related ebml-iterable-specification-derive crate can be used to simplify implementation of this spec. +//! + +/// +/// Contains an empty specification for use with examples or very basic testing. +/// +pub mod empty_spec; + +/// +/// Different data types defined in the EBML specification. +/// +/// # Notes +/// +/// This library made a concious decision to not work with "Date" elements from EBML due to lack of built-in support for dates in Rust. Specification implementations should treat Date elements as Binary so that consumers have the option of parsing the unaltered data using their library of choice, if needed. +/// + +// Possible future feature flag to enable Date functionality by having `chrono` as an optional dependency? +#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug)] +pub enum TagDataType { + Master, + UnsignedInt, + Integer, + Utf8, + Binary, + Float, +} + +#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug)] +pub enum PathPart { + Id(u64), + Global((Option, Option)), +} + +/// +/// This trait, along with [`EbmlTag`], should be implemented to define a specification so that EBML can be parsed correctly. Typically implemented on an Enum of tag variants. +/// +/// Any specification using EBML can take advantage of this library to parse or write binary data. As stated in the docs, [`TagWriter`](https://docs.rs/ebml-iterable/latest/ebml_iterable/struct.TagWriter.html) needs nothing special if you stick with the `write_raw` method, but [`TagIterator`](https://docs.rs/ebml-iterable/latest/ebml_iterable/struct.TagIterator.html) requires a struct implementing this trait. Custom specification implementations can refer to [webm-iterable](https://crates.io/crates/webm_iterable) as an example. +/// +/// This trait and [`EbmlTag`] are typically implemented simultaneously. They are separate traits as they have primarily different uses - [`EbmlSpecification`] should be brought into scope when dealing with the specification as a whole, whereas [`EbmlTag`] should be brought into scope when dealing with specific tags. +pub trait EbmlSpecification + EbmlTag + Clone> { + /// + /// Pulls the data type for a tag from the spec, based on the tag id. + /// + /// This function *must* return [`None`] if the input id is not in the specification. Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. + /// + fn get_tag_data_type(id: u64) -> Option; + + /// + /// Gets the id of a specific tag variant. + /// + /// Default implementation uses the [`EbmlTag`] implementation. Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. + /// + fn get_tag_id(item: &T) -> u64 { + item.get_id() + } + + /// + /// Gets the schema path of a specific tag. + /// + /// This function is used to find the schema defined path of a tag. If the tag is a root element, this function should return an empty array. + /// + fn get_path_by_id(id: u64) -> &'static [PathPart]; + + /// + /// Gets the schema path of a specific tag variant. + /// + /// Default implementation uses [`Self::get_path_by_id`] after obtaining the tag id using the [`EbmlTag`] implementation. + /// + fn get_path_by_tag(item: &T) -> &'static [PathPart] { + Self::get_path_by_id(item.get_id()) + } + + /// + /// Creates an unsigned integer type tag from the spec. + /// + /// This function *must* return `None` if the input id is not in the specification or if the input id data type is not [`TagDataType::UnsignedInt`]. Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. + /// + fn get_unsigned_int_tag(id: u64, data: u64) -> Option; + + /// + /// Creates a signed integer type tag from the spec. + /// + /// This function *must* return `None` if the input id is not in the specification or if the input id data type is not [`TagDataType::Integer`]. Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. + /// + fn get_signed_int_tag(id: u64, data: i64) -> Option; + + /// + /// Creates a utf8 type tag from the spec. + /// + /// This function *must* return `None` if the input id is not in the specification or if the input id data type is not [`TagDataType::Utf8`]. Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. + /// + fn get_utf8_tag(id: u64, data: String) -> Option; + + /// + /// Creates a binary type tag from the spec. + /// + /// This function *must* return `None` if the input id is not in the specification or if the input id data type is not [`TagDataType::Binary`]. Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. + /// + fn get_binary_tag(id: u64, data: &[u8]) -> Option; + + /// + /// Creates a float type tag from the spec. + /// + /// This function *must* return `None` if the input id is not in the specification or if the input id data type is not [`TagDataType::Float`]. Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. + /// + fn get_float_tag(id: u64, data: f64) -> Option; + + /// + /// Creates a master type tag from the spec. + /// + /// This function *must* return `None` if the input id is not in the specification or if the input id data type is not [`TagDataType::Master`]. Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. + /// + fn get_master_tag(id: u64, data: Master) -> Option; + + /// + /// Creates a tag that does not conform to the spec. + /// + /// This function should return a "RawTag" variant that contains the tag id and tag data. Tag data should only be retrievable as binary data. Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. + /// + fn get_raw_tag(id: u64, data: &[u8]) -> T; +} + +/// +/// This trait, along with [`EbmlSpecification`], should be implemented to define a specification so that EBML can be parsed correctly. Typically implemented on an Enum of tag variants. +/// +/// Any specification using EBML can take advantage of this library to parse or write binary data. As stated in the docs, [`TagWriter`](https://docs.rs/ebml-iterable/latest/ebml_iterable/struct.TagWriter.html) needs nothing special if you stick with the `write_raw` method, but [`TagIterator`](https://docs.rs/ebml-iterable/latest/ebml_iterable/struct.TagIterator.html) requires a struct implementing this trait. Custom specification implementations can refer to [webm-iterable](https://crates.io/crates/webm_iterable) as an example. +/// +/// This trait and [`EbmlSpecification`] are typically implemented simultaneously. They are separate traits as they have primarily different uses - [`EbmlSpecification`] should be brought into scope when dealing with the specification as a whole, whereas [`EbmlTag`] should be brought into scope when dealing with specific tags. +pub trait EbmlTag { + /// + /// Gets the id of `self`. + /// + /// Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. + /// + fn get_id(&self) -> u64; + + /// + /// Gets a reference to the data contained in `self` as an unsigned integer. + /// + /// This function *must* return `None` if the associated data type of `self` is not [`TagDataType::UnsignedInt`]. Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. + /// + fn as_unsigned_int(&self) -> Option<&u64>; + + /// + /// Gets a reference to the data contained in `self` as an integer. + /// + /// This function *must* return `None` if the associated data type of `self` is not [`TagDataType::Integer`]. Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. + /// + fn as_signed_int(&self) -> Option<&i64>; + + /// + /// Gets a reference to the data contained in `self` as string slice. + /// + /// This function *must* return `None` if the associated data type of `self` is not [`TagDataType::Utf8`]. Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. + /// + fn as_utf8(&self) -> Option<&str>; + + /// + /// Gets a reference to the data contained in `self` as binary data. + /// + /// This function *must* return `None` if the associated data type of `self` is not [`TagDataType::Binary`]. Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. + /// + fn as_binary(&self) -> Option<&[u8]>; + + /// + /// Gets a reference to the data contained in `self` as float data. + /// + /// This function *must* return `None` if the associated data type of `self` is not [`TagDataType::Float`]. Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. + /// + fn as_float(&self) -> Option<&f64>; + + /// + /// Gets a reference to master data contained in `self`. + /// + /// This function *must* return `None` if the associated data type of `self` is not [`TagDataType::Master`]. Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. + /// + fn as_master(&self) -> Option<&Master>; +} + +/// +/// An enum that defines different possible states of a [`TagDataType::Master`] tag. +/// +/// A "master" tag is a type of tag that contains other tags within it. Because these tags are dynamically sized, the [`TagIterator`](https://docs.rs/ebml-iterable/latest/ebml_iterable/struct.TagIterator.html) emits these tags as [`Master::Start`] and [`Master::End`] variants by default so that the entire tag does not need to be buffered into memory all at once. The [`Master::Full`] variant is a complete "master" tag that includes all child tags within it. +/// +#[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug)] +pub enum Master { + /// + /// Designates the start of a tag. + /// + Start, + + /// + /// Designates the end of a tag. + /// + End, + + /// + /// Designates a full tag. `Vec` contains all child tags contained in this master tag. + /// + Full(Vec), +} + +impl Master { + /// + /// Convenience method to pull children from a master tag. + /// + /// # Panics + /// + /// Panics if `self` is not a `Full` variant. + /// + /// # Examples + /// + /// ``` + /// # use ebml_iterable_specification::empty_spec::EmptySpec; + /// use ebml_iterable_specification::Master; + /// + /// let children = vec![EmptySpec::with_data(0x1253, &[1]), EmptySpec::with_data(0x1234, &[2])]; + /// // Clone children because creating a Master consumes it + /// let tag = Master::Full(children.clone()); + /// let retrieved_children = tag.get_children(); + /// assert_eq!(retrieved_children, children); + /// ``` + /// + pub fn get_children(self) -> Vec { + match self { + Master::Full(data) => data, + Master::Start => panic!("`get_children` called on Master::Start variant"), + Master::End => panic!("`get_children` called on Master::End variant"), + } + } +} diff --git a/crates/ebml-iterable/src/errors.rs b/crates/ebml-iterable/src/errors.rs new file mode 100644 index 000000000..5cf00a78d --- /dev/null +++ b/crates/ebml-iterable/src/errors.rs @@ -0,0 +1,363 @@ +use std::error::Error; +use std::fmt; + +pub mod tool { + use std::string::FromUtf8Error; + + use super::{fmt, Error}; + + #[derive(Debug)] + pub enum ToolError { + ReadVintOverflow, + WriteVintOverflow(u64), + WriteSignedVintOverflow(i64), + ReadU64Overflow(Vec), + ReadI64Overflow(Vec), + ReadF64Mismatch(Vec), + FromUtf8Error(Vec, FromUtf8Error), + } + + impl fmt::Display for ToolError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ToolError::ReadVintOverflow => write!(f, "Unrepresentable Vint size encountered."), + ToolError::WriteVintOverflow(val) => write!(f, "Value too large to be written as a vint: {val}"), + ToolError::WriteSignedVintOverflow(val) => { + write!(f, "Value outside range to be written as a vint: {val}") + } + ToolError::ReadU64Overflow(arr) => write!(f, "Could not read unsigned int from array: {arr:?}"), + ToolError::ReadI64Overflow(arr) => write!(f, "Could not read int from array: {arr:?}"), + ToolError::ReadF64Mismatch(arr) => write!(f, "Could not read float from array: {arr:?}"), + ToolError::FromUtf8Error(arr, _source) => write!(f, "Could not read utf8 data: {arr:?}"), + } + } + } + + impl Error for ToolError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + ToolError::FromUtf8Error(_arr, source) => Some(source), + _ => None, + } + } + } +} + +pub mod tag_iterator { + use std::io; + + use super::tool::ToolError; + use super::{fmt, Error}; + + /// + /// Errors that indicate file data is corrupted. + /// + #[derive(Debug)] + pub enum CorruptedFileError { + /// + /// An error indicating the reader found an ebml tag id not defined in the current specification. + /// + InvalidTagId { + /// + /// The position of the element. + /// + position: usize, + + /// + /// The id of the tag that was found. + /// + tag_id: u64, + }, + + /// + /// An error indicating the reader could not parse a valid tag due to corrupted tag data (size/contents). + /// + InvalidTagData { + /// + /// The position of the element. + /// + position: usize, + + /// + /// The id of the tag that was found. + /// + tag_id: u64, + }, + + /// + /// An error indicating the reader found an element outside of its expected hierarchy. + /// + HierarchyError { + /// + /// The id of the tag that was found. + /// + found_tag_id: u64, + + /// + /// The id of the current "master" element that contains the tag that was found. + /// + current_parent_id: Option, + }, + + /// + /// An error indicating the reader found a child element with incorrect sizing. + /// + OversizedChildElement { + /// + /// The position of the element. + /// + position: usize, + + /// + /// The id of the tag that was found. + /// + tag_id: u64, + + /// + /// The size of the tag that was found. + /// + size: usize, + }, + + /// + /// An error indicating the reader found a tag with an invalid size. + /// + InvalidTagSize { + /// + /// The position of the element. + /// + position: usize, + + /// + /// The id of the tag that was found. + /// + tag_id: u64, + + /// + /// The size of the tag that was found. + /// + size: usize, + }, + } + + impl fmt::Display for CorruptedFileError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + CorruptedFileError::InvalidTagId { + position, + tag_id + } => write!(f, "Encountered invalid tag id [0x{tag_id:x?}] at position {position}"), + CorruptedFileError::InvalidTagData { + position, + tag_id + } => write!(f, "Encountered invalid tag data for tag id [0x{tag_id:x?}] at position {position}"), + CorruptedFileError::HierarchyError { + found_tag_id, + current_parent_id, + } => write!(f, "Found child tag [{found_tag_id:x?}] when processing parent [{current_parent_id:x?}]"), + CorruptedFileError::OversizedChildElement { + position, + tag_id, + size : _ + } => write!(f, "Found an oversized tag [0x{tag_id:x?}] at position {position}"), + CorruptedFileError::InvalidTagSize { + position, + tag_id, + size, + } => write!(f, "Found an oversized tag [0x{tag_id:x?}] at position {position} with size {size}. Max supported size is 8GB."), + } + } + } + + /// + /// Errors that can occur when reading ebml data. + /// + #[derive(Debug)] + pub enum TagIteratorError { + /// + /// An error indicating that data in the file being read is not valid. + /// + CorruptedFileData(CorruptedFileError), + + /// + /// An error indicating that the iterator reached the end of the input stream unexpectedly while reading a tag. + /// + /// This error will occur if the iterator is expecting more data (either due to expecting a size after reading a tag id or based on a tag size) but nothing is available in the input stream. + /// + UnexpectedEOF { + /// + /// The start position of the tag that was being read when EOF was reached. + /// + tag_start: usize, + + /// + /// The id of the partially read tag, if available. + /// + tag_id: Option, + + /// + /// The size of the partially read tag, if available. + /// + tag_size: Option, + + /// + /// Any available data that was read for the tag before reaching EOF. + /// + partial_data: Option>, + }, + + /// + /// An error indicating that tag data appears to be corrupted. + /// + /// This error typically occurs if tag data cannot be read as its expected data type (e.g. trying to read `[32,42,8]` as float data, since floats require either 4 or 8 bytes). + /// + CorruptedTagData { + /// + /// The id of the corrupted tag. + /// + tag_id: u64, + + /// + /// An error describing why the data is corrupted. + /// + problem: ToolError, + }, + + /// + /// An error that wraps an IO error when reading from the underlying source. + /// + ReadError { + /// + /// The [`io::Error`] that caused this problem. + /// + source: io::Error, + }, + } + + impl fmt::Display for TagIteratorError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + TagIteratorError::CorruptedFileData(err) => write!(f, "Encountered corrupted data. Message: {err}"), + TagIteratorError::UnexpectedEOF { + tag_start, + tag_id, + tag_size, + partial_data: _ + } => write!(f, "Reached EOF unexpectedly. Partial tag data: {{tag offset:{tag_start}}} {{id:{tag_id:x?}}} {{size:{tag_size:?}}}"), + TagIteratorError::CorruptedTagData { + tag_id, + problem, + } => write!(f, "Error reading data for tag id (0x{tag_id:x?}). {problem}"), + TagIteratorError::ReadError { source: _ } => write!(f, "Error reading from source."), + } + } + } + + impl Error for TagIteratorError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + TagIteratorError::CorruptedFileData(_) => None, + TagIteratorError::UnexpectedEOF { + tag_start: _, + tag_id: _, + tag_size: _, + partial_data: _, + } => None, + TagIteratorError::CorruptedTagData { tag_id: _, problem } => problem.source(), + TagIteratorError::ReadError { source } => Some(source), + } + } + } +} + +pub mod tag_writer { + use std::io; + + use super::{fmt, Error}; + + /// + /// Errors that can occur when writing ebml data. + /// + #[derive(Debug)] + pub enum TagWriterError { + /// + /// An error indicating the tag to be written doesn't conform to the current specification. + /// + /// This error occurs if you attempt to write a tag outside of a valid document path. See the [EBML RFC](https://www.rfc-editor.org/rfc/rfc8794.html#section-11.1.6.2) for details on element paths. + /// + UnexpectedTag { tag_id: u64, current_path: Vec }, + + /// + /// An error with a tag id. + /// + /// This error should only occur if writing "RawTag" variants, and only if the input id is not a valid vint. + /// + TagIdError(u64), + + /// + /// An error with the size of a tag. + /// + /// Can occur if the tag size overflows the max value representable by a vint (`2^57 - 1`, or `144,115,188,075,855,871`). + /// + /// This can also occur if a non-[`Master`][`crate::specs::TagDataType::Master`] tag is sent to be written with an unknown size. + /// + TagSizeError(String), + + /// + /// An error indicating a tag was closed unexpectedly. + /// + /// Can occur if a [`Master::End`][`crate::specs::Master::End`] variant is passed to the [`TagWriter`][`crate::TagWriter`] but the id doesn't match the currently open tag. + /// + UnexpectedClosingTag { + /// + /// The id of the tag being closed. + /// + tag_id: u64, + + /// + /// The id of the currently open tag. + /// + expected_id: Option, + }, + + /// + /// An error that wraps an IO error when writing to the underlying destination. + /// + WriteError { source: io::Error }, + } + + impl fmt::Display for TagWriterError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + TagWriterError::UnexpectedTag { tag_id, current_path } => { + write!(f, "Unexpected tag 0x{tag_id:x?} when writing to {current_path:x?}") + } + TagWriterError::TagIdError(id) => write!(f, "Tag id 0x{id:x?} is not a valid vint"), + TagWriterError::TagSizeError(message) => write!(f, "Problem writing data tag size. {message}"), + TagWriterError::UnexpectedClosingTag { tag_id, expected_id } => match expected_id { + Some(expected) => write!(f, "Unexpected closing tag 0x'{tag_id:x?}'. Expected 0x'{expected:x?}'"), + None => write!(f, "Unexpected closing tag 0x'{tag_id:x?}'"), + }, + TagWriterError::WriteError { source: _ } => write!(f, "Error writing to destination."), + } + } + } + + impl Error for TagWriterError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + TagWriterError::UnexpectedTag { + tag_id: _, + current_path: _, + } => None, + TagWriterError::TagIdError(_) => None, + TagWriterError::TagSizeError(_) => None, + TagWriterError::UnexpectedClosingTag { + tag_id: _, + expected_id: _, + } => None, + TagWriterError::WriteError { source } => Some(source), + } + } + } +} diff --git a/crates/ebml-iterable/src/lib.rs b/crates/ebml-iterable/src/lib.rs new file mode 100644 index 000000000..33156c6c4 --- /dev/null +++ b/crates/ebml-iterable/src/lib.rs @@ -0,0 +1,58 @@ +//! This crate provides an iterator and a serializer for [EBML][EBML] files. Its primary goal is to provide typed iteration and serialization as lightly and quickly as possible. +//! +//! [EBML][EBML] stands for Extensible Binary Meta-Language and is somewhat of a +//! binary version of XML. It's used for container formats like [WebM][webm] or +//! [MKV][mkv]. +//! +//! # Important - Specifications +//! The iterator contained in this crate is spec-agnostic and requires a specification implementing the [`specs::EbmlSpecification`] and [`specs::EbmlTag`] traits to read files. Typically, you would only use this crate to implement a custom specification - most often you would prefer a crate providing an existing specification, like [webm-iterable][webm-iterable]. +//! +//! Implementing custom specifications can be made less painful and safer by enabling the `"derive-spec"` feature flag in this crate and using the [`#[ebml_specification]`](https://docs.rs/ebml-iterable-specification-derive/latest/ebml_iterable_specification_derive/attr.ebml_specification.html) macro. +//! +//! # Features +//! +//! There is currently only one optional feature in this crate, but that may change over time as needs arise. +//! +//! * **derive-spec** - +//! When enabled, this provides the [`#[ebml_specification]`](https://docs.rs/ebml-iterable-specification-derive/latest/ebml_iterable_specification_derive/attr.ebml_specification.html) attribute macro to simplify implementation of the [`EbmlSpecification`][`specs::EbmlSpecification`] and [`EbmlTag`][`specs::EbmlTag`] traits. This introduces dependencies on [`syn`](https://crates.io/crates/syn), [`quote`](https://crates.io/crates/quote), and [`proc-macro2`](https://crates.io/crates/proc-macro2), so expect compile times to increase a little. +//! +//! [EBML]: http://ebml.sourceforge.net/ +//! [webm]: https://www.webmproject.org/ +//! [mkv]: http://www.matroska.org/technical/specs/index.html +//! [rfc8794]: https://datatracker.ietf.org/doc/rfc8794/ +//! [webm-iterable]: https://crates.io/crates/webm_iterable +//! + +mod errors; +mod spec_util; +pub mod specs; +mod tag_decoder; +mod tag_iterator; +mod tag_iterator_util; +mod tag_parse; +mod tag_writer; +pub mod tools; + +#[cfg(feature = "futures")] +pub mod nonblocking; + +pub use self::tag_decoder::{PositionedTag, TagDecoder}; +pub use self::tag_iterator::TagIterator; +pub use self::tag_writer::{TagWriter, WriteOptions}; + +pub mod iterator { + pub use super::tag_iterator_util::AllowableErrors; +} + +pub mod error { + + //! + //! Potential errors that can occur when reading or writing EBML data. + //! + pub use super::errors::tag_iterator::{CorruptedFileError, TagIteratorError}; + pub use super::errors::tag_writer::TagWriterError; + /// + /// Error details that may be included in some thrown errors + /// + pub use super::errors::tool::ToolError; +} diff --git a/crates/ebml-iterable/src/nonblocking.rs b/crates/ebml-iterable/src/nonblocking.rs new file mode 100644 index 000000000..a7eb8f6ce --- /dev/null +++ b/crates/ebml-iterable/src/nonblocking.rs @@ -0,0 +1,59 @@ +use std::io::Cursor; + +use ebml_iterable_specification::{EbmlSpecification, EbmlTag}; +use futures::{AsyncRead, AsyncReadExt, Stream}; + +use crate::error::TagIteratorError; +use crate::TagIterator; + +/// +/// This can be transformed into a [`Stream`] using [`into_stream`][TagIteratorAsync::into_stream], or consumed directly by calling [`.next().await`] in a loop. +/// +/// The struct can be created with the [`new()`][TagIteratorAsync::new] function on any source that implements the [`futures::AsyncRead`] trait. +/// +pub struct TagIteratorAsync +where + TSpec: EbmlSpecification + EbmlTag + Clone, +{ + source: R, + buffer: Box<[u8]>, + iterator: TagIterator>, TSpec>, +} + +impl TagIteratorAsync +where + TSpec: EbmlSpecification + EbmlTag + Clone, +{ + pub fn new(source: R, tags_to_buffer: &[TSpec]) -> Self { + let buffer = vec![0u8; 1024 * 64]; + Self { + source, + buffer: buffer.into_boxed_slice(), + iterator: TagIterator::new(Cursor::new(Vec::new()), tags_to_buffer), + } + } + + pub async fn next(&mut self) -> Option> { + match self.source.read(&mut self.buffer).await { + Ok(len) => { + self.iterator + .get_mut() + .get_mut() + .append(&mut self.buffer[..len].to_vec()); + self.iterator.next() + } + Err(e) => Some(Err(TagIteratorError::ReadError { source: e })), + } + } + + pub fn into_stream(self) -> impl Stream> { + futures::stream::unfold(self, |mut read| async { + let next = read.next().await; + next.map(move |it| (it, read)) + }) + } + + pub fn last_emitted_tag_offset(&self) -> usize { + self.iterator.last_emitted_tag_offset() + } +} diff --git a/crates/ebml-iterable/src/spec_util.rs b/crates/ebml-iterable/src/spec_util.rs new file mode 100644 index 000000000..32f6f48ba --- /dev/null +++ b/crates/ebml-iterable/src/spec_util.rs @@ -0,0 +1,89 @@ +use ebml_iterable_specification::{EbmlSpecification, EbmlTag, PathPart}; + +use crate::tag_iterator_util::EBMLSize; + +/// +/// Returns whether or not the a `test_id` is a parent of `current_id`. +/// +pub fn is_parent + EbmlTag + Clone>(current_id: u64, test_id: u64) -> bool { + let path = ::get_path_by_id(current_id); + path.iter().any(|p| matches!(p, PathPart::Id(p) if p == &test_id)) +} + +/// +/// Returns whether or not the `test_id` is a sibling of `current_id`. +/// +/// A sibling tag is one which shares the same direct parent. A separate instance of the current tag counts as a sibling. +/// +pub fn is_sibling + EbmlTag + Clone>(current_id: u64, test_id: u64) -> bool { + ::get_path_by_id(current_id) == ::get_path_by_id(test_id) +} + +/// +/// Returns whether or not the `test_id` would end this "Unknown" sized `current_id`. +/// +/// Regarding this method, unknown sized tags can be ended if we reach an element that is: +/// - A parent of the tag +/// - A direct sibling of the tag +/// - A Root element +/// +/// There are a couple of other cases where an Unknown sized tag can end, but they rely on knowing document position and tag sizes. More details can be found in the [EBML RFC](https://www.rfc-editor.org/rfc/rfc8794.html#name-unknown-data-size). +/// +pub fn is_ended_by + EbmlTag + Clone>(current_id: u64, test_id: u64) -> bool { + is_parent::(current_id, test_id) || // parent + is_sibling::(current_id, test_id) || // sibling + ( // Root element + ::get_tag_data_type(test_id).is_some() && + ::get_path_by_id(test_id).is_empty() + ) +} + +#[inline(always)] +pub fn validate_tag_path + EbmlTag + Clone>( + tag_id: u64, + doc_path: impl Iterator, +) -> bool { + let path = ::get_path_by_id(tag_id); + let mut path_marker = 0; + let mut global_counter = 0; + for item in doc_path { + let current_node_id = item.0; + + if !item.1.is_known() && is_ended_by::(current_node_id, tag_id) { + return true; + } + + if path_marker >= path.len() { + return false; + } + + match path[path_marker] { + PathPart::Id(id) => { + if id != current_node_id { + return false; + } + path_marker += 1; + } + PathPart::Global((min, max)) => { + global_counter += 1; + if max.is_some() && global_counter > max.unwrap_or_default() { + return false; + } + if path.len() > (path_marker + 1) + && matches!(path[path_marker + 1], PathPart::Id(id) if id == current_node_id) + { + if min.is_some() && global_counter < min.unwrap_or_default() { + return false; + } + path_marker += 2; + global_counter = 0; + } + } + } + } + + // Validate that we compared ALL parents in the path + path.len() == path_marker || + // or that the last parent was a global whose minimum was met + ((path.len() - 1) == path_marker && matches!(path[path_marker], PathPart::Global((min, _)) if global_counter >= min.unwrap_or(0))) +} diff --git a/crates/ebml-iterable/src/specs.rs b/crates/ebml-iterable/src/specs.rs new file mode 100644 index 000000000..b7458bd9d --- /dev/null +++ b/crates/ebml-iterable/src/specs.rs @@ -0,0 +1,11 @@ +//! +//! Provides the EBML specification types. +//! +//! Typically won't be used unless you are implementing a custom specification that uses EBML. You can enable the `"derive-spec"` feature to obtain a macro to make implementation easier. +//! + +pub use ebml_iterable_specification::{EbmlSpecification, EbmlTag, Master, PathPart, TagDataType}; +#[cfg(feature = "derive-spec")] +pub use ebml_iterable_specification_derive::easy_ebml; +#[cfg(feature = "derive-spec")] +pub use ebml_iterable_specification_derive::ebml_specification; diff --git a/crates/ebml-iterable/src/tag_decoder.rs b/crates/ebml-iterable/src/tag_decoder.rs new file mode 100644 index 000000000..5c2889f2c --- /dev/null +++ b/crates/ebml-iterable/src/tag_decoder.rs @@ -0,0 +1,406 @@ +// This decoder extends Austin Blake's MIT-licensed ebml-iterable 0.6.3. +// Gateway vendors it because growing recordings must keep partial tags across appends. + +use std::collections::{HashSet, VecDeque}; + +use bytes::{Buf, BytesMut}; + +use crate::errors::tag_iterator::{CorruptedFileError, TagIteratorError}; +use crate::spec_util::{is_ended_by, validate_tag_path}; +use crate::specs::{EbmlSpecification, EbmlTag, Master, PathPart, TagDataType}; +use crate::tag_iterator_util::EBMLSize::{Known, Unknown}; +use crate::tag_iterator_util::{AllowableErrors, EBMLSize}; +use crate::tag_parse::{self, TagHeader}; +use crate::tools; + +const INVALID_TAG_ID_ERROR: u8 = 0x01; +const INVALID_HIERARCHY_ERROR: u8 = 0x02; +const OVERSIZED_CHILD_ERROR: u8 = 0x04; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct PositionedTag { + pub tag: TSpec, + pub offset: usize, +} + +struct OpenTag { + id: u64, + size: EBMLSize, + tag_start: usize, + data_start: usize, + capture: bool, + children: Vec, +} + +pub struct TagDecoder +where + TSpec: EbmlSpecification + EbmlTag + Clone, +{ + tag_ids_to_buffer: HashSet, + allowed_errors: u8, + max_allowed_tag_size: Option, + position: usize, + tag_stack: Vec>, + emission_queue: VecDeque>, + has_determined_doc_path: bool, + finished: bool, +} + +impl TagDecoder +where + TSpec: EbmlSpecification + EbmlTag + Clone, +{ + pub fn new(tags_to_buffer: &[TSpec]) -> Self { + Self { + tag_ids_to_buffer: tags_to_buffer.iter().map(EbmlTag::get_id).collect(), + allowed_errors: 0, + max_allowed_tag_size: Some(4 * usize::pow(1000, 3)), + position: 0, + tag_stack: Vec::new(), + emission_queue: VecDeque::new(), + has_determined_doc_path: false, + finished: false, + } + } + + pub fn allow_errors(&mut self, errors: &[AllowableErrors]) { + self.allowed_errors = errors.iter().fold(0u8, |allowed, error| match error { + AllowableErrors::InvalidTagIds => allowed | INVALID_TAG_ID_ERROR, + AllowableErrors::HierarchyProblems => allowed | INVALID_HIERARCHY_ERROR, + AllowableErrors::OversizedTags => allowed | OVERSIZED_CHILD_ERROR, + }); + } + + pub fn set_max_allowable_tag_size(&mut self, size: Option) { + self.max_allowed_tag_size = size; + } + + pub fn position(&self) -> usize { + self.position + } + + pub fn decode(&mut self, input: &mut BytesMut) -> Result>, TagIteratorError> { + assert!( + !self.finished || input.is_empty(), + "cannot append EBML data after end of input" + ); + if let Some(tag) = self.emission_queue.pop_front() { + return Ok(Some(tag)); + } + if self.finished { + return Ok(None); + } + + loop { + self.close_completed_tags()?; + if let Some(tag) = self.emission_queue.pop_front() { + return Ok(Some(tag)); + } + + if input.is_empty() { + return Ok(None); + } + + let Some(header) = tag_parse::read_header::(input, self.position)? else { + return Ok(None); + }; + + self.close_unknown_tags(header.id)?; + if let Some(tag) = self.emission_queue.pop_front() { + return Ok(Some(tag)); + } + + self.validate_header(&header)?; + if matches!(header.data_type, Some(TagDataType::Master)) { + self.open_master(input, header)?; + } else { + let Some(tag) = self.read_data_tag(input, &header)? else { + return Ok(None); + }; + self.deliver(tag, self.position); + self.advance(input, header.len + header.size.value()); + } + + if let Some(tag) = self.emission_queue.pop_front() { + return Ok(Some(tag)); + } + } + } + + pub fn decode_eof(&mut self, input: &mut BytesMut) -> Result>, TagIteratorError> { + if let Some(tag) = self.decode(input)? { + return Ok(Some(tag)); + } + + if !input.is_empty() { + return Err(self.unexpected_eof(input)); + } + + if let Some(open_tag) = self + .tag_stack + .iter() + .find(|tag| matches!(tag.size, Known(size) if self.position < tag.data_start.saturating_add(size))) + { + return Err(TagIteratorError::UnexpectedEOF { + tag_start: open_tag.tag_start, + tag_id: Some(open_tag.id), + tag_size: match open_tag.size { + Known(size) => Some(size), + Unknown => None, + }, + partial_data: None, + }); + } + + while !self.tag_stack.is_empty() { + self.close_top()?; + } + self.finished = true; + + Ok(self.emission_queue.pop_front()) + } + + pub fn is_finished(&self) -> bool { + self.finished && self.emission_queue.is_empty() + } + + fn validate_header(&mut self, header: &TagHeader) -> Result<(), TagIteratorError> { + if self.allowed_errors & INVALID_TAG_ID_ERROR == 0 && header.data_type.is_none() { + return Err(TagIteratorError::CorruptedFileData(CorruptedFileError::InvalidTagId { + position: self.position, + tag_id: header.id, + })); + } + + self.determine_doc_path(header.id); + if self.allowed_errors & INVALID_HIERARCHY_ERROR == 0 + && header.data_type.is_some() + && self.has_determined_doc_path + && !validate_tag_path::(header.id, self.tag_stack.iter().map(|tag| (tag.id, tag.size, 0))) + { + return Err(TagIteratorError::CorruptedFileData( + CorruptedFileError::HierarchyError { + found_tag_id: header.id, + current_parent_id: self.tag_stack.last().map(|tag| tag.id), + }, + )); + } + + if let Known(size) = header.size { + let total_size = header.len.checked_add(size).ok_or({ + TagIteratorError::CorruptedFileData(CorruptedFileError::InvalidTagSize { + position: self.position, + tag_id: header.id, + size, + }) + })?; + let element_end = self.position.checked_add(total_size).ok_or({ + TagIteratorError::CorruptedFileData(CorruptedFileError::InvalidTagSize { + position: self.position, + tag_id: header.id, + size, + }) + })?; + + if self.allowed_errors & OVERSIZED_CHILD_ERROR == 0 + && self.tag_stack.iter().any(|tag| { + matches!(tag.size, Known(parent_size) if tag.data_start.saturating_add(parent_size) < element_end) + }) + { + return Err(TagIteratorError::CorruptedFileData( + CorruptedFileError::OversizedChildElement { + position: self.position, + tag_id: header.id, + size, + }, + )); + } + + if self.max_allowed_tag_size.is_some_and(|max_size| size > max_size) { + return Err(TagIteratorError::CorruptedFileData( + CorruptedFileError::InvalidTagSize { + position: self.position, + tag_id: header.id, + size, + }, + )); + } + } + + Ok(()) + } + + fn determine_doc_path(&mut self, tag_id: u64) { + if self.has_determined_doc_path { + return; + } + + let path = TSpec::get_path_by_id(tag_id); + if !path.iter().all(|part| matches!(part, PathPart::Id(_))) { + return; + } + + self.tag_stack = path + .iter() + .map(|part| match part { + PathPart::Id(id) => OpenTag { + id: *id, + size: Unknown, + tag_start: 0, + data_start: 0, + capture: false, + children: Vec::new(), + }, + PathPart::Global(_) => unreachable!(), + }) + .collect(); + self.has_determined_doc_path = true; + } + + fn open_master(&mut self, input: &mut BytesMut, header: TagHeader) -> Result<(), TagIteratorError> { + let tag_start = self.position; + let data_start = self + .position + .checked_add(header.len) + .expect("validated tag header length should fit in usize"); + let capture = + self.tag_ids_to_buffer.contains(&header.id) || self.tag_stack.last().is_some_and(|parent| parent.capture); + let start = TSpec::get_master_tag(header.id, Master::Start).unwrap_or_else(|| { + panic!( + "Bad specification implementation: Tag id 0x{:x?} type was master, but could not get tag!", + header.id + ) + }); + + self.advance(input, header.len); + self.tag_stack.push(OpenTag { + id: header.id, + size: header.size, + tag_start, + data_start, + capture, + children: Vec::new(), + }); + if !capture { + self.emission_queue.push_back(PositionedTag { + tag: start, + offset: tag_start, + }); + } + + Ok(()) + } + + fn read_data_tag(&self, input: &[u8], header: &TagHeader) -> Result, TagIteratorError> { + let Known(size) = header.size else { + return Err(TagIteratorError::CorruptedFileData( + CorruptedFileError::InvalidTagData { + tag_id: header.id, + position: self.position, + }, + )); + }; + let total_size = header + .len + .checked_add(size) + .expect("validated tag size should fit in usize"); + if input.len() < total_size { + return Ok(None); + } + + let raw_data = &input[header.len..total_size]; + tag_parse::read_data_tag(header.id, header.data_type, raw_data).map(Some) + } + + fn close_completed_tags(&mut self) -> Result<(), TagIteratorError> { + let ended_index = self + .tag_stack + .iter() + .position(|tag| matches!(tag.size, Known(size) if self.position >= tag.data_start.saturating_add(size))); + if let Some(index) = ended_index { + while self.tag_stack.len() > index { + self.close_top()?; + } + } + Ok(()) + } + + fn close_unknown_tags(&mut self, next_id: u64) -> Result<(), TagIteratorError> { + while self + .tag_stack + .last() + .is_some_and(|tag| tag.size == Unknown && is_ended_by::(tag.id, next_id)) + { + self.close_top()?; + } + Ok(()) + } + + fn close_top(&mut self) -> Result<(), TagIteratorError> { + let open_tag = self + .tag_stack + .pop() + .expect("an open tag should exist before it is closed"); + let id = open_tag.id; + let tag_start = open_tag.tag_start; + let tag = if open_tag.capture { + TSpec::get_master_tag(id, Master::Full(open_tag.children)) + } else { + TSpec::get_master_tag(id, Master::End) + } + .unwrap_or_else(|| { + panic!( + "Bad specification implementation: Tag id 0x{:x?} type was master, but could not get tag!", + id + ) + }); + self.deliver(tag, tag_start); + Ok(()) + } + + fn deliver(&mut self, tag: TSpec, offset: usize) { + if let Some(parent) = self.tag_stack.iter_mut().rev().find(|parent| parent.capture) { + parent.children.push(tag); + } else { + self.emission_queue.push_back(PositionedTag { tag, offset }); + } + } + + fn advance(&mut self, input: &mut BytesMut, count: usize) { + input.advance(count); + self.position = self + .position + .checked_add(count) + .expect("validated tag size should keep the decoder position in usize"); + } + + fn unexpected_eof(&self, input: &[u8]) -> TagIteratorError { + let first = input.first().copied(); + let id_len = first.map_or(0, |byte| if byte == 0 { 1 } else { 8 - byte.ilog2() as usize }); + let tag_id = (id_len > 0 && input.len() >= id_len).then(|| { + input[..id_len] + .iter() + .fold(0u64, |value, byte| (value << 8) + u64::from(*byte)) + }); + let size = tag_id.and_then(|_| { + tools::read_vint(input.get(id_len..).unwrap_or_default()) + .ok() + .flatten() + .and_then(|(size, size_len)| match EBMLSize::new(size, size_len) { + Known(size) => Some((size, id_len + size_len)), + Unknown => None, + }) + }); + let partial_data = size.map_or_else( + || Some(input.to_vec()), + |(_, header_len)| Some(input.get(header_len..).unwrap_or_default().to_vec()), + ); + + TagIteratorError::UnexpectedEOF { + tag_start: self.position, + tag_id, + tag_size: size.map(|(size, _)| size), + partial_data, + } + } +} diff --git a/crates/ebml-iterable/src/tag_iterator.rs b/crates/ebml-iterable/src/tag_iterator.rs new file mode 100644 index 000000000..d35b5f517 --- /dev/null +++ b/crates/ebml-iterable/src/tag_iterator.rs @@ -0,0 +1,597 @@ +use std::collections::{HashSet, VecDeque}; +use std::io::Read; + +use super::errors::tag_iterator::{CorruptedFileError, TagIteratorError}; +use super::specs::{EbmlSpecification, EbmlTag, Master, PathPart, TagDataType}; +use crate::spec_util::validate_tag_path; +use crate::tag_iterator_util::EBMLSize::{Known, Unknown}; +use crate::tag_iterator_util::{AllowableErrors, EBMLSize, ProcessingTag, DEFAULT_BUFFER_LEN}; +use crate::tag_parse; + +const INVALID_TAG_ID_ERROR: u8 = 0x01; +const INVALID_HIERARCHY_ERROR: u8 = 0x02; +const OVERSIZED_CHILD_ERROR: u8 = 0x04; + +/// +/// Provides an iterator over EBML files (read from a source implementing the [`std::io::Read`] trait). Can be configured to read specific "Master" tags as complete objects rather than just emitting when they start and end. +/// +/// This is a generic struct that requires a specification implementing [`EbmlSpecification`] and [`EbmlTag`]. No specifications are included in this crate - you will need to either use another crate providing a spec (such as the Matroska spec implemented in the [webm-iterable](https://crates.io/crates/webm_iterable) or write your own spec if you want to iterate over a custom EBML file. The iterator outputs `TSpec` variants representing the type of tag (defined by the specification) and the accompanying tag data. "Master" tags (defined by the specification) usually will be read as `Start` and `End` variants, but the iterator can be configured to buffer Master tags into a `Full` variant using the `tags_to_buffer` parameter. +/// +/// Note: The [`Self::with_capacity()`] method can be used to construct a `TagIterator` with a specified default buffer size. This is only useful as a microoptimization to memory management if you know the maximum tag size of the file you're reading. +/// +/// ## Example +/// +/// ```no_run +/// use std::fs::File; +/// use ebml_iterable::TagIterator; +/// # +/// # use ebml_iterable::specs::{EbmlSpecification, TagDataType}; +/// # use ebml_iterable_specification::empty_spec::EmptySpec; +/// +/// # fn main() -> Result<(), Box> { +/// let file = File::open("my_ebml_file.ebml")?; +/// let mut my_iterator: TagIterator<_, EmptySpec> = TagIterator::new(file, &[]); +/// for tag in my_iterator { +/// println!("{:?}", tag?); +/// } +/// # Ok(()) +/// # } +/// ``` +/// +/// ## Errors +/// +/// The `Item` type for the associated [`Iterator`] implementation is a [`Result`], meaning each `next()` call has the potential to fail. This is because the source data is not parsed all at once - it is incrementally parsed as the iterator progresses. If the iterator runs into an error (such as corrupted data or an unexpected end-of-file), it needs to be propagated to the logic trying to read the tags. The different possible error states are enumerated in [`TagIteratorError`]. +/// +/// ## Panics +/// +/// The iterator can panic if `` is an internally inconsistent specification (i.e. it claims that a specific tag id has a specific data type but fails to produce a tag variant using data of that type). This won't happen if the specification being used was created using the [`#[ebml_specification]`](https://docs.rs/ebml-iterable-specification-derive/latest/ebml_iterable_specification_derive/attr.ebml_specification.html) attribute macro. +/// +pub struct TagIterator +where + TSpec: EbmlSpecification + EbmlTag + Clone, +{ + source: R, + tag_ids_to_buffer: HashSet, + allowed_errors: u8, + max_allowed_tag_size: Option, + + buffer: Box<[u8]>, + buffer_offset: Option, + buffered_byte_length: usize, + internal_buffer_position: usize, + tag_stack: Vec>, + emission_queue: VecDeque>, + last_emitted_tag_offset: usize, + has_determined_doc_path: bool, + + emit_master_end_when_eof: bool, +} + +impl TagIterator +where + TSpec: EbmlSpecification + EbmlTag + Clone, +{ + /// + /// Returns a new [`TagIterator`] instance. + /// + /// The `source` parameter must implement [`std::io::Read`]. The second argument, `tags_to_buffer`, specifies which "Master" tags should be read as [`Master::Full`]s rather than as [`Master::Start`] and [`Master::End`]s. Refer to the documentation on [`TagIterator`] for more explanation of how to use the returned instance. + /// + pub fn new(source: R, tags_to_buffer: &[TSpec]) -> Self { + TagIterator::with_capacity(source, tags_to_buffer, DEFAULT_BUFFER_LEN) + } + + /// + /// Returns a new [`TagIterator`] instance with the specified internal buffer capacity. + /// + /// This initializes the [`TagIterator`] with a specific byte capacity. The iterator will still reallocate if necessary. (Reallocation occurs if the iterator comes across a tag that should be output as a [`Master::Full`] and its size in bytes is greater than the iterator's current buffer capacity.) + /// + pub fn with_capacity(source: R, tags_to_buffer: &[TSpec], capacity: usize) -> Self { + let buffer = vec![0; capacity]; + + TagIterator { + source, + tag_ids_to_buffer: tags_to_buffer.iter().map(|tag| tag.get_id()).collect(), + allowed_errors: 0, + max_allowed_tag_size: Some(4 * usize::pow(1000, 3)), // 4GB + buffer: buffer.into_boxed_slice(), + buffered_byte_length: 0, + buffer_offset: None, + internal_buffer_position: 0, + tag_stack: Vec::new(), + emission_queue: VecDeque::new(), + last_emitted_tag_offset: 0, + has_determined_doc_path: false, + emit_master_end_when_eof: true, + } + } + + /// + /// Configures how strictly the iterator abides ``. + /// + /// By default (as of v0.5.0), the iterator assumes `` is complete and that any tags that do not conform to `` are due to corrupted file data. This method can be used to relax some of these checks so that fewer [`TagIteratorError::CorruptedFileData`] errors occur. + /// + /// # Important + /// + /// Relaxing these checks do not necessarily make the iterator more robust. If all errors are allowed, the iterator will assume any incoming tag id and tag data size are valid, and it will produce "RawTag"s containing binary contents for any tag ids not in ``. However, if the file truly has corrupted data, the "size" of these elements will likely be corrupt as well. This can typically result in requests for massive allocations, causing delays and eventual crashing. By eagerly returning errors (the default), applications can decide how to handle corrupted elements more quickly and efficiently. + /// + /// tldr; allow errors at your own risk + /// + /// > Note: TagIterators returned by [`Self::new()`] and [`Self::with_capacity()`] allow no errors by default. + /// + pub fn allow_errors(&mut self, errors: &[AllowableErrors]) { + self.allowed_errors = errors.iter().fold(0u8, |a, c| match c { + AllowableErrors::InvalidTagIds => a | INVALID_TAG_ID_ERROR, + AllowableErrors::HierarchyProblems => a | INVALID_HIERARCHY_ERROR, + AllowableErrors::OversizedTags => a | OVERSIZED_CHILD_ERROR, + }); + } + + /// + /// Configures the maximum size a tag is allowed to be before the iterator considers it invalid. + /// + /// By default (as of v0.6.1), the iterator will throw an [`CorruptedFileError::InvalidTagSize`] error if it comes across any tags that declare their data to be more than 4GB. This method can be used to change (and optionally remove) this behavior. Note that increasing this size can potentially result in massive allocations, causing delays and panics. + /// + pub fn set_max_allowable_tag_size(&mut self, size: Option) { + self.max_allowed_tag_size = size; + } + + /// + /// Instructs the iterator to attempt to recover after reaching corrupted file data. + /// + /// This method can be used to skip over corrupted sections of a read stream without recreating a new iterator. The iterator will seek forward from its current internal position until it reaches either a valid EBML tag id or EOF. After recovery, [`Iterator::next()`] *should* return an [`Ok`] result. + /// + pub fn try_recover(&mut self) -> Result<(), TagIteratorError> { + let original_position = self.current_offset(); + loop { + if !self.ensure_data_read(1)? { + return Err(TagIteratorError::UnexpectedEOF { + tag_start: self.current_offset(), + tag_id: None, + tag_size: None, + partial_data: None, + }); + } + + self.internal_buffer_position += 1; + if self.peek_valid_tag_header().is_ok() { + break; + } + } + + // As part of recovery, update internal tag stack sizes so that we don't get "oversized children" errors after skipping corrupted data + let diff = self.current_offset() - original_position; + for tag in self.tag_stack.iter_mut() { + if let EBMLSize::Known(size) = &tag.size { + tag.size = EBMLSize::Known(size + diff); + } + } + + Ok(()) + } + + /// + /// Consumes self and returns the underlying read stream. + /// + /// Note that any leftover tags in the internal emission queue are lost, and any data read into [`TagIterator`]'s internal buffer is dropped. Therefore, constructing a new [`TagIterator`] using the returned stream may lead to data loss unless it is rewound. + /// + pub fn into_inner(self) -> R { + self.source + } + + /// + /// Gets a mutable reference to the underlying read stream. + /// + /// It is inadvisable to directly read from the underlying stream. + /// + pub fn get_mut(&mut self) -> &mut R { + &mut self.source + } + + /// + /// Gets a reference to the underlying read stream. + /// + /// It is inadvisable to directly read from the underlying stream. + /// + pub fn get_ref(&self) -> &R { + &self.source + } + + /// + /// Returns the byte offset of the last emitted tag. + /// + /// This function returns a byte index specifying the start of the last emitted tag in the context of the [`TagIterator`]'s source read stream. This value is *not guaranteed to always increase as the file is read*. Whenever the iterator emits a [`Master::End`] variant, [`Self::last_emitted_tag_offset()`] will reflect the start index of the "Master" tag, which will be before previous values that were obtainable when any children of the master were emitted. + /// + pub fn last_emitted_tag_offset(&self) -> usize { + self.last_emitted_tag_offset + } + + /// + /// Control whether the iterator should emit closing tags when it reaches EOF. + /// + /// By default, the iterator will emit [`Master::End`] items for all currently open tags when it reaches the end of the file. You may override this behavior by passing `false` to this method. + /// + /// This is recommended if you supply a [`std::io::Read`] source that can supply more data after reaching EOF, as in some streaming scenarios. + /// + pub fn emit_master_end_when_eof(&mut self, emit: bool) { + self.emit_master_end_when_eof = emit; + } + + #[inline(always)] + fn current_offset(&self) -> usize { + self.buffer_offset.unwrap_or(0) + self.internal_buffer_position + } + + fn private_read(&mut self, internal_buffer_start: usize) -> Result { + let bytes_read = self + .source + .read(&mut self.buffer[internal_buffer_start..]) + .map_err(|source| TagIteratorError::ReadError { source })?; + if bytes_read == 0 { + Ok(false) + } else { + self.buffered_byte_length += bytes_read; + Ok(true) + } + } + + fn ensure_capacity(&mut self, required_capacity: usize) { + if required_capacity > self.buffer.len() { + let mut new_buffer = Vec::from(&self.buffer[..]); + new_buffer.resize(required_capacity, 0); + self.buffer = new_buffer.into_boxed_slice(); + } + } + + fn ensure_data_read(&mut self, length: usize) -> Result { + if self.internal_buffer_position + length <= self.buffered_byte_length { + return Ok(true); + } + + if self.buffer_offset.is_none() { + if !self.private_read(0)? { + return Ok(false); + } + self.buffer_offset = Some(0); + self.internal_buffer_position = 0; + } else { + while self.internal_buffer_position + length > self.buffered_byte_length { + self.buffer + .copy_within(self.internal_buffer_position..self.buffered_byte_length, 0); + self.buffered_byte_length -= self.internal_buffer_position; + self.buffer_offset = Some(self.current_offset()); + self.internal_buffer_position = 0; + if !self.private_read(self.buffered_byte_length)? { + return Ok(false); + } + } + } + Ok(true) + } + + #[inline] + fn peek_valid_tag_header(&mut self) -> Result<(u64, Option, EBMLSize, usize), TagIteratorError> { + self.ensure_data_read(16)?; + let current_offset = self.current_offset(); + let available = &self.buffer[self.internal_buffer_position..self.buffered_byte_length]; + let header = + tag_parse::read_header::(available, current_offset)?.ok_or(TagIteratorError::UnexpectedEOF { + tag_start: current_offset, + tag_id: None, + tag_size: None, + partial_data: None, + })?; + let tag_id = header.id; + let spec_tag_type = header.data_type; + let size = header.size; + let header_len = header.len; + + if (self.allowed_errors & INVALID_TAG_ID_ERROR == 0) && spec_tag_type.is_none() { + return Err(TagIteratorError::CorruptedFileData(CorruptedFileError::InvalidTagId { + tag_id, + position: self.current_offset(), + })); + } + + if (self.allowed_errors & INVALID_HIERARCHY_ERROR == 0) && spec_tag_type.is_some() { + // Do not run check for raw tags ^^^^^^^^^^^^^^^^^^^^^^^ + if !self.has_determined_doc_path { + //Trust that the first tag in the stream is valid (like if the read stream was seeked to this location) + let path = ::get_path_by_id(tag_id); + if path.iter().all(|p| matches!(p, PathPart::Id(_))) { + //We only know the current path if we read a tag that is non-global + self.tag_stack = path.iter().map(|id| { + match id { + PathPart::Id(id) => { + ProcessingTag { + tag: ::get_master_tag(*id, Master::Start).unwrap_or_else(|| panic!("Bad specification implementation: Tag id 0x{:x?} type was in path, but could not get master tag!", id)), + size: EBMLSize::Unknown, + tag_start: 0, + data_start: 0, + } + }, + PathPart::Global(_) => unreachable!() + } + }).collect(); + self.has_determined_doc_path = true; + } + } + if self.has_determined_doc_path && !self.validate_tag_path(tag_id) { + return Err(TagIteratorError::CorruptedFileData( + CorruptedFileError::HierarchyError { + found_tag_id: tag_id, + current_parent_id: self.tag_stack.last().map(|tag| tag.tag.get_id()), + }, + )); + } + } + + if (self.allowed_errors & OVERSIZED_CHILD_ERROR == 0) + && size.is_known() + && self.is_invalid_tag_size(header_len + size.value()) + { + return Err(TagIteratorError::CorruptedFileData( + CorruptedFileError::OversizedChildElement { + position: self.current_offset(), + tag_id, + size: size.value(), + }, + )); + } + + if let Some(max_size) = self.max_allowed_tag_size { + if size.is_known() && size.value() > max_size { + return Err(TagIteratorError::CorruptedFileData( + CorruptedFileError::InvalidTagSize { + position: self.current_offset(), + tag_id, + size: size.value(), + }, + )); + } + } + + Ok((tag_id, spec_tag_type, size, header_len)) + } + + #[inline(always)] + fn read_valid_tag_header(&mut self) -> Result<(u64, Option, EBMLSize), TagIteratorError> { + let (tag_id, spec_tag_type, size, header_len) = self.peek_valid_tag_header()?; + + self.internal_buffer_position += header_len; + Ok((tag_id, spec_tag_type, size)) + } + + fn read_tag_data(&mut self, size: usize) -> Result, TagIteratorError> { + self.ensure_capacity(size); + if !self.ensure_data_read(size)? { + return Ok(None); + } + + self.internal_buffer_position += size; + Ok(Some( + &self.buffer[(self.internal_buffer_position - size)..self.internal_buffer_position], + )) + } + + fn read_tag(&mut self) -> Result, TagIteratorError> { + let tag_start = self.current_offset(); + + let (tag_id, spec_tag_type, size) = self.read_valid_tag_header()?; + + let data_start = self.current_offset(); + let raw_data = if matches!(spec_tag_type, Some(TagDataType::Master)) { + &[] + } else if let Known(size) = size { + if let Some(data) = self.read_tag_data(size)? { + data + } else { + return Err(TagIteratorError::UnexpectedEOF { + tag_start, + tag_id: Some(tag_id), + tag_size: Some(size), + partial_data: Some(self.buffer[self.internal_buffer_position..].to_vec()), + }); + } + } else { + return Err(TagIteratorError::CorruptedFileData( + CorruptedFileError::InvalidTagData { + tag_id, + position: tag_start, + }, + )); + }; + + let tag = if matches!(spec_tag_type, Some(TagDataType::Master)) { + TSpec::get_master_tag(tag_id, Master::Start).unwrap_or_else(|| { + panic!( + "Bad specification implementation: Tag id 0x{:x?} type was master, but could not get tag!", + tag_id + ) + }) + } else { + tag_parse::read_data_tag(tag_id, spec_tag_type, raw_data)? + }; + + Ok(ProcessingTag { + tag, + size, + tag_start, + data_start, + }) + } + + fn read_tag_checked(&mut self) -> Option, TagIteratorError>> { + if self.internal_buffer_position == self.buffered_byte_length { + //If we've already consumed the entire internal buffer + //ensure there is nothing else in the data source before returning `None` + let read_result = self.ensure_data_read(1); + match read_result { + Err(err) => return Some(Err(err)), + Ok(data_remaining) => { + if !data_remaining { + return None; + } + } + } + } + + if self.internal_buffer_position > self.buffered_byte_length { + panic!("read position exceeded buffer length"); + } + + Some(self.read_tag()) + } + + fn read_next(&mut self) { + //If we have reached the known end of any open master tags, queue that tag and all children to emit ends + let ended_tag_index = self + .tag_stack + .iter() + .position(|tag| matches!(tag.size, Known(size) if self.current_offset() >= tag.data_start + size)); + if let Some(index) = ended_tag_index { + self.emission_queue + .extend(self.tag_stack.drain(index..).map(|t| Ok((t.tag, t.tag_start))).rev()); + } + + if let Some(next_read) = self.read_tag_checked() { + if let Ok(next_tag) = &next_read { + while matches!(self.tag_stack.last(), Some(open_tag) if open_tag.size == Unknown) { + let open_tag = self.tag_stack.last().unwrap(); + let previous_tag_ended = open_tag.is_ended_by(next_tag.tag.get_id()); + + if previous_tag_ended { + let t = self.tag_stack.pop().unwrap(); + self.emission_queue.push_back(Ok((t.tag, t.tag_start))); + } else { + break; + } + } + + if let Some(Master::Start) = next_tag.tag.as_master() { + let tag_id = next_tag.tag.get_id(); + + self.tag_stack.push(ProcessingTag { + tag: TSpec::get_master_tag(tag_id, Master::End).unwrap(), + size: next_tag.size, + tag_start: next_tag.tag_start, + data_start: next_tag.data_start, + }); + + if self.tag_ids_to_buffer.contains(&tag_id) { + self.buffer_master(tag_id); + return; + } + } + } + + self.emission_queue.push_back(next_read.map(|r| (r.tag, r.tag_start))); + } else if self.emit_master_end_when_eof { + while let Some(tag) = self.tag_stack.pop() { + self.emission_queue.push_back(Ok((tag.tag, tag.tag_start))); + } + } + } + + fn buffer_master(&mut self, tag_id: u64) { + let tag_start = self.current_offset(); + let pre_queue_len = self.emission_queue.len(); + + let mut position = pre_queue_len; + 'endTagSearch: loop { + if position >= self.emission_queue.len() { + self.read_next(); + + if position >= self.emission_queue.len() { + self.emission_queue.push_back(Err(TagIteratorError::UnexpectedEOF { + tag_start, + tag_id: Some(tag_id), + tag_size: None, + partial_data: None, + })); + return; + } + } + + while position < self.emission_queue.len() { + if let Some(r) = self.emission_queue.get(position) { + match r { + Err(_) => break 'endTagSearch, + Ok(t) => { + if t.0.get_id() == tag_id && matches!(t.0.as_master(), Some(Master::End)) { + break 'endTagSearch; + } + } + } + } + position += 1; + } + } + + let mut children = self.emission_queue.split_off(pre_queue_len); + let split_to = position - pre_queue_len; + if children.get(split_to).unwrap().is_ok() { + let remaining = children.split_off(split_to).into_iter().skip(1); + let full_tag = Self::roll_up_children(tag_id, children.into_iter().map(|c| c.unwrap().0).collect()); + self.emission_queue.push_back(Ok((full_tag, tag_start))); + self.emission_queue.extend(remaining); + } else { + self.emission_queue.extend(children.drain(split_to..).take(1)); + } + } + + fn roll_up_children(tag_id: u64, children: Vec) -> TSpec { + let mut rolled_children = Vec::new(); + + let mut iter = children.into_iter(); + while let Some(child) = iter.next() { + if let Some(Master::Start) = child.as_master() { + let child_id = child.get_id(); + let subchildren = iter + .by_ref() + .take_while(|c| !matches!(c.as_master(), Some(Master::End)) || c.get_id() != child_id) + .collect(); + rolled_children.push(Self::roll_up_children(child_id, subchildren)); + } else { + rolled_children.push(child); + } + } + + TSpec::get_master_tag(tag_id, Master::Full(rolled_children)).unwrap_or_else(|| { + panic!( + "Bad specification implementation: Tag id 0x{:x?} type was master, but could not get tag!", + tag_id + ) + }) + } + + #[inline(always)] + fn validate_tag_path(&self, tag_id: u64) -> bool { + validate_tag_path::(tag_id, self.tag_stack.iter().map(|p| (p.tag.get_id(), p.size, 0))) + } + + #[inline(always)] + fn is_invalid_tag_size(&self, size: usize) -> bool { + self.tag_stack + .iter() + .filter(|p| p.size.is_known()) + .any(|t| (t.data_start + t.size.value()) < (self.current_offset() + size)) + } +} + +impl Iterator for TagIterator +where + TSpec: EbmlSpecification + EbmlTag + Clone, +{ + type Item = Result; + + fn next(&mut self) -> Option { + if self.emission_queue.is_empty() { + self.read_next(); + } + let next_item = self.emission_queue.pop_front(); + if let Some(Ok(ref tuple)) = next_item { + self.last_emitted_tag_offset = tuple.1; + } + next_item.map(|r| r.map(|t| t.0)) + } +} diff --git a/crates/ebml-iterable/src/tag_iterator_async.rs b/crates/ebml-iterable/src/tag_iterator_async.rs new file mode 100644 index 000000000..da33d4314 --- /dev/null +++ b/crates/ebml-iterable/src/tag_iterator_async.rs @@ -0,0 +1,191 @@ +use std::io::ErrorKind; +use std::iter::repeat; +use std::mem; +use ebml_iterable_specification::{EbmlSpecification, EbmlTag, Master, TagDataType}; +use futures::{AsyncRead, AsyncReadExt, Stream}; +use crate::error::{TagIteratorError, ToolError}; +use crate::errors::tag_iterator::CorruptedFileError; +use crate::tag_iterator_util::{EBMLSize, ProcessingTag}; +use crate::tag_iterator_util::EBMLSize::Known; +use crate::tools; + +/// +/// This Can be transformed into a [`Stream`] using [`into_stream`][TagIteratorAsync::into_stream], or consumed directly by calling [`.next().await`] in a loop. +/// +/// The struct can be created with the [`new()`][TagIteratorAsync::new] function on any source that implements the [`futures::AsyncRead`] trait. +/// +pub struct TagIteratorAsync + where + TSpec: EbmlSpecification + EbmlTag + Clone +{ + read: R, + buf: Vec, + offset: usize, + tag_stack: Vec> +} + +impl TagIteratorAsync + where + TSpec: EbmlSpecification + EbmlTag + Clone +{ + + pub fn new(read: R) -> Self { + Self { + read, + buf: Default::default(), + offset: 0, + tag_stack: Default::default() + } + } + + fn current_offset(&self) -> usize { + self.offset + } + + fn advance(&mut self, length: usize) { + self.offset += length; + self.buf.drain(0..length); + } + + fn advance_get(&mut self, length: usize) -> Vec { + self.offset += length; + let upper = self.buf.split_off(length); + mem::replace(&mut self.buf, upper) + } + + async fn ensure_data_read(&mut self, len: usize) -> Result { + let size = self.buf.len(); + if size < len { + let remaining = len - size; + self.buf.extend(repeat(0).take(remaining)); + if let Err(source) = self.read.read_exact(&mut self.buf[size..]).await { + return match source.kind() { + ErrorKind::UnexpectedEof => { + Ok(false) + } + _ => Err(TagIteratorError::ReadError { source }) + } + } + } + Ok(true) + } + + async fn read_tag_id(&mut self) -> Result { + self.ensure_data_read(8).await?; + match tools::read_vint(&self.buf).unwrap_or(Some((0, 1))) { + Some((value, length)) => { + self.advance(length); + Ok(value + (1 << (7 * length))) + }, + None => Err(TagIteratorError::UnexpectedEOF{ tag_start: self.current_offset(), tag_id: None, tag_size: None, partial_data: None }), + } + } + + async fn read_tag_size(&mut self) -> Result { + self.ensure_data_read(8).await?; + match tools::read_vint(&self.buf).or(Err(TagIteratorError::CorruptedFileData(CorruptedFileError::InvalidTagData { tag_id: 0, position: self.current_offset() })))? { + Some((value, length)) => { + self.advance(length); + Ok(EBMLSize::new(value, length)) + }, + None => Err(TagIteratorError::UnexpectedEOF{ tag_start: self.current_offset(), tag_id: None, tag_size: None, partial_data: None }), + } + } + + async fn read_tag_data(&mut self, size: usize) -> Result, TagIteratorError> { + if !self.ensure_data_read(size).await? { + return Err(TagIteratorError::UnexpectedEOF{ tag_start: self.current_offset(), tag_id: None, tag_size: None, partial_data: None }); + } + Ok(self.advance_get(size)) + } + + async fn read_tag(&mut self) -> Result { + let tag_id = self.read_tag_id().await?; + let spec_tag_type = TSpec::get_tag_data_type(tag_id); + let size = self.read_tag_size().await?; + let current_offset = self.current_offset(); + + let is_master = matches!(spec_tag_type, Some(TagDataType::Master)); + if is_master { + self.tag_stack.push(ProcessingTag { + tag: TSpec::get_master_tag(tag_id, Master::End).unwrap_or_else(|| panic!("Bad specification implementation: Tag id {} type was master, but could not get tag!", tag_id)), + size, + data_start: current_offset, + tag_start: 0, //not implemented here + }); + Ok(TSpec::get_master_tag(tag_id, Master::Start).unwrap_or_else(|| panic!("Bad specification implementation: Tag id {} type was master, but could not get tag!", tag_id))) + } else { + let size = if let Known(size) = size { + size + } else { + return Err(TagIteratorError::CorruptedFileData(CorruptedFileError::InvalidTagData { tag_id, position: current_offset })); + }; + + let raw_data = self.read_tag_data(size).await?; + let tag = match spec_tag_type { + Some(TagDataType::Master) => { unreachable!("Master should have been handled before querying data") }, + Some(TagDataType::UnsignedInt) => { + let val = tools::arr_to_u64(&raw_data).map_err(|e| TagIteratorError::CorruptedTagData{ tag_id, problem: e })?; + TSpec::get_unsigned_int_tag(tag_id, val).unwrap_or_else(|| panic!("Bad specification implementation: Tag id {} type was unsigned int, but could not get tag!", tag_id)) + }, + Some(TagDataType::Integer) => { + let val = tools::arr_to_i64(&raw_data).map_err(|e| TagIteratorError::CorruptedTagData{ tag_id, problem: e })?; + TSpec::get_signed_int_tag(tag_id, val).unwrap_or_else(|| panic!("Bad specification implementation: Tag id {} type was integer, but could not get tag!", tag_id)) + }, + Some(TagDataType::Utf8) => { + let val = String::from_utf8(raw_data.to_vec()).map_err(|e| TagIteratorError::CorruptedTagData{ tag_id, problem: ToolError::FromUtf8Error(raw_data, e) })?; + TSpec::get_utf8_tag(tag_id, val).unwrap_or_else(|| panic!("Bad specification implementation: Tag id {} type was utf8, but could not get tag!", tag_id)) + }, + Some(TagDataType::Binary) | None => { + TSpec::get_binary_tag(tag_id, &raw_data).unwrap_or_else(|| TSpec::get_raw_tag(tag_id, &raw_data)) + }, + Some(TagDataType::Float) => { + let val = tools::arr_to_f64(&raw_data).map_err(|e| TagIteratorError::CorruptedTagData{ tag_id, problem: e })?; + TSpec::get_float_tag(tag_id, val).unwrap_or_else(|| panic!("Bad specification implementation: Tag id {} type was float, but could not get tag!", tag_id)) + }, + }; + + match self.tag_stack.last() { + None => Ok(tag), + Some(previous_tag) => { + let previous_tag_ended = previous_tag.is_ended_by(tag_id); + + if previous_tag_ended { + Ok(mem::replace(self.tag_stack.last_mut().unwrap(), ProcessingTag { tag, size: Known(size), data_start: current_offset, tag_start: 0 }).into_inner()) + } else { + Ok(tag) + } + } + } + } + } + + /// can be consumed + pub async fn next(&mut self) -> Option> { + if let Some(tag) = self.tag_stack.pop() { + if let Known(size) = tag.size { + if self.current_offset() >= tag.data_start + size { + return Some(Ok(tag.tag)); + } + } + self.tag_stack.push(tag); + } + + match self.ensure_data_read(1).await { + Err(err) => return Some(Err(err)), + Ok(data_remaining) => { + if !data_remaining { + return self.tag_stack.pop().map(|tag| Ok(tag.into_inner())); + } + } + } + Some(self.read_tag().await) + } + + pub fn into_stream(self) -> impl Stream> { + futures::stream::unfold(self, |mut read| async { + let next = read.next().await; + next.map(move |it| (it, read)) + }) + } +} diff --git a/crates/ebml-iterable/src/tag_iterator_util.rs b/crates/ebml-iterable/src/tag_iterator_util.rs new file mode 100644 index 000000000..329d542f4 --- /dev/null +++ b/crates/ebml-iterable/src/tag_iterator_util.rs @@ -0,0 +1,125 @@ +use std::convert::TryInto; + +use ebml_iterable_specification::{EbmlSpecification, EbmlTag}; + +use crate::spec_util::is_ended_by; +use crate::tag_iterator_util::EBMLSize::{Known, Unknown}; + +#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] +pub enum EBMLSize { + Known(usize), + Unknown, +} + +impl EBMLSize { + pub fn new(size: u64, vint_length: usize) -> Self { + match vint_length { + 1 => { + if size == ((1 << (7)) - 1) { + return Unknown; + } + } + 2 => { + if size == ((1 << (7 * 2)) - 1) { + return Unknown; + } + } + 3 => { + if size == ((1 << (7 * 3)) - 1) { + return Unknown; + } + } + 4 => { + if size == ((1 << (7 * 4)) - 1) { + return Unknown; + } + } + 5 => { + if size == ((1 << (7 * 5)) - 1) { + return Unknown; + } + } + 6 => { + if size == ((1 << (7 * 6)) - 1) { + return Unknown; + } + } + 7 => { + if size == ((1 << (7 * 7)) - 1) { + return Unknown; + } + } + 8 => { + if size == ((1 << (7 * 8)) - 1) { + return Unknown; + } + } + _ => {} + } + + match size.try_into() { + Ok(value) => Known(value), + Err(_) => Unknown, + } + } + + #[inline(always)] + pub fn is_known(&self) -> bool { + matches!(&self, &EBMLSize::Known(_)) + } + + /// + /// # Panics + /// + /// Panics if the current variant is not EBMLSize::Known + /// + #[inline(always)] + pub fn value(&self) -> usize { + match &self { + EBMLSize::Known(val) => *val, + _ => panic!("Called EBMLSize::value() on an unknown size!"), + } + } +} + +#[derive(Copy, Clone, Debug)] +pub struct ProcessingTag +where + TSpec: EbmlSpecification + EbmlTag + Clone, +{ + pub tag: TSpec, + pub size: EBMLSize, + pub tag_start: usize, + pub data_start: usize, +} + +impl ProcessingTag +where + TSpec: EbmlSpecification + EbmlTag + Clone, +{ + pub fn is_ended_by(&self, id: u64) -> bool { + is_ended_by::(self.tag.get_id(), id) + } +} + +pub const DEFAULT_BUFFER_LEN: usize = 1024 * 64; + +/// +/// Used to relax rules on how strictly a [`TagIterator`](crate::TagIterator) should validate the read stream. +/// +pub enum AllowableErrors { + /// + /// Causes the [`TagIterator`](crate::TagIterator) to produce "RawTag" binary variants for any unknown tag ids rather than throwing an error. + /// + InvalidTagIds, + + /// + /// Causes the [`TagIterator`](crate::TagIterator) to emit tags even if they appear outside of their defined parent element. + /// + HierarchyProblems, + + /// + /// Causes the [`TagIterator`](crate::TagIterator) to emit tags even if they exceed the length of a parent element. + /// + OversizedTags, +} diff --git a/crates/ebml-iterable/src/tag_parse.rs b/crates/ebml-iterable/src/tag_parse.rs new file mode 100644 index 000000000..5e1c5b8fe --- /dev/null +++ b/crates/ebml-iterable/src/tag_parse.rs @@ -0,0 +1,110 @@ +use crate::errors::tag_iterator::{CorruptedFileError, TagIteratorError}; +use crate::errors::tool::ToolError; +use crate::specs::{EbmlSpecification, EbmlTag, TagDataType}; +use crate::tag_iterator_util::EBMLSize; +use crate::tools; + +pub(crate) struct TagHeader { + pub id: u64, + pub data_type: Option, + pub size: EBMLSize, + pub len: usize, +} + +pub(crate) fn read_header(input: &[u8], position: usize) -> Result, TagIteratorError> +where + TSpec: EbmlSpecification + EbmlTag + Clone, +{ + let Some(first) = input.first().copied() else { + return Ok(None); + }; + let id_len = if first == 0 { 1 } else { 8 - first.ilog2() as usize }; + if input.len() < id_len { + return Ok(None); + } + + let id = input[..id_len] + .iter() + .fold(0u64, |value, byte| (value << 8) + u64::from(*byte)); + let Some((size, size_len)) = tools::read_vint(&input[id_len..]).map_err(|_| { + TagIteratorError::CorruptedFileData(CorruptedFileError::InvalidTagData { tag_id: id, position }) + })? + else { + return Ok(None); + }; + let data_type = TSpec::get_tag_data_type(id); + + if matches!( + data_type, + Some(TagDataType::UnsignedInt | TagDataType::Integer | TagDataType::Float) + ) && size > 8 + { + return Err(TagIteratorError::CorruptedFileData( + CorruptedFileError::InvalidTagData { tag_id: id, position }, + )); + } + + Ok(Some(TagHeader { + id, + data_type, + size: EBMLSize::new(size, size_len), + len: id_len + size_len, + })) +} + +pub(crate) fn read_data_tag( + id: u64, + data_type: Option, + raw_data: &[u8], +) -> Result +where + TSpec: EbmlSpecification + EbmlTag + Clone, +{ + let tag = match data_type { + Some(TagDataType::Master) => unreachable!(), + Some(TagDataType::UnsignedInt) => { + let value = if raw_data.is_empty() { + 0 + } else { + tools::arr_to_u64(raw_data) + .map_err(|problem| TagIteratorError::CorruptedTagData { tag_id: id, problem })? + }; + TSpec::get_unsigned_int_tag(id, value) + } + Some(TagDataType::Integer) => { + let value = if raw_data.is_empty() { + 0 + } else { + tools::arr_to_i64(raw_data) + .map_err(|problem| TagIteratorError::CorruptedTagData { tag_id: id, problem })? + }; + TSpec::get_signed_int_tag(id, value) + } + Some(TagDataType::Utf8) => { + let value = String::from_utf8(raw_data.to_vec()).map_err(|error| TagIteratorError::CorruptedTagData { + tag_id: id, + problem: ToolError::FromUtf8Error(raw_data.to_vec(), error), + })?; + TSpec::get_utf8_tag(id, value) + } + Some(TagDataType::Binary) => TSpec::get_binary_tag(id, raw_data), + Some(TagDataType::Float) => { + let value = if raw_data.is_empty() { + 0.0 + } else { + tools::arr_to_f64(raw_data) + .map_err(|problem| TagIteratorError::CorruptedTagData { tag_id: id, problem })? + }; + TSpec::get_float_tag(id, value) + } + None => return Ok(TSpec::get_raw_tag(id, raw_data)), + } + .unwrap_or_else(|| { + panic!( + "Bad specification implementation: Tag id 0x{:x?} had an incompatible data type!", + id + ) + }); + + Ok(tag) +} diff --git a/crates/ebml-iterable/src/tag_writer.rs b/crates/ebml-iterable/src/tag_writer.rs new file mode 100644 index 000000000..35e3ad537 --- /dev/null +++ b/crates/ebml-iterable/src/tag_writer.rs @@ -0,0 +1,760 @@ +use std::convert::{TryFrom, TryInto}; +use std::io::Write; + +use super::errors::tag_writer::TagWriterError; +use super::specs::{EbmlSpecification, EbmlTag, Master, TagDataType}; +use super::tag_iterator_util::EBMLSize::{self, Known, Unknown}; +use super::tools::{is_vint, Vint}; +use crate::errors::tool::ToolError; +use crate::spec_util::validate_tag_path; + +/// +/// Options that can be passed to the writer to customize written output +/// +pub struct WriteOptions { + size_byte_length: Option, + unknown_sized_element: bool, +} + +impl WriteOptions { + /// + /// Specifies the byte length for the element's "size" + /// + /// This function generates [`WriteOptions`] that will force the Element Data Size to be a specific number of bytes for the written tag. + /// + /// ## Panics + /// + /// This method asserts that `len` is within 1-8 (inclusive). Values outside this range will cause a panic. + /// + pub fn set_size_byte_count(len: usize) -> Self { + assert!( + len > 0 && len < 9, + "Size byte count for written vints must be within 1-8 (inclusive)" + ); + Self { + size_byte_length: Some(len), + unknown_sized_element: false, + } + } + + /// + /// Specifies that the element has an Unknown Data Size. + /// + /// The [`WriteOptions`] generated by this function allow you to start a tag that doesn't have a known size. Useful for streaming, or when the data is expected to be too large to fit into memory. This should *only* be used with Master type tags. + /// + pub fn is_unknown_sized_element() -> Self { + Self { + size_byte_length: None, + unknown_sized_element: true, + } + } +} + +/// +/// Provides a tool to write EBML files based on Tags. Writes to a destination that implements [`std::io::Write`]. +/// +/// Unlike the [`TagIterator`][`super::TagIterator`], this does not require a specification to write data. This writer provides the [`write_raw()`](#method.write_raw) method which can be used to write data that is outside of any specification. The regular [`write()`](#method.write) method can be used to write any `TSpec` objects regardless of whether they came from a [`TagIterator`][`super::TagIterator`] or not. +/// +pub struct TagWriter { + dest: W, + open_tags: Vec<(u64, EBMLSize, usize)>, + working_buffer: Vec, +} + +impl TagWriter { + /// + /// Returns a new [`TagWriter`] instance. + /// + /// The `dest` parameter can be anything that implements [`std::io::Write`]. + /// + pub fn new(dest: W) -> Self { + TagWriter { + dest, + open_tags: Vec::new(), + working_buffer: Vec::new(), + } + } + + /// + /// Consumes self and returns the underlying write stream. + /// + /// Any incomplete tags are written out before returning the stream. + /// + pub fn into_inner(mut self) -> Result { + self.flush()?; + Ok(self.dest) + } + + /// + /// Gets a mutable reference to the underlying write stream. + /// + pub fn get_mut(&mut self) -> &mut W { + &mut self.dest + } + + /// + /// Gets a reference to the underlying write stream. + /// + pub fn get_ref(&self) -> &W { + &self.dest + } + + fn start_tag(&mut self, id: u64, size_length: usize) { + self.open_tags.push((id, Known(self.working_buffer.len()), size_length)); + } + + fn start_unknown_size_tag(&mut self, id: u64) { + self.working_buffer + .extend(id.to_be_bytes().iter().skip_while(|&v| *v == 0u8)); + self.working_buffer.extend_from_slice(&(u64::MAX >> 7).to_be_bytes()); + self.open_tags.push((id, Unknown, 0)); + } + + fn end_tag(&mut self, id: u64) -> Result<(), TagWriterError> { + match self.open_tags.pop() { + Some(open_tag) => { + if open_tag.0 == id { + if let Known(start) = open_tag.1 { + let size: u64 = self + .working_buffer + .len() + .checked_sub(start) + .expect("overflow subtracting tag size from working buffer length") + .try_into() + .expect("couldn't convert usize to u64"); + + match open_tag.2 { + 1 => { + let size_vint = size + .as_vint_with_length::<1>() + .map_err(|e| TagWriterError::TagSizeError(e.to_string()))?; + self.working_buffer.splice( + start..start, + open_tag + .0 + .to_be_bytes() + .iter() + .skip_while(|&v| *v == 0u8) + .chain(size_vint.iter()) + .copied(), + ); + } + 2 => { + let size_vint = size + .as_vint_with_length::<2>() + .map_err(|e| TagWriterError::TagSizeError(e.to_string()))?; + self.working_buffer.splice( + start..start, + open_tag + .0 + .to_be_bytes() + .iter() + .skip_while(|&v| *v == 0u8) + .chain(size_vint.iter()) + .copied(), + ); + } + 3 => { + let size_vint = size + .as_vint_with_length::<3>() + .map_err(|e| TagWriterError::TagSizeError(e.to_string()))?; + self.working_buffer.splice( + start..start, + open_tag + .0 + .to_be_bytes() + .iter() + .skip_while(|&v| *v == 0u8) + .chain(size_vint.iter()) + .copied(), + ); + } + 4 => { + let size_vint = size + .as_vint_with_length::<4>() + .map_err(|e| TagWriterError::TagSizeError(e.to_string()))?; + self.working_buffer.splice( + start..start, + open_tag + .0 + .to_be_bytes() + .iter() + .skip_while(|&v| *v == 0u8) + .chain(size_vint.iter()) + .copied(), + ); + } + 5 => { + let size_vint = size + .as_vint_with_length::<5>() + .map_err(|e| TagWriterError::TagSizeError(e.to_string()))?; + self.working_buffer.splice( + start..start, + open_tag + .0 + .to_be_bytes() + .iter() + .skip_while(|&v| *v == 0u8) + .chain(size_vint.iter()) + .copied(), + ); + } + 6 => { + let size_vint = size + .as_vint_with_length::<6>() + .map_err(|e| TagWriterError::TagSizeError(e.to_string()))?; + self.working_buffer.splice( + start..start, + open_tag + .0 + .to_be_bytes() + .iter() + .skip_while(|&v| *v == 0u8) + .chain(size_vint.iter()) + .copied(), + ); + } + 7 => { + let size_vint = size + .as_vint_with_length::<7>() + .map_err(|e| TagWriterError::TagSizeError(e.to_string()))?; + self.working_buffer.splice( + start..start, + open_tag + .0 + .to_be_bytes() + .iter() + .skip_while(|&v| *v == 0u8) + .chain(size_vint.iter()) + .copied(), + ); + } + 8 => { + let size_vint = size + .as_vint_with_length::<8>() + .map_err(|e| TagWriterError::TagSizeError(e.to_string()))?; + self.working_buffer.splice( + start..start, + open_tag + .0 + .to_be_bytes() + .iter() + .skip_while(|&v| *v == 0u8) + .chain(size_vint.iter()) + .copied(), + ); + } + _ => { + let size_vint = size + .as_vint() + .map_err(|e| TagWriterError::TagSizeError(e.to_string()))?; + self.working_buffer.splice( + start..start, + open_tag + .0 + .to_be_bytes() + .iter() + .skip_while(|&v| *v == 0u8) + .chain(size_vint.iter()) + .copied(), + ); + } + }; + } + Ok(()) + } else { + Err(TagWriterError::UnexpectedClosingTag { + tag_id: id, + expected_id: Some(open_tag.0), + }) + } + } + None => Err(TagWriterError::UnexpectedClosingTag { + tag_id: id, + expected_id: None, + }), + } + } + + fn private_flush(&mut self) -> Result<(), TagWriterError> { + self.dest + .write_all(self.working_buffer.drain(..).as_slice()) + .map_err(|source| TagWriterError::WriteError { source })?; + self.dest + .flush() + .map_err(|source| TagWriterError::WriteError { source }) + } + + fn write_unsigned_int_tag(&mut self, id: u64, data: &u64) -> Result<(), TagWriterError> { + self.working_buffer + .extend(id.to_be_bytes().iter().skip_while(|&v| *v == 0u8)); + let data = *data; + + u8::try_from(data) + .map(|n| { + if SIZE_LENGTH == 0 { + self.working_buffer.push(0x81); // vint representation of "1" + self.working_buffer.extend_from_slice(&n.to_be_bytes()); + } else { + self.working_buffer + .extend_from_slice(&1u8.as_vint_with_length::()?); + self.working_buffer.extend_from_slice(&n.to_be_bytes()); + } + Ok(()) + }) + .or_else(|_| { + u16::try_from(data).map(|n| { + if SIZE_LENGTH == 0 { + self.working_buffer.push(0x82); // vint representation of "2" + self.working_buffer.extend_from_slice(&n.to_be_bytes()); + } else { + self.working_buffer + .extend_from_slice(&2u8.as_vint_with_length::()?); + self.working_buffer.extend_from_slice(&n.to_be_bytes()); + } + Ok(()) + }) + }) + .or_else(|_| { + u32::try_from(data).map(|n| { + if SIZE_LENGTH == 0 { + self.working_buffer.push(0x84); // vint representation of "4" + self.working_buffer.extend_from_slice(&n.to_be_bytes()); + } else { + self.working_buffer + .extend_from_slice(&4u8.as_vint_with_length::()?); + self.working_buffer.extend_from_slice(&n.to_be_bytes()); + } + Ok(()) + }) + }) + .unwrap_or_else(|_| { + if SIZE_LENGTH == 0 { + self.working_buffer.push(0x88); // vint representation of "8" + self.working_buffer.extend_from_slice(&data.to_be_bytes()); + } else { + self.working_buffer + .extend_from_slice(&8u8.as_vint_with_length::()?); + self.working_buffer.extend_from_slice(&data.to_be_bytes()); + } + Ok(()) + }) + .map_err(|err: ToolError| TagWriterError::TagSizeError(err.to_string())) + } + + fn write_signed_int_tag(&mut self, id: u64, data: &i64) -> Result<(), TagWriterError> { + self.working_buffer + .extend(id.to_be_bytes().iter().skip_while(|&v| *v == 0u8)); + let data = *data; + i8::try_from(data) + .map(|n| { + if SIZE_LENGTH == 0 { + self.working_buffer.push(0x81); // vint representation of "1" + self.working_buffer.extend_from_slice(&n.to_be_bytes()); + } else { + self.working_buffer + .extend_from_slice(&1u8.as_vint_with_length::()?); + self.working_buffer.extend_from_slice(&n.to_be_bytes()); + } + Ok(()) + }) + .or_else(|_| { + i16::try_from(data).map(|n| { + if SIZE_LENGTH == 0 { + self.working_buffer.push(0x82); // vint representation of "2" + self.working_buffer.extend_from_slice(&n.to_be_bytes()); + } else { + self.working_buffer + .extend_from_slice(&2u8.as_vint_with_length::()?); + self.working_buffer.extend_from_slice(&n.to_be_bytes()); + } + Ok(()) + }) + }) + .or_else(|_| { + i32::try_from(data).map(|n| { + if SIZE_LENGTH == 0 { + self.working_buffer.push(0x84); // vint representation of "4" + self.working_buffer.extend_from_slice(&n.to_be_bytes()); + } else { + self.working_buffer + .extend_from_slice(&4u8.as_vint_with_length::()?); + self.working_buffer.extend_from_slice(&n.to_be_bytes()); + } + Ok(()) + }) + }) + .unwrap_or_else(|_| { + if SIZE_LENGTH == 0 { + self.working_buffer.push(0x88); // vint representation of "8" + self.working_buffer.extend_from_slice(&data.to_be_bytes()); + } else { + self.working_buffer + .extend_from_slice(&8u8.as_vint_with_length::()?); + self.working_buffer.extend_from_slice(&data.to_be_bytes()); + } + Ok(()) + }) + .map_err(|err: ToolError| TagWriterError::TagSizeError(err.to_string())) + } + + fn write_utf8_tag(&mut self, id: u64, data: &str) -> Result<(), TagWriterError> { + self.working_buffer + .extend(id.to_be_bytes().iter().skip_while(|&v| *v == 0u8)); + + let slice: &[u8] = data.as_bytes(); + let size: u64 = slice.len().try_into().expect("couldn't convert usize to u64"); + if SIZE_LENGTH == 0 { + let size_vint = size + .as_vint() + .map_err(|e| TagWriterError::TagSizeError(e.to_string()))?; + self.working_buffer.extend_from_slice(&size_vint); + } else { + let size_vint = size + .as_vint_with_length::() + .map_err(|e| TagWriterError::TagSizeError(e.to_string()))?; + self.working_buffer.extend_from_slice(&size_vint); + }; + + self.working_buffer.extend_from_slice(slice); + Ok(()) + } + + fn write_binary_tag(&mut self, id: u64, data: &[u8]) -> Result<(), TagWriterError> { + self.working_buffer + .extend(id.to_be_bytes().iter().skip_while(|&v| *v == 0u8)); + + let size: u64 = data.len().try_into().expect("couldn't convert usize to u64"); + if SIZE_LENGTH == 0 { + let size_vint = size + .as_vint() + .map_err(|e| TagWriterError::TagSizeError(e.to_string()))?; + self.working_buffer.extend_from_slice(&size_vint); + } else { + let size_vint = size + .as_vint_with_length::() + .map_err(|e| TagWriterError::TagSizeError(e.to_string()))?; + self.working_buffer.extend_from_slice(&size_vint); + } + + self.working_buffer.extend_from_slice(data); + Ok(()) + } + + fn write_float_tag(&mut self, id: u64, data: &f64) -> Result<(), TagWriterError> { + self.working_buffer + .extend(id.to_be_bytes().iter().skip_while(|&v| *v == 0u8)); + if SIZE_LENGTH == 0 { + self.working_buffer.push(0x88); // vint representation of "8" + } else { + let size_vint = 8u8 + .as_vint_with_length::() + .map_err(|e| TagWriterError::TagSizeError(e.to_string()))?; + self.working_buffer.extend_from_slice(&size_vint); + } + self.working_buffer.extend_from_slice(&data.to_be_bytes()); + Ok(()) + } + + /// + /// Write a tag to this instance's destination. + /// + /// This method writes a tag from any specification. There are no restrictions on the type of specification being written - it simply needs to implement the [`EbmlSpecification`] and [`EbmlTag`] traits. + /// + /// ## Errors + /// + /// This method can error if there is a problem writing the input tag. The different possible error states are enumerated in [`TagWriterError`]. + /// + /// ## Panics + /// + /// This method can panic if `` is an internally inconsistent specification (i.e. it claims that a specific tag variant is a specific data type but it is not). This won't happen if the specification being used was created using the [`#[ebml_specification]`](https://docs.rs/ebml-iterable-specification-derive/latest/ebml_iterable_specification_derive/attr.ebml_specification.html) attribute macro. + /// + /// ## Examples + /// + /// ```no_run + /// use std::fs::File; + /// use ebml_iterable::TagWriter; + /// use ebml_iterable::specs::Master; + /// # use ebml_iterable_specification::empty_spec::EmptySpec; + /// + /// # fn main() -> Result<(), Box> { + /// let mut file = File::create("my_ebml_file.ebml")?; + /// let mut my_writer = TagWriter::new(&mut file); + /// my_writer.write(&EmptySpec::with_children( + /// 0x1a45dfa3, + /// vec![EmptySpec::with_data(0x18538067, &[0x01])]) + /// )?; + /// # Ok(()) + /// # } + /// ``` + /// + pub fn write + EbmlTag + Clone>( + &mut self, + tag: &TSpec, + ) -> Result<(), TagWriterError> { + self.write_advanced( + tag, + WriteOptions { + size_byte_length: None, + unknown_sized_element: false, + }, + ) + } + + /// + /// Write a tag to this instance's destination using advanced options. + /// + /// This method is just like the normal [`write()`](#method.write) method, but allows for tailoring the output binary to better suit your needs. See [`WriteOptions`] for more detail on available options. + /// + /// ## Errors + /// + /// This method can error if there is a problem writing the input tag. The different possible error states are enumerated in [`TagWriterError`]. + /// + /// ## Panics + /// + /// This method can panic if `` is an internally inconsistent specification (i.e. it claims that a specific tag variant is a specific data type but it is not). This won't happen if the specification being used was created using the [`#[ebml_specification]`](https://docs.rs/ebml-iterable-specification-derive/latest/ebml_iterable_specification_derive/attr.ebml_specification.html) attribute macro. + /// + pub fn write_advanced + EbmlTag + Clone>( + &mut self, + tag: &TSpec, + options: WriteOptions, + ) -> Result<(), TagWriterError> { + let tag_id = tag.get_id(); + let tag_type = TSpec::get_tag_data_type(tag_id); + + if options.unknown_sized_element { + match tag_type { + Some(TagDataType::Master) => {} + _ => { + return Err(TagWriterError::TagSizeError(format!( + "Cannot write an unknown size for tag of type {tag_type:?}" + ))) + } + }; + self.start_unknown_size_tag(tag_id); + } else { + let should_validate = tag_type.is_some() + && (!matches!(tag_type, Some(TagDataType::Master)) + || !matches!( + tag.as_master().unwrap_or_else(|| panic!( + "Bad specification implementation: Tag id {} type was master, but could not get tag!", + tag_id + )), + Master::End + )); + if should_validate && !validate_tag_path::(tag_id, self.open_tags.iter().copied()) { + return Err(TagWriterError::UnexpectedTag { + tag_id, + current_path: self.open_tags.iter().map(|t| t.0).collect(), + }); + } + + match options.size_byte_length { + Some(1) => self.write_explicit_sized::(tag, tag_id, tag_type)?, + Some(2) => self.write_explicit_sized::(tag, tag_id, tag_type)?, + Some(3) => self.write_explicit_sized::(tag, tag_id, tag_type)?, + Some(4) => self.write_explicit_sized::(tag, tag_id, tag_type)?, + Some(5) => self.write_explicit_sized::(tag, tag_id, tag_type)?, + Some(6) => self.write_explicit_sized::(tag, tag_id, tag_type)?, + Some(7) => self.write_explicit_sized::(tag, tag_id, tag_type)?, + Some(8) => self.write_explicit_sized::(tag, tag_id, tag_type)?, + _ => self.write_explicit_sized::(tag, tag_id, tag_type)?, + } + } + + Ok(()) + } + + fn write_explicit_sized + EbmlTag + Clone, const SIZE_LENGTH: usize>( + &mut self, + tag: &TSpec, + tag_id: u64, + tag_type: Option, + ) -> Result<(), TagWriterError> { + assert!(SIZE_LENGTH < 9, "Vint length must be less than 9 bytes"); + match tag_type { + Some(TagDataType::UnsignedInt) => { + let val = tag.as_unsigned_int().unwrap_or_else(|| { + panic!( + "Bad specification implementation: Tag id {} type was unsigned int, but could not get tag!", + tag_id + ) + }); + self.write_unsigned_int_tag::(tag_id, val)? + } + Some(TagDataType::Integer) => { + let val = tag.as_signed_int().unwrap_or_else(|| { + panic!( + "Bad specification implementation: Tag id {} type was integer, but could not get tag!", + tag_id + ) + }); + self.write_signed_int_tag::(tag_id, val)? + } + Some(TagDataType::Utf8) => { + let val = tag.as_utf8().unwrap_or_else(|| { + panic!( + "Bad specification implementation: Tag id {} type was utf8, but could not get tag!", + tag_id + ) + }); + self.write_utf8_tag::(tag_id, val)? + } + Some(TagDataType::Binary) => { + let val = tag.as_binary().unwrap_or_else(|| { + panic!( + "Bad specification implementation: Tag id {} type was binary, but could not get tag!", + tag_id + ) + }); + self.write_binary_tag::(tag_id, val)? + } + Some(TagDataType::Float) => { + let val = tag.as_float().unwrap_or_else(|| { + panic!( + "Bad specification implementation: Tag id {} type was float, but could not get tag!", + tag_id + ) + }); + self.write_float_tag::(tag_id, val)? + } + Some(TagDataType::Master) => { + let position = tag.as_master().unwrap_or_else(|| { + panic!( + "Bad specification implementation: Tag id {} type was master, but could not get tag!", + tag_id + ) + }); + + match position { + Master::Start => self.start_tag(tag_id, SIZE_LENGTH), + Master::End => self.end_tag(tag_id)?, + Master::Full(children) => { + self.start_tag(tag_id, SIZE_LENGTH); + for child in children { + self.write(child)?; + } + self.end_tag(tag_id)?; + } + } + } + None => { + // Should be a "raw tag" + if !is_vint(tag_id) { + return Err(TagWriterError::TagIdError(tag_id)); + } else { + let val = tag.as_binary().unwrap_or_else(|| panic!("Bad specification implementation: Tag id {} type was raw tag, but could not get binary data!", tag_id)); + self.write_binary_tag::(tag_id, val)? + } + } + } + + if !self.open_tags.iter().any(|t| matches!(t.1, Known(_))) { + self.private_flush() + } else { + Ok(()) + } + } + + /// + /// Write a tag with an unknown size to this instance's destination. + /// + /// DEPRECATED - Prefer using the [`write_advanced()`](#method.write_advanced) method with [`WriteOptions`] obtained from their [`is_unknown_sized_element()`](struct.WriteOptions.html#method.is_unknown_sized_element) instead. + /// + /// This method allows you to start a tag that doesn't have a known size. Useful for streaming, or when the data is expected to be too large to fit into memory. This method can *only* be used on Master type tags. + /// + /// ## Errors + /// + /// This method will return an error if the input tag is not a Master type tag, as those are the only types allowed to be of unknown size. + /// + #[deprecated( + since = "0.6.0", + note = "Please use 'write_advanced' with WriteOptions obtained using 'is_unknown_sized_element' instead" + )] + pub fn write_unknown_size + EbmlTag + Clone>( + &mut self, + tag: &TSpec, + ) -> Result<(), TagWriterError> { + let tag_id = tag.get_id(); + let tag_type = TSpec::get_tag_data_type(tag_id); + match tag_type { + Some(TagDataType::Master) => {} + _ => { + return Err(TagWriterError::TagSizeError(format!( + "Cannot write an unknown size for tag of type {tag_type:?}" + ))) + } + }; + self.start_unknown_size_tag(tag_id); + Ok(()) + } + + /// + /// Write raw tag data to this instance's destination. + /// + /// This method allows writing any tag id with any arbitrary data without using a specification. Specifications should generally provide an `Unknown` variant to handle arbitrary unknown data which can be written through the regular [`write()`](#method.write) method, so use of this method is typically discouraged. + /// + /// ## Errors + /// + /// This method can error if there is a problem writing the input tag. The different possible error states are enumerated in [`TagWriterError`]. + /// + /// ## Examples + /// + /// ```no_run + /// use std::fs::File; + /// use ebml_iterable::TagWriter; + /// + /// # fn main() -> Result<(), Box> { + /// let mut file = File::create("my_ebml_file.ebml")?; + /// let mut my_writer = TagWriter::new(&mut file); + /// my_writer.write_raw(0x1a45dfa3, &[0x18, 0x53, 0x80, 0x67, 0x81, 0x01])?; + /// # Ok(()) + /// # } + /// ``` + /// + pub fn write_raw(&mut self, tag_id: u64, data: &[u8]) -> Result<(), TagWriterError> { + self.write_binary_tag::<0>(tag_id, data)?; + + if !self.open_tags.iter().any(|t| matches!(t.1, Known(_))) { + self.private_flush() + } else { + Ok(()) + } + } + + /// + /// Attempts to flush all unwritten tags to the underlying destination. + /// + /// This method can be used to finalize any open [`Master`] type tags that have not been ended. The writer makes an attempt to close every open tag and write all bytes to the instance's destination. + /// + /// ## Errors + /// + /// This method can error if there is a problem writing to the destination. + /// + pub fn flush(&mut self) -> Result<(), TagWriterError> { + while let Some(id) = self.open_tags.last().map(|t| t.0) { + self.end_tag(id)?; + } + self.private_flush() + } + + //TODO: panic on drop if there is an open tag that hasn't been written. Or maybe flush stream of any open tags? +} + +#[cfg(test)] +mod tests { + use std::io::Cursor; + + use super::super::tools::Vint; + use super::TagWriter; + + #[test] + fn write_ebml_tag() { + let mut dest = Cursor::new(Vec::new()); + let mut writer = TagWriter::new(&mut dest); + writer.write_raw(0x1a45dfa3, &[]).expect("Error writing tag"); + + let zero_size = 0u64.as_vint().expect("Error converting [0] to vint")[0]; + assert_eq!(vec![0x1a, 0x45, 0xdf, 0xa3, zero_size], dest.get_ref().to_vec()); + } +} diff --git a/crates/ebml-iterable/src/tools.rs b/crates/ebml-iterable/src/tools.rs new file mode 100644 index 000000000..882910aa7 --- /dev/null +++ b/crates/ebml-iterable/src/tools.rs @@ -0,0 +1,523 @@ +//! +//! Contains a number of tools that are useful when working with EBML encoded files. +//! + +use std::convert::TryInto; + +use super::errors::tool::ToolError; + +/// +/// Trait to enable easy serialization to a vint. +/// +/// This is only available for types that can be cast as `u64`. +/// +pub trait Vint: Into + Copy { + /// + /// Returns a representation of the current value as a vint array. + /// + /// # Errors + /// + /// This can return an error if the value is too large to be representable as a vint. + /// + fn as_vint(self) -> Result, ToolError> { + let val: u64 = self.into(); + check_size_u64(val, 8)?; + + if val < (1 << 7) { + Ok(as_vint_no_check_u64::<1>(val).to_vec()) + } else if val < (1 << (7 * 2)) { + Ok(as_vint_no_check_u64::<2>(val).to_vec()) + } else if val < (1 << (7 * 3)) { + Ok(as_vint_no_check_u64::<3>(val).to_vec()) + } else if val < (1 << (7 * 4)) { + Ok(as_vint_no_check_u64::<4>(val).to_vec()) + } else if val < (1 << (7 * 5)) { + Ok(as_vint_no_check_u64::<5>(val).to_vec()) + } else if val < (1 << (7 * 6)) { + Ok(as_vint_no_check_u64::<6>(val).to_vec()) + } else if val < (1 << (7 * 7)) { + Ok(as_vint_no_check_u64::<7>(val).to_vec()) + } else { + Ok(as_vint_no_check_u64::<8>(val).to_vec()) + } + } + + /// + /// Returns a representation of the current value as a vint array with a specified length. + /// + /// # Errors + /// + /// This can return an error if the value is too large to be representable as a vint. + /// + fn as_vint_with_length(&self) -> Result<[u8; LENGTH], ToolError> { + let val: u64 = (*self).into(); + check_size_u64(val, LENGTH)?; + Ok(as_vint_no_check_u64::(val)) + } +} + +impl Vint for u64 {} +impl Vint for u32 {} +impl Vint for u16 {} +impl Vint for u8 {} + +#[inline] +fn check_size_u64(val: u64, max_length: usize) -> Result<(), ToolError> { + if val >= 1 << (max_length * 7) { + Err(ToolError::WriteVintOverflow(val)) + } else { + Ok(()) + } +} + +#[inline] +fn as_vint_no_check_u64(val: u64) -> [u8; LENGTH] { + let mut bytes: [u8; 8] = val.to_be_bytes(); + bytes[8 - LENGTH] |= 1 << (8 - LENGTH); + bytes[8 - LENGTH..].try_into().expect("8 - (8-length) != length !?!?") +} + +/// +/// Reads a vint from the beginning of the input array slice. +/// +/// This method returns an option with the `None` variant used to indicate there was not enough data in the buffer to completely read a vint. +/// +/// The returned tuple contains the value of the vint (`u64`) and the length of the vint (`usize`). The length will be less than or equal to the length of the input slice. +/// +/// # Errors +/// +/// This method can return a `ToolError` if the input array cannot be read as a vint. +/// +pub fn read_vint(buffer: &[u8]) -> Result, ToolError> { + if buffer.is_empty() { + return Ok(None); + } + + if buffer[0] == 0 { + return Err(ToolError::ReadVintOverflow); + } + + let length = 8 - buffer[0].ilog2() as usize; + + if length > buffer.len() { + // Not enough data in the buffer to read out the vint value + return Ok(None); + } + + let mut value = buffer[0] as u64; + value -= 1 << (8 - length); + + for item in buffer.iter().take(length).skip(1) { + value <<= 8; + value += *item as u64; + } + + Ok(Some((value, length))) +} + +pub fn is_vint(val: u64) -> bool { + if val == 0 { + return false; + } + + val.ilog2().is_multiple_of(7) +} + +/// +/// Trait to enable easy serialization to a signed vint. +/// +/// This is only available for types that can be cast as `i64`. A signed vint can be written as a variable number of bytes just like a regular vint, but the value portion of the vint is expressed in two's complement notation. +/// +/// For example, the decimal number "-33" would be written as [0xDF = 1101 1111]. This value is determined by first taking the two's complement of 33 [0x21 = 0010 0001] **but only using the bits available for the vint value**. In this case, that is 7 bits (because the vint marker takes up the 8th bit). The two's complement is [101 1111]. A handy calculator for two's complement can be found [here](https://www.omnicalculator.com/math/twos-complement). Once the two's complement has been found, simply prepend the vint marker as usual to get [1101 1111 = 0xDF]. +/// +/// Some more examples: +/// ``` +/// use ebml_iterable::tools::SignedVint; +/// +/// # fn main() -> Result<(), Box> { +/// assert_eq!(vec![0xDF], (-33i64).as_signed_vint().unwrap()); +/// assert_eq!(vec![0x40, 0xC8], (200i64).as_signed_vint().unwrap()); +/// assert_eq!(vec![0x7F, 0x38], (-200i64).as_signed_vint().unwrap()); +/// assert_eq!(vec![0xFF], (-1i64).as_signed_vint().unwrap()); +/// # Ok(()) +/// # } +/// ``` +pub trait SignedVint: Into + Copy { + /// + /// Returns a representation of the current value as a vint array. + /// + /// # Errors + /// + /// This can return an error if the value is outside of the range that can be represented as a vint. + /// + fn as_signed_vint(&self) -> Result, ToolError> { + let val: i64 = (*self).into(); + check_size_i64(val, 8)?; + let mut length = 1; + while length <= 8 { + if val >= -(1 << (7 * length - 1)) && val < (1 << (7 * length - 1)) { + break; + } + length += 1; + } + + Ok(as_vint_no_check_i64(val, length)) + } + + /// + /// Returns a representation of the current value as a vint array with a specified length. + /// + /// # Errors + /// + /// This can return an error if the value is outside of the range that can be represented as a vint. + /// + fn as_signed_vint_with_length(&self, length: usize) -> Result, ToolError> { + let val: i64 = (*self).into(); + check_size_i64(val, length)?; + Ok(as_vint_no_check_i64(val, length)) + } +} + +impl SignedVint for i64 {} +impl SignedVint for i32 {} +impl SignedVint for i16 {} +impl SignedVint for i8 {} + +#[inline] +fn check_size_i64(val: i64, max_length: usize) -> Result<(), ToolError> { + if val <= -(1 << (max_length * 7 - 1)) || val >= (1 << (max_length * 7 - 1)) { + Err(ToolError::WriteSignedVintOverflow(val)) + } else { + Ok(()) + } +} + +#[inline] +fn as_vint_no_check_i64(val: i64, length: usize) -> Vec { + let bytes: [u8; 8] = val.to_be_bytes(); + let mut result: Vec = Vec::from(&bytes[(8 - length)..]); + if val < 0 { + result[0] &= 0xFF >> (length - 1); + } else { + result[0] |= 1 << (8 - length); + } + result +} + +/// +/// Reads a signed vint from the beginning of the input array slice. +/// +/// This method returns an option with the `None` variant used to indicate there was not enough data in the buffer to completely read a vint. +/// +/// The returned tuple contains the value of the vint (`i64`) and the length of the vint (`usize`). The length will be less than or equal to the length of the input slice. +/// +/// # Errors +/// +/// This method can return a `ToolError` if the input array cannot be read as a vint. +/// +pub fn read_signed_vint(buffer: &[u8]) -> Result, ToolError> { + if buffer.is_empty() { + return Ok(None); + } + + if buffer[0] == 0 { + return Err(ToolError::ReadVintOverflow); + } + + let length = 8 - buffer[0].ilog2() as usize; + + if length > buffer.len() { + // Not enough data in the buffer to read out the vint value + return Ok(None); + } + + let is_negative = if length == 8 { + buffer[1] & 0x80 + } else { + buffer[0] & (0x80 >> length) + } > 0; + + let mut value = if is_negative { + (buffer[0] as i64) | (!0i64 << (8 - length)) + } else { + (buffer[0] & (0xFF >> length)) as i64 + }; + + for item in buffer.iter().take(length).skip(1) { + value <<= 8; + value += *item as i64; + } + + Ok(Some((value, length))) +} + +/// +/// Reads a `u64` value from any length array slice. +/// +/// Rather than forcing the input to be a `[u8; 8]` like standard library methods, this can interpret a `u64` from a slice of any length < 8. Bytes are assumed to be least significant when reading the value - i.e. an array of `[4, 0]` would return a value of `1024`. +/// +/// # Errors +/// +/// This method will return an error if the input slice has a length > 8. +/// +/// ## Example +/// +/// ``` +/// # use ebml_iterable::tools::arr_to_u64; +/// # fn main() -> Result<(), Box> { +/// let result = arr_to_u64(&[16,0])?; +/// assert_eq!(result, 4096); +/// # Ok(()) +/// # } +/// ``` +/// +pub fn arr_to_u64(arr: &[u8]) -> Result { + if arr.len() > 8 { + return Err(ToolError::ReadU64Overflow(Vec::from(arr))); + } + + let mut val = 0u64; + for byte in arr { + val *= 256; + val += *byte as u64; + } + Ok(val) +} + +/// +/// Reads an `i64` value from any length array slice. +/// +/// Rather than forcing the input to be a `[u8; 8]` like standard library methods, this can interpret an `i64` from a slice of any length < 8. Bytes are assumed to be least significant when reading the value - i.e. an array of `[4, 0]` would return a value of `1024`. +/// +/// # Errors +/// +/// This method will return an error if the input slice has a length > 8. +/// +/// ## Example +/// +/// ``` +/// # use ebml_iterable::tools::arr_to_i64; +/// # fn main() -> Result<(), Box> { +/// let result = arr_to_i64(&[4,0])?; +/// assert_eq!(result, 1024); +/// # Ok(()) +/// # } +/// ``` +/// +pub fn arr_to_i64(arr: &[u8]) -> Result { + if arr.len() > 8 { + return Err(ToolError::ReadI64Overflow(Vec::from(arr))); + } + + if arr[0] > 127 { + if arr.len() == 8 { + Ok(i64::from_be_bytes( + arr.try_into().expect("[u8;8] should be convertible to i64"), + )) + } else { + Ok(-((1 << (arr.len() * 8)) + - (arr_to_u64(arr).expect("arr_to_u64 shouldn't error if length is <= 8") as i64))) + } + } else { + Ok(arr_to_u64(arr).expect("arr_to_u64 shouldn't error if length is <= 8") as i64) + } +} + +/// +/// Reads an `f64` value from an array slice of length 4 or 8. +/// +/// This method wraps `f32` and `f64` conversions from big endian byte arrays and casts the result as an `f64`. +/// +/// # Errors +/// +/// This method will throw an error if the input slice length is not 4 or 8. +/// +pub fn arr_to_f64(arr: &[u8]) -> Result { + if arr.len() == 4 { + Ok(f32::from_be_bytes(arr.try_into().expect("arr should be [u8;4]")) as f64) + } else if arr.len() == 8 { + Ok(f64::from_be_bytes(arr.try_into().expect("arr should be [u8;8]"))) + } else { + Err(ToolError::ReadF64Mismatch(Vec::from(arr))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn read_vint_sixteen() { + let buffer = [144]; + let result = read_vint(&buffer).unwrap().expect("Reading vint failed"); + + assert_eq!(16, result.0); + assert_eq!(1, result.1); + } + + #[test] + fn write_vint_sixteen() { + let result = 16u64.as_vint().expect("Writing vint failed"); + assert_eq!(vec![144u8], result); + } + + #[test] + fn read_vint_one_twenty_seven() { + let buffer = [255u8]; + let result = read_vint(&buffer).unwrap().expect("Reading vint failed"); + + assert_eq!(127, result.0); + assert_eq!(1, result.1); + } + + #[test] + fn write_vint_one_twenty_seven() { + let result = 127u64.as_vint().expect("Writing vint failed"); + assert_eq!(vec![255u8], result); + } + + #[test] + fn read_vint_two_hundred() { + let buffer = [64, 200]; + let result = read_vint(&buffer).unwrap().expect("Reading vint failed"); + + assert_eq!(200, result.0); + assert_eq!(2, result.1); + } + + #[test] + fn write_vint_two_hundred() { + let result = 200u64.as_vint().expect("Writing vint failed"); + assert_eq!(vec![64u8, 200u8], result); + } + + #[test] + fn read_vint_for_ebml_tag() { + let buffer = [0x1a, 0x45, 0xdf, 0xa3]; + let result = read_vint(&buffer).unwrap().expect("Reading vint failed"); + + assert_eq!(0x0a45dfa3, result.0); + assert_eq!(4, result.1); + } + + #[test] + fn read_vint_very_long() { + let buffer = [1, 0, 0, 0, 0, 0, 0, 1]; + let result = read_vint(&buffer).unwrap().expect("Reading vint failed"); + + assert_eq!(1, result.0); + assert_eq!(8, result.1); + } + + #[test] + fn write_vint_very_long() { + let result = 1u64.as_vint_with_length::<8>().expect("Writing vint failed"); + assert_eq!(vec![1, 0, 0, 0, 0, 0, 0, 1], result); + } + + #[test] + fn read_vint_overflow() { + let buffer = [1, 0, 0, 0]; + let result = read_vint(&buffer).expect("Reading vint failed"); + + assert_eq!(true, result.is_none()); + } + + #[test] + #[should_panic] + fn too_big_for_vint() { + (1u64 << 56).as_vint().expect("Writing vint failed"); + } + + #[test] + fn vint_encode_decode_range() { + for val in 0..500_000 { + let bytes = val.as_vint().unwrap(); + let result = read_vint(bytes.as_slice()).unwrap().unwrap().0; + assert_eq!(val, result); + } + } + + #[test] + fn signed_vint_encode_decode_range() { + for val in -500_000..500_000 { + let bytes = val.as_signed_vint().unwrap(); + let result = read_signed_vint(bytes.as_slice()).unwrap().unwrap().0; + assert_eq!(val, result); + } + } + + #[test] + fn read_u64_values() { + let mut buffer = vec![]; + let mut expected = 0; + for _ in 0..8 { + buffer.push(0x25); + expected = (expected << 8) + 0x25; + + let result = arr_to_u64(&buffer).unwrap(); + assert_eq!(expected, result); + } + } + + #[test] + fn read_i64_values() { + let mut buffer = vec![]; + let mut expected = 0; + for _ in 0..8 { + buffer.push(0x0a); + expected = (expected << 8) + 0x0a; + + let result = arr_to_i64(&buffer).unwrap(); + assert_eq!(expected, result); + + let neg_result = arr_to_i64(&(buffer.iter().map(|b| !b).collect::>())).unwrap() + 1; + assert_eq!(-expected, neg_result); + } + } + + #[test] + fn valid_vints() { + assert!(is_vint(0x1F43B675)); + assert!(is_vint(0xA0)); + assert!(is_vint(0xA1)); + assert!(is_vint(0x75A1)); + assert!(is_vint(0xA6)); + assert!(is_vint(0xEE)); + assert!(is_vint(0xA5)); + assert!(is_vint(0x9B)); + assert!(is_vint(0xA2)); + assert!(is_vint(0xA4)); + assert!(is_vint(0x75A2)); + assert!(is_vint(0xFB)); + assert!(is_vint(0xC8)); + assert!(is_vint(0xC9)); + assert!(is_vint(0xCA)); + assert!(is_vint(0xFA)); + assert!(is_vint(0xFD)); + assert!(is_vint(0x8E)); + assert!(is_vint(0xE8)); + assert!(is_vint(0xCB)); + assert!(is_vint(0xCE)); + assert!(is_vint(0xCD)); + assert!(is_vint(0xCC)); + assert!(is_vint(0xCF)); + assert!(is_vint(0xAF)); + assert!(is_vint(0xA7)); + assert!(is_vint(0xAB)); + assert!(is_vint(0x5854)); + assert!(is_vint(0x58D7)); + assert!(is_vint(0xA3)); + assert!(is_vint(0xE7)); + assert!(is_vint(0x3E83BB)); + assert!(is_vint(0x3EB923)); + assert!(is_vint(0x3C83AB)); + assert!(is_vint(0x3CB923)); + + assert!(!is_vint(1234)); + assert!(!is_vint(0x11)); + assert!(!is_vint(0x7a)); + assert!(!is_vint(0xfa4c)); + assert!(!is_vint(0x1a5d)); + } +} diff --git a/crates/ebml-iterable/tests/corrupt_data_tests.rs b/crates/ebml-iterable/tests/corrupt_data_tests.rs new file mode 100644 index 000000000..5b6d6ac82 --- /dev/null +++ b/crates/ebml-iterable/tests/corrupt_data_tests.rs @@ -0,0 +1,239 @@ +mod test_spec; + +pub mod corrupt_data_tests { + use std::io::Cursor; + + use ebml_iterable::error::{CorruptedFileError, TagIteratorError}; + use ebml_iterable::iterator::AllowableErrors; + use ebml_iterable::specs::Master; + use ebml_iterable::{TagIterator, TagWriter, WriteOptions}; + + use super::test_spec::TestSpec; + + fn get_data_with_invalid_ids() -> Cursor> { + let tags: Vec = vec![ + TestSpec::Segment(Master::Start), + TestSpec::TrackType(0x01), + TestSpec::RawTag(0xf2, vec![0x01]), + TestSpec::Segment(Master::End), + ]; + + let mut dest = Cursor::new(Vec::new()); + let mut writer = TagWriter::new(&mut dest); + + for tag in tags.iter() { + writer.write(tag).expect("Test shouldn't error"); + } + + println!("dest {:x?}", dest); + dest.set_position(0); + dest + } + + #[test] + pub fn error_on_invalid_ids() { + let mut cursor = get_data_with_invalid_ids(); + let mut reader: TagIterator<_, TestSpec> = TagIterator::new(&mut cursor, &[]); + assert!(reader.next().unwrap().is_ok()); + assert!(reader.next().unwrap().is_ok()); + assert!(matches!( + reader.next().unwrap(), + Err(TagIteratorError::CorruptedFileData(CorruptedFileError::InvalidTagId { + tag_id: _, + position: _ + })) + )); + } + + #[test] + pub fn allow_errors_invalid_ids() { + let mut cursor = get_data_with_invalid_ids(); + let mut reader: TagIterator<_, TestSpec> = TagIterator::new(&mut cursor, &[]); + reader.allow_errors(&[AllowableErrors::InvalidTagIds]); + reader.for_each(|t| assert!(t.is_ok())); + } + + fn get_data_with_hierarchy_problems() -> Cursor> { + // Test based on this structure. Can't use the writer for this test since it checks hierarchy when writing. + + // let tags: Vec = vec![ + // TestSpec::Segment(Master::Start), + // TestSpec::Count(1), + // TestSpec::Segment(Master::End), + // ]; + + let mut dest = Cursor::new(vec![0x18, 0x53, 0x80, 0x67, 0x84, 0x41, 0x00, 0x81, 0x01]); + + println!("dest {:x?}", dest); + dest.set_position(0); + dest + } + + #[test] + pub fn error_on_hierarchy_problems() { + let mut cursor = get_data_with_hierarchy_problems(); + let mut reader: TagIterator<_, TestSpec> = TagIterator::new(&mut cursor, &[]); + assert!(reader.next().unwrap().is_ok()); + assert!(matches!( + reader.next().unwrap(), + Err(TagIteratorError::CorruptedFileData( + CorruptedFileError::HierarchyError { + found_tag_id: _, + current_parent_id: _ + } + )) + )); + } + + #[test] + pub fn allow_errors_hierarchy_problems() { + let mut cursor = get_data_with_hierarchy_problems(); + let mut reader: TagIterator<_, TestSpec> = TagIterator::new(&mut cursor, &[]); + reader.allow_errors(&[AllowableErrors::HierarchyProblems]); + reader.for_each(|t| assert!(t.is_ok())); + } + + fn get_data_with_oversized_child() -> Cursor> { + let tags: Vec = vec![ + TestSpec::Segment(Master::Start), + TestSpec::Cluster(Master::Start), + TestSpec::Block(vec![0x01, 0x02, 0x03, 0x04]), + TestSpec::Cluster(Master::End), + TestSpec::Segment(Master::End), + ]; + + let mut dest = Cursor::new(Vec::new()); + let mut writer = TagWriter::new(&mut dest); + + for tag in tags.iter() { + writer.write(tag).expect("Test shouldn't error"); + } + + // Extend size of block element without resizing parents + dest.get_mut()[11] = 0x86; + dest.get_mut().push(0x0a); + dest.get_mut().push(0x0a); + + println!("dest {:x?}", dest); + dest.set_position(0); + dest + } + + #[test] + pub fn error_on_oversized_child() { + let mut cursor = get_data_with_oversized_child(); + let mut reader: TagIterator<_, TestSpec> = TagIterator::new(&mut cursor, &[]); + assert!(reader.next().unwrap().is_ok()); + assert!(reader.next().unwrap().is_ok()); + assert!(matches!( + reader.next().unwrap(), + Err(TagIteratorError::CorruptedFileData( + CorruptedFileError::OversizedChildElement { + position: _, + tag_id: _, + size: _ + } + )) + )); + } + + #[test] + pub fn allow_errors_oversized_child() { + let mut cursor = get_data_with_oversized_child(); + let mut reader: TagIterator<_, TestSpec> = TagIterator::new(&mut cursor, &[]); + reader.allow_errors(&[AllowableErrors::OversizedTags]); + reader.for_each(|t| assert!(t.is_ok())); + } + + fn get_data_with_6_byte_tag() -> Cursor> { + let tags: Vec = vec![ + TestSpec::Segment(Master::Start), + TestSpec::Cluster(Master::Start), + TestSpec::Block(vec![0x01, 0x02, 0x03, 0x04, 0x05, 0x06]), + TestSpec::Cluster(Master::End), + TestSpec::Segment(Master::End), + ]; + + let mut dest = Cursor::new(Vec::new()); + let mut writer = TagWriter::new(&mut dest); + + for tag in tags.iter() { + if matches!(tag, TestSpec::Segment(_)) || matches!(tag, TestSpec::Cluster(_)) { + writer + .write_advanced(tag, WriteOptions::is_unknown_sized_element()) + .expect("Test shouldn't error"); + } else { + writer.write(tag).expect("Test shouldn't error"); + } + } + + // // Rewrite size of block element + // dest.get_mut()[25] = 0x09; + // dest.get_mut()[26] = 0x65; + // dest.get_mut()[27] = 0xa0; + // dest.get_mut()[28] = 0xbc; + // dest.get_mut()[29] = 0x00; + + println!("dest {:x?}", dest); + dest.set_position(0); + dest + } + + #[test] + pub fn error_on_oversized_tag() { + let mut cursor = get_data_with_6_byte_tag(); + let mut reader: TagIterator<_, TestSpec> = TagIterator::new(&mut cursor, &[]); + reader.set_max_allowable_tag_size(Some(5)); + assert!(reader.next().unwrap().is_ok()); + assert!(reader.next().unwrap().is_ok()); + assert!(matches!( + reader.next().unwrap(), + Err(TagIteratorError::CorruptedFileData( + CorruptedFileError::InvalidTagSize { + position: _, + tag_id: _, + size: _ + } + )) + )); + } + + #[test] + pub fn recover_on_global_element() { + let tags: Vec = vec![ + TestSpec::Segment(Master::Start), + TestSpec::Cluster(Master::Start), + TestSpec::Crc32(vec![0x01]), + TestSpec::Count(1), + TestSpec::Cluster(Master::End), + TestSpec::Segment(Master::End), + ]; + + let mut dest = Cursor::new(Vec::new()); + let mut writer = TagWriter::new(&mut dest); + + for tag in tags.iter() { + writer.write(tag).expect("Test shouldn't error"); + } + + // Inserting some junk data to skip + dest.get_mut().insert(10, 0x0a); + dest.get_mut().insert(10, 0x0a); + dest.get_mut().insert(10, 0x0a); + dest.set_position(0); + + println!("dest {:x?}", dest); + + let mut reader: TagIterator<_, TestSpec> = TagIterator::new(&mut dest, &[]); + assert!(matches!(reader.next(), Some(t) if t.is_ok())); + assert!(matches!(reader.next(), Some(t) if t.is_ok())); + assert!(matches!(reader.next(), Some(t) if t.is_err())); + assert!(reader.try_recover().is_ok()); + reader.for_each(|t| { + if let Err(err) = t { + println!("{err:?}"); + assert!(false); + } + }); + } +} diff --git a/crates/ebml-iterable/tests/derive_spec_compile.rs b/crates/ebml-iterable/tests/derive_spec_compile.rs new file mode 100644 index 000000000..d1e7006f6 --- /dev/null +++ b/crates/ebml-iterable/tests/derive_spec_compile.rs @@ -0,0 +1,45 @@ +#[cfg(feature = "derive-spec")] +pub mod derive_spec_compile { + use ebml_iterable::specs::{ebml_specification, EbmlSpecification, Master, TagDataType}; + + #[ebml_specification] + #[derive(Clone, Debug, PartialEq)] + pub enum Trial { + #[id(0x01)] + #[data_type(TagDataType::Master)] + Root, + + #[id(0x02)] + #[data_type(TagDataType::Master)] + Parent, + + #[id(0x100)] + #[data_type(TagDataType::UnsignedInt)] + Count, + + #[id(0x200)] + #[data_type(TagDataType::Binary)] + Data, + + #[id(0x201)] + #[data_type(TagDataType::Utf8)] + Name, + + #[id(0x102)] + #[data_type(TagDataType::Float)] + Amount, + + #[id(0x101)] + #[data_type(TagDataType::Integer)] + Id, + } + + #[test] + pub fn compile_worked() { + let data_type = Trial::get_tag_data_type(0x01); + assert_eq!(Some(TagDataType::Master), data_type); + + let tag = Trial::get_master_tag(0x01, Master::Start).unwrap(); + assert_eq!(Trial::Root(Master::Start), tag); + } +} diff --git a/crates/ebml-iterable/tests/derive_spec_compile_with_hierarchy.rs b/crates/ebml-iterable/tests/derive_spec_compile_with_hierarchy.rs new file mode 100644 index 000000000..170f6f6f9 --- /dev/null +++ b/crates/ebml-iterable/tests/derive_spec_compile_with_hierarchy.rs @@ -0,0 +1,51 @@ +#[cfg(feature = "derive-spec")] +pub mod derive_spec_compile { + use ebml_iterable::specs::{ebml_specification, EbmlSpecification, Master, TagDataType}; + + #[ebml_specification] + #[derive(Clone, Debug, PartialEq)] + pub enum Trial { + #[id(0x01)] + #[data_type(TagDataType::Master)] + Root, + + #[id(0x02)] + #[data_type(TagDataType::Master)] + #[doc_path(Root)] + Parent, + + #[id(0x100)] + #[data_type(TagDataType::UnsignedInt)] + #[doc_path(Root/Parent)] + Count, + + #[id(0x200)] + #[data_type(TagDataType::Binary)] + #[doc_path(Root/Parent)] + Data, + + #[id(0x201)] + #[data_type(TagDataType::Utf8)] + #[doc_path(Root/Parent)] + Name, + + #[id(0x102)] + #[data_type(TagDataType::Float)] + #[doc_path(Root/Parent)] + Amount, + + #[id(0x101)] + #[data_type(TagDataType::Integer)] + #[doc_path(Root/Parent)] + Id, + } + + #[test] + pub fn compile_worked() { + let data_type = Trial::get_tag_data_type(0x01); + assert_eq!(Some(TagDataType::Master), data_type); + + let tag = Trial::get_master_tag(0x01, Master::Start).unwrap(); + assert_eq!(Trial::Root(Master::Start), tag); + } +} diff --git a/crates/ebml-iterable/tests/spec_write_read.rs b/crates/ebml-iterable/tests/spec_write_read.rs new file mode 100644 index 000000000..a4d974745 --- /dev/null +++ b/crates/ebml-iterable/tests/spec_write_read.rs @@ -0,0 +1,375 @@ +mod test_spec; + +pub mod spec_write_read { + use std::io::Cursor; + + use ebml_iterable::error::TagIteratorError; + use ebml_iterable::specs::{EbmlTag, Master}; + use ebml_iterable::{TagIterator, TagWriter, WriteOptions}; + + use super::test_spec::TestSpec; + + #[test] + pub fn simple_read_write() { + let tags: Vec = vec![ + TestSpec::Ebml(Master::Start), + TestSpec::Ebml(Master::End), + TestSpec::Segment(Master::Start), + TestSpec::TrackType(0x01), + TestSpec::Segment(Master::End), + ]; + + let mut dest = Cursor::new(Vec::new()); + let mut writer = TagWriter::new(&mut dest); + + for tag in tags.iter() { + writer.write(tag).expect("Test shouldn't error"); + } + + println!("dest {:?}", dest); + + let mut src = Cursor::new(dest.get_ref().to_vec()); + let reader = TagIterator::new(&mut src, &[]); + let read_tags: Vec = reader.into_iter().map(|t| t.unwrap()).collect(); + + println!("tags {:?}", read_tags); + + for i in 0..read_tags.len() { + assert_eq!(tags[i], read_tags[i]); + } + } + + #[test] + pub fn read_write_buffered_tag() { + let tags: Vec = vec![ + TestSpec::Segment(Master::Start), + TestSpec::Cluster(Master::Full(vec![TestSpec::CueRefCluster(0x02)])), + TestSpec::Segment(Master::End), + ]; + + let mut dest = Cursor::new(Vec::new()); + let mut writer = TagWriter::new(&mut dest); + + for tag in tags.iter() { + writer.write(tag).expect("Test shouldn't error"); + } + + println!("dest {:?}", dest); + + let mut src = Cursor::new(dest.get_ref().to_vec()); + let reader = TagIterator::new(&mut src, &[TestSpec::Cluster(Master::Start)]); + let read_tags: Vec = reader.into_iter().map(|t| t.unwrap()).collect(); + + println!("tags {:?}", read_tags); + + for i in 0..read_tags.len() { + assert_eq!(tags[i], read_tags[i]); + } + } + + #[test] + pub fn oversized_tag() { + let mut dest = Cursor::new(Vec::new()); + let mut writer = TagWriter::new(&mut dest); + + writer + .write(&TestSpec::Segment(Master::Start)) + .expect("Error writing tag"); + writer + .write(&TestSpec::Cluster(Master::Start)) + .expect("Error writing tag"); + // Why 0x10001 specifically? This exceeds the default buffer length (0x10000)?! + writer.write_raw(0xa1, &[0x00; 0x10001]).expect("Error writing tag"); + writer.write(&TestSpec::Count(0x00)).expect("Error writing tag"); + writer + .write(&TestSpec::Cluster(Master::End)) + .expect("Error writing tag"); + writer + .write(&TestSpec::Segment(Master::End)) + .expect("Error writing tag"); + drop(writer); + + dest.set_position(0); + let iter = TagIterator::<_, TestSpec>::new(dest, &[]); + + let tags: Vec<_> = iter.into_iter().collect(); + assert_eq!(tags.len(), 4 + 2, "Reading every tag that was written"); + } + + #[test] + pub fn write_unknown_size() { + let mut dest = Cursor::new(Vec::new()); + let mut writer = TagWriter::new(&mut dest); + + writer.write(&TestSpec::Root(Master::Start)).unwrap(); + writer + .write_advanced( + &TestSpec::Parent(Master::Start), + WriteOptions::is_unknown_sized_element(), + ) + .unwrap(); + writer.write(&TestSpec::Child(1)).unwrap(); + writer.write(&TestSpec::Child(2)).unwrap(); + writer.write(&TestSpec::Parent(Master::End)).unwrap(); + writer.write(&TestSpec::Root(Master::End)).unwrap(); + + dest.set_position(0); + + let iter = TagIterator::<_, TestSpec>::new(dest, &[]); + let tags: Vec<_> = iter.into_iter().collect(); + assert_eq!(tags.len(), 6, "Reading every tag that was written"); + } + + #[test] + pub fn buffer_unknown_size() { + let mut dest = Cursor::new(Vec::new()); + let mut writer = TagWriter::new(&mut dest); + + writer.write(&TestSpec::Root(Master::Start)).unwrap(); + writer + .write_advanced( + &TestSpec::Parent(Master::Start), + WriteOptions::is_unknown_sized_element(), + ) + .unwrap(); + writer.write(&TestSpec::Child(1)).unwrap(); + writer.write(&TestSpec::Child(2)).unwrap(); + writer.write(&TestSpec::Parent(Master::End)).unwrap(); + writer.write(&TestSpec::Root(Master::End)).unwrap(); + + dest.set_position(0); + + let iter = TagIterator::<_, TestSpec>::new(dest, &[TestSpec::Parent(Master::Start)]); + let mut tags: Vec<_> = iter.into_iter().collect(); + assert_eq!(tags.len(), 3, "Buffering 'Parent' into full variant"); + + tags.pop(); + let parent = tags.pop().unwrap().unwrap(); + assert!( + matches!(parent.as_master(), Some(Master::Full(c)) if c.len() == 2), + "Did not buffer tag as master with 2 children" + ); + } + + #[test] + pub fn unknown_size_write_read() { + let mut dest = Cursor::new(Vec::new()); + let mut writer = TagWriter::new(&mut dest); + + writer.write(&TestSpec::Root(Master::Start)).unwrap(); + writer + .write_advanced( + &TestSpec::Parent(Master::Start), + WriteOptions::is_unknown_sized_element(), + ) + .unwrap(); + writer.write(&TestSpec::Child(1)).unwrap(); + writer.write(&TestSpec::Child(2)).unwrap(); + writer.write(&TestSpec::Parent(Master::End)).unwrap(); + writer.write(&TestSpec::Int(2)).unwrap(); + writer.write(&TestSpec::Root(Master::End)).unwrap(); + + println!("{dest:x?}"); + dest.set_position(0); + + let mut iter = TagIterator::<_, TestSpec>::new(dest, &[]); + assert!(matches!(iter.next(), Some(Ok(TestSpec::Root(Master::Start))))); + assert!(matches!(iter.next(), Some(Ok(TestSpec::Parent(Master::Start))))); + assert!(matches!(iter.next(), Some(Ok(TestSpec::Child(1))))); + assert!(matches!(iter.next(), Some(Ok(TestSpec::Child(2))))); + assert!(matches!(iter.next(), Some(Ok(TestSpec::Parent(Master::End))))); + assert!(matches!(iter.next(), Some(Ok(TestSpec::Int(2))))); + assert!(matches!(iter.next(), Some(Ok(TestSpec::Root(Master::End))))); + assert!(matches!(iter.next(), None)); + } + + #[test] + pub fn specific_size_length_write_read() { + let mut dest = Cursor::new(Vec::new()); + let mut writer = TagWriter::new(&mut dest); + + writer.write(&TestSpec::Root(Master::Start)).unwrap(); + writer + .write_advanced(&TestSpec::Parent(Master::Start), WriteOptions::set_size_byte_count(8)) + .unwrap(); + writer.write(&TestSpec::Child(1)).unwrap(); + writer.write(&TestSpec::Child(2)).unwrap(); + writer.write(&TestSpec::Parent(Master::End)).unwrap(); + writer.write(&TestSpec::Int(2)).unwrap(); + writer.write(&TestSpec::Root(Master::End)).unwrap(); + + println!("{dest:x?}"); + dest.set_position(0); + + let mut iter = TagIterator::<_, TestSpec>::new(dest, &[]); + assert!(matches!(iter.next(), Some(Ok(TestSpec::Root(Master::Start))))); + assert!(matches!(iter.next(), Some(Ok(TestSpec::Parent(Master::Start))))); + assert!(matches!(iter.next(), Some(Ok(TestSpec::Child(1))))); + assert!(matches!(iter.next(), Some(Ok(TestSpec::Child(2))))); + assert!(matches!(iter.next(), Some(Ok(TestSpec::Parent(Master::End))))); + assert!(matches!(iter.next(), Some(Ok(TestSpec::Int(2))))); + assert!(matches!(iter.next(), Some(Ok(TestSpec::Root(Master::End))))); + assert!(matches!(iter.next(), None)); + } + + #[test] + pub fn eof_error_is_helpful() { + let tags: Vec = vec![ + TestSpec::Segment(Master::Start), + TestSpec::TrackType(0x01), + TestSpec::Cluster(Master::Start), + TestSpec::CueRefCluster(3), + TestSpec::Count(1), + TestSpec::Block(vec![0, 1, 2, 3, 4, 5, 6, 7, 8]), + TestSpec::Cluster(Master::End), + TestSpec::Segment(Master::End), + ]; + + let mut dest = Cursor::new(Vec::new()); + let mut writer = TagWriter::new(&mut dest); + + for tag in tags.iter() { + writer.write(tag).expect("Test shouldn't error"); + } + + println!("dest {:x?}", dest); + + let mut src = Cursor::new(dest.get_ref()[0..26].to_vec()); + let reader = TagIterator::new(&mut src, &[]); + let mut iter = reader + .into_iter() + .skip_while(|x: &Result| x.is_ok()); + + let err = iter.next().expect("Shouldn't have reached end of data"); + + match err.expect_err("Should be an error") { + TagIteratorError::UnexpectedEOF { + tag_start, + tag_id, + tag_size, + partial_data: _, + } => { + assert_eq!(tag_start, 20); + assert_eq!(tag_id, Some(TestSpec::Block(vec![]).get_id())); + assert_eq!(tag_size, Some(9)); + } + other => { + println!("{other:?}"); + assert!(false); + } + } + } + + #[test] + pub fn eof_on_tag_size() { + let tags: Vec = vec![ + TestSpec::Segment(Master::Start), + TestSpec::TrackType(0x01), + TestSpec::Cluster(Master::Start), + TestSpec::CueRefCluster(3), + TestSpec::Count(1), + TestSpec::Block(vec![0, 1, 2, 3, 4, 5, 6, 7]), + TestSpec::Block(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]), + TestSpec::Cluster(Master::End), + TestSpec::Segment(Master::End), + ]; + + let mut dest = Cursor::new(Vec::new()); + let mut writer = TagWriter::new(&mut dest); + + for tag in tags.iter() { + writer.write(tag).expect("Test shouldn't error"); + } + + println!("dest {:x?}", dest); + + let mut src = Cursor::new(dest.get_ref()[0..31].to_vec()); + let reader = TagIterator::with_capacity(&mut src, &[], 10); + let mut iter = reader + .into_iter() + .skip_while(|x: &Result| x.is_ok()); + + let err = iter.next().expect("Shouldn't have reached end of data"); + + match err.expect_err("Should be an error") { + TagIteratorError::UnexpectedEOF { + tag_start, + tag_id, + tag_size, + partial_data: _, + } => { + println!("got error - {tag_start}, {tag_id:?}, {tag_size:?}"); + assert_eq!(tag_start, 30); + assert_eq!(tag_id, Some(TestSpec::Block(vec![]).get_id())); + assert_eq!(tag_size, None); + } + other => { + println!("{other:?}"); + assert!(false); + } + } + } + + #[test] + pub fn allow_start_reading_not_at_root() { + let tags: Vec = vec![ + TestSpec::Segment(Master::Start), + TestSpec::TrackType(0x01), + TestSpec::Cluster(Master::Start), + TestSpec::CueRefCluster(3), + TestSpec::Count(1), + TestSpec::Block(vec![0, 1, 2, 3, 4, 5, 6, 7, 8]), + TestSpec::Cluster(Master::End), + TestSpec::Segment(Master::End), + ]; + + let mut dest = Cursor::new(Vec::new()); + let mut writer = TagWriter::new(&mut dest); + + for tag in tags.iter() { + writer.write(tag).expect("Test shouldn't error"); + } + + println!("dest {:x?}", dest); + + let mut src = Cursor::new(dest.get_ref()[8..].to_vec()); + let reader: TagIterator<_, TestSpec> = TagIterator::new(&mut src, &[]); + reader.for_each(|t| assert!(t.is_ok())); + } + + #[test] + pub fn validate_global_hierarchies() { + let tags: Vec = vec![ + TestSpec::Ebml(Master::Start), + TestSpec::Ebml(Master::End), + TestSpec::Void(vec![0xa0]), + TestSpec::Segment(Master::Start), + TestSpec::Crc32(vec![0x01]), + TestSpec::TrackType(0x01), + TestSpec::Cluster(Master::Start), + TestSpec::Crc32(vec![0x02]), + TestSpec::Count(1), + TestSpec::Cluster(Master::End), + TestSpec::Segment(Master::End), + ]; + + let mut dest = Cursor::new(Vec::new()); + let mut writer = TagWriter::new(&mut dest); + + for tag in tags.iter() { + writer.write(tag).expect("Test shouldn't error"); + } + + println!("dest {:?}", dest); + + let mut src = Cursor::new(dest.get_ref().to_vec()); + let reader = TagIterator::new(&mut src, &[]); + let read_tags: Vec = reader.into_iter().map(|t| t.unwrap()).collect(); + + println!("tags {:?}", read_tags); + + for i in 0..read_tags.len() { + assert_eq!(tags[i], read_tags[i]); + } + } +} diff --git a/crates/ebml-iterable/tests/test_spec.rs b/crates/ebml-iterable/tests/test_spec.rs new file mode 100644 index 000000000..41ac96f7d --- /dev/null +++ b/crates/ebml-iterable/tests/test_spec.rs @@ -0,0 +1,214 @@ +// use ebml_iterable_specification_derive::easy_ebml; +// easy_ebml!( +// pub enum TestSpec { +// Root : Master = 0x81, +// Root/Int : UnsignedInt = 0x4101, +// Root/String : Utf8 = 0x4102, +// Root/Parent : Master = 0x4103, +// Root/Parent/Child: UnsignedInt = 0x210301, + +// Ebml : Master = 0x1a45dfa3, +// Segment : Master = 0x18538067, +// Segment/TrackType : UnsignedInt = 0x83, +// Segment/Cluster : Master = 0x1F43B675, +// Segment/Cluster/CueRefCluster : UnsignedInt = 0x97, +// Segment/Cluster/Count : UnsignedInt = 0x4100, +// Segment/Cluster/Block : Binary = 0xa1, +// Segment/Cluster/SimpleBlock : Binary = 0xa3, +// } +// ) + +use ebml_iterable::specs::TagDataType; + +#[derive(Clone, Debug, PartialEq)] +// Recursive expansion of ebml_specification! macro +// ================================================= + +pub enum TestSpec { + Root(ebml_iterable::specs::Master), + Int(u64), + String(String), + Parent(ebml_iterable::specs::Master), + Child(u64), + Ebml(ebml_iterable::specs::Master), + Segment(ebml_iterable::specs::Master), + TrackType(u64), + Cluster(ebml_iterable::specs::Master), + CueRefCluster(u64), + Count(u64), + Block(::std::vec::Vec), + SimpleBlock(::std::vec::Vec), + Crc32(::std::vec::Vec), + Void(::std::vec::Vec), + RawTag(u64, ::std::vec::Vec), +} +impl ebml_iterable::specs::EbmlSpecification for TestSpec { + fn get_tag_data_type(id: u64) -> Option { + match id { + 129u64 => Some(TagDataType::Master), + 16641u64 => Some(TagDataType::UnsignedInt), + 16642u64 => Some(TagDataType::Utf8), + 16643u64 => Some(TagDataType::Master), + 2163457u64 => Some(TagDataType::UnsignedInt), + 440786851u64 => Some(TagDataType::Master), + 408125543u64 => Some(TagDataType::Master), + 131u64 => Some(TagDataType::UnsignedInt), + 524531317u64 => Some(TagDataType::Master), + 151u64 => Some(TagDataType::UnsignedInt), + 16640u64 => Some(TagDataType::UnsignedInt), + 161u64 => Some(TagDataType::Binary), + 163u64 => Some(TagDataType::Binary), + 191u64 => Some(ebml_iterable::specs::TagDataType::Binary), + 236u64 => Some(ebml_iterable::specs::TagDataType::Binary), + _ => None, + } + } + fn get_path_by_id(id: u64) -> &'static [ebml_iterable::specs::PathPart] { + match id { + 16641u64 => &[ebml_iterable::specs::PathPart::Id(129u64)], + 16642u64 => &[ebml_iterable::specs::PathPart::Id(129u64)], + 16643u64 => &[ebml_iterable::specs::PathPart::Id(129u64)], + 2163457u64 => &[ + ebml_iterable::specs::PathPart::Id(129u64), + ebml_iterable::specs::PathPart::Id(16643u64), + ], + 131u64 => &[ebml_iterable::specs::PathPart::Id(408125543u64)], + 524531317u64 => &[ebml_iterable::specs::PathPart::Id(408125543u64)], + 151u64 => &[ + ebml_iterable::specs::PathPart::Id(408125543u64), + ebml_iterable::specs::PathPart::Id(524531317u64), + ], + 16640u64 => &[ + ebml_iterable::specs::PathPart::Id(408125543u64), + ebml_iterable::specs::PathPart::Id(524531317u64), + ], + 161u64 => &[ + ebml_iterable::specs::PathPart::Id(408125543u64), + ebml_iterable::specs::PathPart::Id(524531317u64), + ], + 163u64 => &[ + ebml_iterable::specs::PathPart::Id(408125543u64), + ebml_iterable::specs::PathPart::Id(524531317u64), + ], + 191u64 => &[ebml_iterable::specs::PathPart::Global((Some(1u64), None))], + 236u64 => &[ebml_iterable::specs::PathPart::Global((None, None))], + _ => &[], + } + } + fn get_unsigned_int_tag(id: u64, data: u64) -> Option { + match id { + 16641u64 => Some(TestSpec::Int(data)), + 2163457u64 => Some(TestSpec::Child(data)), + 131u64 => Some(TestSpec::TrackType(data)), + 151u64 => Some(TestSpec::CueRefCluster(data)), + 16640u64 => Some(TestSpec::Count(data)), + _ => None, + } + } + fn get_signed_int_tag(id: u64, _data: i64) -> Option { + match id { + _ => None, + } + } + fn get_utf8_tag(id: u64, data: String) -> Option { + match id { + 16642u64 => Some(TestSpec::String(data)), + _ => None, + } + } + fn get_binary_tag(id: u64, data: &[u8]) -> Option { + match id { + 161u64 => Some(TestSpec::Block(data.to_vec())), + 163u64 => Some(TestSpec::SimpleBlock(data.to_vec())), + 191u64 => Some(TestSpec::Crc32(data.to_vec())), + 236u64 => Some(TestSpec::Void(data.to_vec())), + _ => None, + } + } + fn get_float_tag(id: u64, _data: f64) -> Option { + match id { + _ => None, + } + } + fn get_master_tag(id: u64, data: ebml_iterable::specs::Master) -> Option { + match id { + 129u64 => Some(TestSpec::Root(data)), + 16643u64 => Some(TestSpec::Parent(data)), + 440786851u64 => Some(TestSpec::Ebml(data)), + 408125543u64 => Some(TestSpec::Segment(data)), + 524531317u64 => Some(TestSpec::Cluster(data)), + _ => None, + } + } + fn get_raw_tag(id: u64, data: &[u8]) -> TestSpec { + TestSpec::RawTag(id, data.to_vec()) + } +} +impl ebml_iterable::specs::EbmlTag for TestSpec { + fn get_id(&self) -> u64 { + match self { + TestSpec::Root(_) => 129u64, + TestSpec::Int(_) => 16641u64, + TestSpec::String(_) => 16642u64, + TestSpec::Parent(_) => 16643u64, + TestSpec::Child(_) => 2163457u64, + TestSpec::Ebml(_) => 440786851u64, + TestSpec::Segment(_) => 408125543u64, + TestSpec::TrackType(_) => 131u64, + TestSpec::Cluster(_) => 524531317u64, + TestSpec::CueRefCluster(_) => 151u64, + TestSpec::Count(_) => 16640u64, + TestSpec::Block(_) => 161u64, + TestSpec::SimpleBlock(_) => 163u64, + TestSpec::Crc32(_) => 191u64, + TestSpec::Void(_) => 236u64, + TestSpec::RawTag(id, _data) => *id, + } + } + fn as_unsigned_int(&self) -> Option<&u64> { + match self { + TestSpec::Int(val) => Some(val), + TestSpec::Child(val) => Some(val), + TestSpec::TrackType(val) => Some(val), + TestSpec::CueRefCluster(val) => Some(val), + TestSpec::Count(val) => Some(val), + _ => None, + } + } + fn as_signed_int(&self) -> Option<&i64> { + match self { + _ => None, + } + } + fn as_utf8(&self) -> Option<&str> { + match self { + TestSpec::String(val) => Some(val), + _ => None, + } + } + fn as_binary(&self) -> Option<&[u8]> { + match self { + TestSpec::Block(val) => Some(val), + TestSpec::SimpleBlock(val) => Some(val), + TestSpec::Crc32(val) => Some(val), + TestSpec::Void(val) => Some(val), + TestSpec::RawTag(_id, data) => Some(data), + _ => None, + } + } + fn as_float(&self) -> Option<&f64> { + match self { + _ => None, + } + } + fn as_master(&self) -> Option<&ebml_iterable::specs::Master> { + match self { + TestSpec::Root(val) => Some(val), + TestSpec::Parent(val) => Some(val), + TestSpec::Ebml(val) => Some(val), + TestSpec::Segment(val) => Some(val), + TestSpec::Cluster(val) => Some(val), + _ => None, + } + } +} From 0fe3a21bacf51f51ac387124fe2a7bb723dee1ee Mon Sep 17 00:00:00 2001 From: Junyi Ou Date: Fri, 14 Aug 2026 13:24:15 -0400 Subject: [PATCH 2/2] build(deps): use tagged ebml-iterable fork --- Cargo.lock | 3 + Cargo.toml | 6 +- crates/ebml-iterable/Cargo.toml | 23 - crates/ebml-iterable/LICENSE | 21 - crates/ebml-iterable/README.md | 89 -- .../specification-derive/Cargo.toml | 18 - .../specification-derive/src/ast.rs | 167 ---- .../specification-derive/src/attr.rs | 488 ----------- .../specification-derive/src/easy_ebml.rs | 124 --- .../specification-derive/src/lib.rs | 143 ---- .../specification-derive/src/pathing.rs | 86 -- crates/ebml-iterable/specification/Cargo.toml | 11 - .../specification/src/empty_spec.rs | 104 --- crates/ebml-iterable/specification/src/lib.rs | 233 ------ crates/ebml-iterable/src/errors.rs | 363 --------- crates/ebml-iterable/src/lib.rs | 58 -- crates/ebml-iterable/src/nonblocking.rs | 59 -- crates/ebml-iterable/src/spec_util.rs | 89 -- crates/ebml-iterable/src/specs.rs | 11 - crates/ebml-iterable/src/tag_decoder.rs | 406 ---------- crates/ebml-iterable/src/tag_iterator.rs | 597 -------------- .../ebml-iterable/src/tag_iterator_async.rs | 191 ----- crates/ebml-iterable/src/tag_iterator_util.rs | 125 --- crates/ebml-iterable/src/tag_parse.rs | 110 --- crates/ebml-iterable/src/tag_writer.rs | 760 ------------------ crates/ebml-iterable/src/tools.rs | 523 ------------ .../ebml-iterable/tests/corrupt_data_tests.rs | 239 ------ .../tests/derive_spec_compile.rs | 45 -- .../derive_spec_compile_with_hierarchy.rs | 51 -- crates/ebml-iterable/tests/spec_write_read.rs | 375 --------- crates/ebml-iterable/tests/test_spec.rs | 214 ----- 31 files changed, 4 insertions(+), 5728 deletions(-) delete mode 100644 crates/ebml-iterable/Cargo.toml delete mode 100644 crates/ebml-iterable/LICENSE delete mode 100644 crates/ebml-iterable/README.md delete mode 100644 crates/ebml-iterable/specification-derive/Cargo.toml delete mode 100644 crates/ebml-iterable/specification-derive/src/ast.rs delete mode 100644 crates/ebml-iterable/specification-derive/src/attr.rs delete mode 100644 crates/ebml-iterable/specification-derive/src/easy_ebml.rs delete mode 100644 crates/ebml-iterable/specification-derive/src/lib.rs delete mode 100644 crates/ebml-iterable/specification-derive/src/pathing.rs delete mode 100644 crates/ebml-iterable/specification/Cargo.toml delete mode 100644 crates/ebml-iterable/specification/src/empty_spec.rs delete mode 100644 crates/ebml-iterable/specification/src/lib.rs delete mode 100644 crates/ebml-iterable/src/errors.rs delete mode 100644 crates/ebml-iterable/src/lib.rs delete mode 100644 crates/ebml-iterable/src/nonblocking.rs delete mode 100644 crates/ebml-iterable/src/spec_util.rs delete mode 100644 crates/ebml-iterable/src/specs.rs delete mode 100644 crates/ebml-iterable/src/tag_decoder.rs delete mode 100644 crates/ebml-iterable/src/tag_iterator.rs delete mode 100644 crates/ebml-iterable/src/tag_iterator_async.rs delete mode 100644 crates/ebml-iterable/src/tag_iterator_util.rs delete mode 100644 crates/ebml-iterable/src/tag_parse.rs delete mode 100644 crates/ebml-iterable/src/tag_writer.rs delete mode 100644 crates/ebml-iterable/src/tools.rs delete mode 100644 crates/ebml-iterable/tests/corrupt_data_tests.rs delete mode 100644 crates/ebml-iterable/tests/derive_spec_compile.rs delete mode 100644 crates/ebml-iterable/tests/derive_spec_compile_with_hierarchy.rs delete mode 100644 crates/ebml-iterable/tests/spec_write_read.rs delete mode 100644 crates/ebml-iterable/tests/test_spec.rs diff --git a/Cargo.lock b/Cargo.lock index 477034ee5..bf5df632a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2198,6 +2198,7 @@ dependencies = [ [[package]] name = "ebml-iterable" version = "0.6.3" +source = "git+https://github.com/irvingoujAtDevolution/ebml-iterable?tag=v0.6.3-devo1#f67a18e0cf3cf99c7198c4636493d397776f8025" dependencies = [ "bytes 1.12.1", "ebml-iterable-specification", @@ -2208,10 +2209,12 @@ dependencies = [ [[package]] name = "ebml-iterable-specification" version = "0.4.0" +source = "git+https://github.com/irvingoujAtDevolution/ebml-iterable?tag=v0.6.3-devo1#f67a18e0cf3cf99c7198c4636493d397776f8025" [[package]] name = "ebml-iterable-specification-derive" version = "0.4.0" +source = "git+https://github.com/irvingoujAtDevolution/ebml-iterable?tag=v0.6.3-devo1#f67a18e0cf3cf99c7198c4636493d397776f8025" dependencies = [ "ebml-iterable-specification", "proc-macro2 1.0.106", diff --git a/Cargo.toml b/Cargo.toml index 5a0751b03..2cc707e17 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,10 +10,6 @@ members = [ "testsuite", "tools/generate-openapi", ] -exclude = [ - "crates/ebml-iterable/specification", - "crates/ebml-iterable/specification-derive", -] default-members = [ "devolutions-agent", "devolutions-agent-updater", @@ -36,7 +32,7 @@ codegen-units = 1 lto = true [patch.crates-io] -ebml-iterable = { path = "crates/ebml-iterable" } +ebml-iterable = { git = "https://github.com/irvingoujAtDevolution/ebml-iterable", tag = "v0.6.3-devo1" } tracing-appender = { git = "https://github.com/CBenoit/tracing.git", rev = "42097daf92e683cf18da7639ddccb056721a796c" } [workspace.lints.rust] diff --git a/crates/ebml-iterable/Cargo.toml b/crates/ebml-iterable/Cargo.toml deleted file mode 100644 index 1fe3e469f..000000000 --- a/crates/ebml-iterable/Cargo.toml +++ /dev/null @@ -1,23 +0,0 @@ -[package] -name = "ebml-iterable" -version = "0.6.3" -authors = ["Austin Blake "] -edition = "2018" -description = "This crate provides an iterator over EBML encoded data. The items provided by the iterator are Tags as defined in EBML. The iterator is spec-agnostic and requires a specification implementing specific traits to read files. Typically, you would only use this crate to implement a custom specification - most often you would prefer a crate providing an existing specification, like `webm-iterable`." -readme = "README.md" -license = "MIT" -keywords = ["ebml", "encoding", "parser", "serialization"] -categories = ["parsing", "data-structures", "encoding", "multimedia::encoding"] -homepage = "https://github.com/austinleroy/ebml-iterable" -repository = "https://github.com/austinleroy/ebml-iterable" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -bytes = "1" -ebml-iterable-specification = { version = "=0.4.0", path = "specification" } -ebml-iterable-specification-derive = { version = "=0.4.0", path = "specification-derive", optional = true } -futures = { version = "0.3.28", optional = true } - -[features] -derive-spec = ["ebml-iterable-specification-derive"] diff --git a/crates/ebml-iterable/LICENSE b/crates/ebml-iterable/LICENSE deleted file mode 100644 index 4e97feb24..000000000 --- a/crates/ebml-iterable/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2021 Austin Blake - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/crates/ebml-iterable/README.md b/crates/ebml-iterable/README.md deleted file mode 100644 index 2b78e03a8..000000000 --- a/crates/ebml-iterable/README.md +++ /dev/null @@ -1,89 +0,0 @@ -[EBML][EBML] stands for Extensible Binary Meta-Language and is somewhat of a -binary version of XML. It's used for container formats like [WebM][webm] or -[MKV][mkv]. - -# Devolutions fork - -Devolutions Gateway vendors this crate from Austin Blake's [`ebml-iterable`](https://github.com/austinleroy/ebml-iterable) commit `25303fa5a9107d8d28fa8458b0b982946c319135`. -The original work and this fork use the MIT license in [`LICENSE`](LICENSE). -This fork adds `TagDecoder`, which keeps incomplete EBML elements pending while a recording grows. - -> IMPORTANT: The iterator contained in this crate is spec-agnostic and requires a specification implementing the `EbmlSpecification` and `EbmlTag` traits to read files. Typically, you would only use this crate to implement a custom specification - most often you would prefer a crate providing an existing specification, like [webm-iterable][webm-iterable]. - -```Cargo.toml -[dependencies] -ebml-iterable = "0.6.3" -``` - -# Usage - -The `TagIterator` struct implements Rust's standard [Iterator][rust-iterator] trait. -This struct can be created with the `new` function on any source that implements the standard [Read][rust-read] trait. The iterator outputs `TSpec` objects based on the defined specification and the tag data. - -> Note: The `with_capacity` method can be used to construct a `TagIterator` with a specified default buffer size. This is only useful as a microoptimization to memory management if you know the maximum tag size of the file you're reading. - -The data in the tag can then be modified as desired (encryption, compression, etc.) and reencoded using the `TagWriter` struct. This struct can be created with the `new` function on any source that implements the standard [Write][rust-write] trait. Once created, this struct can encode EBML using the `write` method on any objects that implement `EbmlSpecification` and `EbmlTag` regardless of whether they came from a `TagIterator`. This will emit binary EBML to the underlying `Write` destination. - -## Master Enum - -Most tag types contain their data directly, but there is a category of tag in EBML called `Master` which contains other tags. This crate contains an enumeration of three different classifications of master tags: - - * `Start` is a marker for the beginning of a "master" tag. - * `End` is a marker for the end of a "master" tag. - * `Full(children)` is a complete tag that includes all child tags of the `Master` tag. This is only emitted by the `TagIterator` for tag types passed in via `tags_to_buffer`. - -## TagDataType - -```rs -pub enum TagDataType { - Master, - UnsignedInt, - Integer, - Utf8, - Binary, - Float, -} -``` - -TagDataType is an enum containing the possible data types stored within a tag. The relationship between the tag variant and the type of data contained in the tag directly corresponds is defined by whichever specification is in use. Because EBML is binary, the correct specification is required to parse tag content. - - * Master: A complete master tag containing any number of child tags. - * UnsignedInt: An unsigned integer. - * Integer: A signed integer. - * Utf8: A Unicode text string. Note that the [EBML spec][rfc8794] includes a separate element type for ASCII. Given that ASCII is a subset of Utf8, this library currently parses and encodes both types using the same Utf8 logic. - * Binary: Binary data, otherwise uninterpreted. - * Float: IEEE-754 floating point number. - -> Note: This library made a conscious decision to not parse "Date" elements from EBML due to lack of built-in support for dates in Rust. Specification implementations should treat Date elements as Binary so that consumers have the option of parsing the unaltered data using their library of choice, if needed. - -# Specification Implementation - -Any specification based on EBML can use this library to parse or write binary data. Writing needs nothing special (if you use the `write_raw()` method), but parsing requires a struct implementing the `EbmlSpecification` and `EbmlTag` traits. These traits currently have a large number of methods to implement and need consistent implementations to avoid errors, so any implementation attempt is recommended to use the `"derive-spec"` feature flag in this crate and using the provided macro. Custom specification implementations can refer to [webm-iterable][webm-iterable] as an example. - -# Features - -There is currently only one optional feature in this crate, but that may change over time as needs arise. - -* **derive-spec** - - When enabled, this provides a macro to simplify implementations of the `EbmlSpecification` and `EbmlTag` traits. This introduces dependencies on [`syn`](https://crates.io/crates/syn), [`quote`](https://crates.io/crates/quote), and [`proc-macro2`](https://crates.io/crates/proc-macro2), so expect compile times to increase a little. - - -# State of this project - -Parsing and writing complete files should both work. Streaming (using tags of unknown size) should now also be supported, as of version 0.4.0. If something is broken, please create [an issue][new-issue]. - -Any additional feature requests can also be submitted as [an issue][new-issue]. - -# Author - -[Austin Blake](https://github.com/austinleroy) - -[EBML]: http://ebml.sourceforge.net/ -[webm]: https://www.webmproject.org/ -[mkv]: http://www.matroska.org/technical/specs/index.html -[rfc8794]: https://datatracker.ietf.org/doc/rfc8794/ -[rust-iterator]: https://doc.rust-lang.org/std/iter/trait.Iterator.html -[rust-read]: https://doc.rust-lang.org/std/io/trait.Read.html -[rust-write]: https://doc.rust-lang.org/std/io/trait.Write.html -[new-issue]: https://github.com/austinleroy/ebml-iterable/issues -[webm-iterable]: https://github.com/austinleroy/webm-iterable diff --git a/crates/ebml-iterable/specification-derive/Cargo.toml b/crates/ebml-iterable/specification-derive/Cargo.toml deleted file mode 100644 index d2a84389b..000000000 --- a/crates/ebml-iterable/specification-derive/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -name = "ebml-iterable-specification-derive" -version = "0.4.0" -authors = ["Austin Blake "] -edition = "2018" -description = "Provides macros for implementing `EbmlSpecification` for the `ebml-iterable` crate." -license = "MIT" -homepage = "https://github.com/austinleroy/ebml-iterable" -repository = "https://github.com/austinleroy/ebml-iterable" - -[lib] -proc-macro = true - -[dependencies] -proc-macro2 = "1.0" -quote = "1.0" -syn = { version = "1.0", features = ["full"] } -ebml-iterable-specification = { version = "=0.4.0", path = "../specification" } \ No newline at end of file diff --git a/crates/ebml-iterable/specification-derive/src/ast.rs b/crates/ebml-iterable/specification-derive/src/ast.rs deleted file mode 100644 index f05af7b31..000000000 --- a/crates/ebml-iterable/specification-derive/src/ast.rs +++ /dev/null @@ -1,167 +0,0 @@ -use std::collections::HashSet; - -use ebml_iterable_specification::TagDataType; -use quote::ToTokens; -use syn::spanned::Spanned; -use syn::{Error, Generics, Ident, ItemEnum, LitInt, Path, Result}; - -use crate::pathing::{EBMLPath, PathPart}; - -pub struct Enum<'a> { - pub original: &'a ItemEnum, - pub ident: Ident, - pub variants: Vec>, - pub generics: &'a Generics, -} - -pub struct Variant<'a> { - pub original: &'a syn::Variant, - pub ident: Ident, - pub id_attr: (u64, Attribute<'a>), - pub data_type_attr: (TagDataType, Path, Attribute<'a>), - pub path_attr: Option<(EBMLPath, Attribute<'a>)>, -} - -pub struct Attribute<'a> { - pub original: &'a syn::Attribute, -} - -impl<'a> Enum<'a> { - pub fn from_syn(node: &'a ItemEnum) -> Result { - let variant_names: HashSet<_> = node.variants.iter().map(|var| var.ident.clone()).collect(); - let variants = node - .variants - .iter() - .map(|node| Variant::from_syn(node, &variant_names)) - .collect::>()?; - - Ok(Enum { - original: node, - ident: node.ident.clone(), - variants, - generics: &node.generics, - }) - } -} - -impl<'a> Variant<'a> { - fn from_syn(node: &'a syn::Variant, variant_names: &HashSet) -> Result { - let mut id_attr: Option<(u64, Attribute<'a>)> = None; - let mut data_type_attr: Option<(TagDataType, Path, Attribute<'a>)> = None; - let mut path_attr: Option<(EBMLPath, Attribute<'a>)> = None; - - for attr in &node.attrs { - if attr.path.is_ident("id") { - if id_attr.is_some() { - return Err(Error::new_spanned( - node, - format!("duplicate {} attribute", attr.to_token_stream()), - )); - } - let val = attr.parse_args::()?.base10_parse::()?; - id_attr = Some((val, Attribute { original: attr })); - } else if attr.path.is_ident("data_type") { - if data_type_attr.is_some() { - return Err(Error::new_spanned( - node, - format!("duplicate {} attribute", attr.to_token_stream()), - )); - } - - let val = attr.parse_args::().map_err(|err| { - Error::new( - err.span(), - format!("{} requires `ebml_iterable::TagDataType`", attr.to_token_stream()), - ) - })?; - let data_type_name = val.segments.iter().last(); - if data_type_name.is_none() { - return Err(Error::new_spanned( - val, - format!("{} requires `ebml_iterable::TagDataType`", attr.to_token_stream()), - )); - } - let data_type_name = data_type_name.unwrap().ident.to_string(); - let data_type_val = if data_type_name == "UnsignedInt" { - TagDataType::UnsignedInt - } else if data_type_name == "Integer" { - TagDataType::Integer - } else if data_type_name == "Utf8" { - TagDataType::Utf8 - } else if data_type_name == "Binary" { - TagDataType::Binary - } else if data_type_name == "Float" { - TagDataType::Float - } else if data_type_name == "Master" { - TagDataType::Master - } else { - return Err(Error::new_spanned( - val, - format!("unrecognized `ebml_iterable::TagDataType` value: {data_type_name}"), - )); - }; - data_type_attr = Some((data_type_val, val, Attribute { original: attr })); - } else if attr.path.is_ident("doc_path") { - if path_attr.is_some() { - return Err(Error::new_spanned( - node, - format!("duplicate {} attribute", attr.to_token_stream()), - )); - } - let path = attr.parse_args::().map_err(|err| { - Error::new(err.span(), format!("{} must be a path string", attr.to_token_stream())) - })?; - let mut last_was_global = false; - for path_part in &path.parts { - match path_part { - PathPart::Ident(id) => { - last_was_global = false; - variant_names - .get(id) - .ok_or(Error::new(id.span(), format!("Unknown variant [{id}] in path")))?; - } - PathPart::Global((_, max)) => { - if matches!(max, Some(0)) { - return Err(Error::new(attr.span(), "Global maximum cannot be 0".to_string())); - } - if last_was_global { - return Err(Error::new( - attr.span(), - "Cannot use GlobalPlaceholders back-to-back in path".to_string(), - )); - } - last_was_global = true; - } - } - } - path_attr = Some((path, Attribute { original: attr })) - } - } - - let id_attr = if let Some(id_attr) = id_attr { - id_attr - } else { - return Err(Error::new_spanned( - node, - "#[id] attribute is required when using #[ebml_specification] attribute", - )); - }; - - let data_type_attr = if let Some(data_type_attr) = data_type_attr { - data_type_attr - } else { - return Err(Error::new_spanned( - node, - "#[data_type] attribute is required when using #[ebml_specification] attribute", - )); - }; - - Ok(Variant { - original: node, - ident: node.ident.clone(), - id_attr, - data_type_attr, - path_attr, - }) - } -} diff --git a/crates/ebml-iterable/specification-derive/src/attr.rs b/crates/ebml-iterable/specification-derive/src/attr.rs deleted file mode 100644 index 41f658386..000000000 --- a/crates/ebml-iterable/specification-derive/src/attr.rs +++ /dev/null @@ -1,488 +0,0 @@ -use std::collections::HashMap; -use std::str::FromStr; - -use ebml_iterable_specification::TagDataType; -use ebml_iterable_specification::TagDataType::Master; -use proc_macro2::TokenStream; -use quote::{quote, quote_spanned, ToTokens}; -use syn::spanned::Spanned; -use syn::{Attribute, Error, Fields, FieldsUnnamed, Ident, ItemEnum, Path, Result, Variant, Visibility}; - -use super::ast::Enum; -use super::pathing::PathPart; - -pub fn impl_ebml_specification(original: &mut ItemEnum) -> Result { - let tag_data_type = spanned_tag_data_type(original); - original.variants.push(syn::parse2::(quote! { - #[id(0xbf)] - #[data_type(#tag_data_type::Binary)] - #[doc_path((1-))] - Crc32 - })?); - original.variants.push(syn::parse2::(quote! { - #[id(0xec)] - #[data_type(#tag_data_type::Binary)] - #[doc_path((-))] - Void - })?); - - let input = Enum::from_syn(original)?; - - let mut used_ids = HashMap::::new(); - for var in &input.variants { - if let Some(original) = used_ids.insert(var.id_attr.0, var.original) { - let mut err = Error::new_spanned( - var.original, - format!("duplicate {} detected", var.id_attr.1.original.to_token_stream()), - ); - err.combine(Error::new_spanned( - original, - format!("{} already used previously", var.id_attr.1.original.to_token_stream()), - )); - return Err(err); - } - } - - let map: HashMap<_, _> = input.variants.iter().map(|var| (&var.ident, var)).collect(); - for origin in &input.variants { - if !matches!(origin.data_type_attr.0, TagDataType::Master) && origin.path_attr.is_some() { - validate_path(origin, &map)?; - } - } - - let ebml_specification_impl = get_impl(input)?; - let modified_orig = modify_orig(original)?; - - Ok(quote!( - #modified_orig - - #ebml_specification_impl - )) -} - -// verify all parents are Master type elements and their path lines up with this item's path -fn validate_path(origin: &crate::ast::Variant, variants_map: &HashMap<&Ident, &crate::ast::Variant>) -> Result<()> { - // Only validate the element if it has a path attribute - if let Some(path_parts) = origin.path_attr.as_ref().map(|(path, _)| &path.parts) { - // Only validate if there is a specific parent element - if let Some(parent) = path_parts - .iter() - .rev() - .filter_map(|p| { - if let PathPart::Ident(ident) = p { - Some(ident) - } else { - None - } - }) - .next() - { - let parent = *variants_map.get(parent).unwrap(); - if parent.data_type_attr.0 != Master { - return Err(Error::new_spanned(parent.original, "Parents must be of Master type")); - } - - if let Some((parent_path, _)) = parent.path_attr.as_ref() { - for i in 0..parent_path.parts.len() { - if parent_path.parts[i] != path_parts[i] { - return Err(Error::new_spanned( - origin.original, - format!( - "Path segment [{}] did not align with parent [{}] path.", - path_parts[i], parent.ident - ), - )); - } - } - validate_path(parent, variants_map)?; - } - } - } - - Ok(()) -} - -fn modify_orig(original: &mut ItemEnum) -> Result { - let spanned_master_enum = spanned_master_enum(original); - for var in original.variants.iter_mut() { - let data_type_attribute: &Attribute = var - .attrs - .iter() - .find(|a| a.path.is_ident("data_type")) - .expect("#[data_type()] attribute required for variants under #[ebml_specification]"); - - let data_type_path = data_type_attribute.parse_args::().map_err(|err| { - Error::new( - err.span(), - format!( - "{} requires `ebml_iterable::TagDataType`", - data_type_attribute.to_token_stream() - ), - ) - })?; - let data_type = get_last_path_ident(&data_type_path).ok_or_else(|| { - Error::new_spanned( - data_type_attribute.clone(), - format!( - "{} requires `ebml_iterable::TagDataType`", - data_type_attribute.to_token_stream() - ), - ) - })?; - - let data_type = if data_type == "Master" { - let orig_ident = &original.ident; - quote!( (#spanned_master_enum<#orig_ident>) ) - } else if data_type == "UnsignedInt" { - quote!((u64)) - } else if data_type == "Integer" { - quote!((i64)) - } else if data_type == "Utf8" { - quote!((String)) - } else if data_type == "Binary" { - quote!((::std::vec::Vec)) - } else if data_type == "Float" { - quote!((f64)) - } else { - return Err(Error::new_spanned( - data_type_attribute.clone(), - format!("unknown data_type \"{data_type}\""), - )); - }; - - var.attrs - .retain(|a| !(a.path.is_ident("id") || a.path.is_ident("data_type") || a.path.is_ident("doc_path"))); - var.fields = Fields::Unnamed(syn::parse2::(data_type)?); - } - original - .variants - .push(syn::parse_str::("RawTag(u64, ::std::vec::Vec)")?); - - Ok(quote!(#original)) -} - -fn get_impl(input: Enum) -> Result { - let ty = &input.ident; - let spanned_master_enum = spanned_master_enum(input.original); - - let get_tag_data_type = input.variants.iter().map(|var: &crate::ast::Variant| { - let id = &var.id_attr.0; - let data_type = &var.data_type_attr.1; - - quote_spanned! { var.data_type_attr.2.original.span() => - #id => Some(#data_type), - } - }); - - let get_id = input.variants.iter().map(|var: &crate::ast::Variant| { - let name = &var.ident; - let id = &var.id_attr.0; - - quote_spanned! { var.id_attr.1.original.span() => - #ty::#name(_) => #id, - } - }); - - let get_tag = |ret_val: String| { - move |var: &crate::ast::Variant| { - let name = &var.ident; - let id = &var.id_attr.0; - let ret_val = TokenStream::from_str(&ret_val) - .expect("Misuse of get_tag function in ebml_iterable_specification_derive_attr"); - - quote_spanned! { var.original.span() => - #id => Some(#ty::#name(#ret_val)), - } - } - }; - - let path_part = spanned_path_part(input.original); - let variant_map: HashMap<_, _> = input.variants.iter().map(|var| (&var.ident, var)).collect(); - let get_path_by_id = input.variants.iter().filter_map(|v| match v.path_attr.as_ref() { - None => None, - Some(path) => { - let id = &v.id_attr.0; - let path_array: Vec = path - .0 - .parts - .iter() - .map(|p| match p { - PathPart::Ident(ident) => { - let id = variant_map.get(&ident).map(|v| v.id_attr.0).unwrap(); - quote_spanned! { path.1.original.span() => #path_part::Id(#id) } - } - PathPart::Global((min, max)) => { - let min_tokens = if let Some(min) = min { - quote! {Some(#min)} - } else { - quote! {None} - }; - let max_tokens = if let Some(max) = max { - quote! {Some(#max)} - } else { - quote! {None} - }; - quote_spanned! { path.1.original.span() => #path_part::Global((#min_tokens, #max_tokens)) } - } - }) - .collect(); - Some(quote_spanned! { v.original.span() => - #id => &[#(#path_array),*], - }) - } - }); - - let get_unsigned_int_tag = input - .variants - .iter() - .filter(|v| matches!(&v.data_type_attr.0, TagDataType::UnsignedInt)) - .map(get_tag(String::from("data"))); - - let get_signed_int_tag = input - .variants - .iter() - .filter(|v| matches!(&v.data_type_attr.0, TagDataType::Integer)) - .map(get_tag(String::from("data"))); - - let get_utf8_tag = input - .variants - .iter() - .filter(|v| matches!(&v.data_type_attr.0, TagDataType::Utf8)) - .map(get_tag(String::from("data"))); - - let get_binary_tag = input - .variants - .iter() - .filter(|v| matches!(&v.data_type_attr.0, TagDataType::Binary)) - .map(get_tag(String::from("data.to_vec()"))); - - let get_float_tag = input - .variants - .iter() - .filter(|v| matches!(&v.data_type_attr.0, TagDataType::Float)) - .map(get_tag(String::from("data"))); - - let get_master_tag = input - .variants - .iter() - .filter(|v| matches!(&v.data_type_attr.0, TagDataType::Master)) - .map(get_tag(String::from("data"))); - - let as_data = |var: &crate::ast::Variant| { - let name = &var.ident; - - quote! { - #ty::#name(val) => Some(val), - } - }; - - let as_unsigned_int = input - .variants - .iter() - .filter(|v| matches!(&v.data_type_attr.0, TagDataType::UnsignedInt)) - .map(as_data); - - let as_signed_int = input - .variants - .iter() - .filter(|v| matches!(&v.data_type_attr.0, TagDataType::Integer)) - .map(as_data); - - let as_utf8 = input - .variants - .iter() - .filter(|v| matches!(&v.data_type_attr.0, TagDataType::Utf8)) - .map(as_data); - - let as_binary = input - .variants - .iter() - .filter(|v| matches!(&v.data_type_attr.0, TagDataType::Binary)) - .map(as_data); - - let as_float = input - .variants - .iter() - .filter(|v| matches!(&v.data_type_attr.0, TagDataType::Float)) - .map(as_data); - - let as_master = input - .variants - .iter() - .filter(|v| matches!(&v.data_type_attr.0, TagDataType::Master)) - .map(as_data); - - let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl(); - let ebml_spec_trait = spanned_ebml_specification_trait(input.original); - let ebml_tag_trait = spanned_ebml_tag_trait(input.original); - let tag_data_type = spanned_tag_data_type(input.original); - - Ok(quote! { - impl #impl_generics #ebml_spec_trait <#ty> for #ty #ty_generics #where_clause { - fn get_tag_data_type(id: u64) -> Option<#tag_data_type> { - match id { - #(#get_tag_data_type)* - _ => None - } - } - - fn get_path_by_id(id: u64) -> &'static [#path_part] { - match id { - #(#get_path_by_id)* - _ => &[] - } - } - - fn get_unsigned_int_tag(id: u64, data: u64) -> Option<#ty> { - match id { - #(#get_unsigned_int_tag)* - _ => None - } - } - - fn get_signed_int_tag(id: u64, data: i64) -> Option<#ty> { - match id { - #(#get_signed_int_tag)* - _ => None - } - } - - fn get_utf8_tag(id: u64, data: String) -> Option<#ty> { - match id { - #(#get_utf8_tag)* - _ => None - } - } - - fn get_binary_tag(id: u64, data: &[u8]) -> Option<#ty> { - match id { - #(#get_binary_tag)* - _ => None - } - } - - fn get_float_tag(id: u64, data: f64) -> Option<#ty> { - match id { - #(#get_float_tag)* - _ => None - } - } - - fn get_master_tag(id: u64, data: #spanned_master_enum<#ty>) -> Option<#ty> { - match id { - #(#get_master_tag)* - _ => None - } - } - - fn get_raw_tag(id: u64, data: &[u8]) -> #ty { - #ty::RawTag(id, data.to_vec()) - } - } - - impl #impl_generics #ebml_tag_trait <#ty> for #ty #ty_generics #where_clause { - - fn get_id(&self) -> u64 { - match self { - #(#get_id)* - #ty::RawTag(id, _data) => *id, - } - } - - fn as_unsigned_int(&self) -> Option<&u64> { - match self { - #(#as_unsigned_int)* - _ => None, - } - } - - fn as_signed_int(&self) -> Option<&i64> { - match self { - #(#as_signed_int)* - _ => None, - } - } - - fn as_utf8(&self) -> Option<&str> { - match self { - #(#as_utf8)* - _ => None, - } - } - - fn as_binary(&self) -> Option<&[u8]> { - match self { - #(#as_binary)* - #ty::RawTag(_id, data) => Some(data), - _ => None, - } - } - - fn as_float(&self) -> Option<&f64> { - match self { - #(#as_float)* - _ => None, - } - } - - fn as_master(&self) -> Option<&#spanned_master_enum<#ty>> { - match self { - #(#as_master)* - _ => None, - } - } - } - }) -} - -fn spanned_ebml_iterable_specs(input: &ItemEnum) -> TokenStream { - let vis_span = match &input.vis { - Visibility::Public(vis) => Some(vis.pub_token.span()), - Visibility::Crate(vis) => Some(vis.crate_token.span()), - Visibility::Restricted(vis) => Some(vis.pub_token.span()), - Visibility::Inherited => None, - }; - let data_span = input.enum_token.span(); - let first_span = vis_span.unwrap_or(data_span); - quote_spanned!(first_span=> ebml_iterable::specs::) -} - -fn spanned_master_enum(input: &ItemEnum) -> TokenStream { - let path = spanned_ebml_iterable_specs(input); - let last_span = input.ident.span(); - let r#enum = quote_spanned!(last_span=> Master); - quote!(#path #r#enum) -} - -fn spanned_ebml_specification_trait(input: &ItemEnum) -> TokenStream { - let path = spanned_ebml_iterable_specs(input); - let last_span = input.ident.span(); - let spec = quote_spanned!(last_span=> EbmlSpecification); - quote!(#path #spec) -} - -fn spanned_ebml_tag_trait(input: &ItemEnum) -> TokenStream { - let path = spanned_ebml_iterable_specs(input); - let last_span = input.ident.span(); - let spec = quote_spanned!(last_span=> EbmlTag); - quote!(#path #spec) -} - -fn spanned_tag_data_type(input: &ItemEnum) -> TokenStream { - let path = spanned_ebml_iterable_specs(input); - let last_span = input.ident.span(); - let r#type = quote_spanned!(last_span=> TagDataType); - quote!(#path #r#type) -} - -fn spanned_path_part(input: &ItemEnum) -> TokenStream { - let path = spanned_ebml_iterable_specs(input); - let last_span = input.ident.span(); - let r#type = quote_spanned!(last_span=> PathPart); - quote!(#path #r#type) -} - -fn get_last_path_ident(path: &Path) -> Option<&Ident> { - let seg = path.segments.iter().last(); - seg.map(|seg| &seg.ident) -} diff --git a/crates/ebml-iterable/specification-derive/src/easy_ebml.rs b/crates/ebml-iterable/specification-derive/src/easy_ebml.rs deleted file mode 100644 index 6bb9159bf..000000000 --- a/crates/ebml-iterable/specification-derive/src/easy_ebml.rs +++ /dev/null @@ -1,124 +0,0 @@ -use proc_macro2::TokenStream; -use quote::quote; -use syn::parse::{Parse, ParseBuffer, ParseStream}; -use syn::punctuated::Punctuated; -use syn::{AttrStyle, Attribute, Error, Ident, LitInt, Result, Token, Variant, Visibility}; - -use crate::pathing::{EBMLPath, PathPart}; - -pub struct EasyEBML { - attrs: Vec, - visibility: Visibility, - ident: Ident, - variants: Punctuated, -} - -impl Parse for EasyEBML { - fn parse(input: ParseStream) -> Result { - let attrs = input.call(Attribute::parse_outer)?; - let visibility: Visibility = input.parse()?; - input.parse::()?; - let ident = input.parse::()?; - let content: ParseBuffer; - syn::braced!(content in input); - let variants = content.parse_terminated(EasyEBMLVariant::parse)?; - Ok(Self { - attrs, - visibility, - ident, - variants, - }) - } -} - -impl EasyEBML { - pub fn implement(self) -> Result { - let EasyEBML { - attrs, - visibility, - ident, - variants, - } = self; - - let variants: Vec<_> = variants - .into_iter() - .map(EasyEBMLVariant::into_variant) - .collect::>()?; - - Ok(quote!( - #[ebml_iterable::specs::ebml_specification] - #(#attrs)* - #visibility enum #ident { - #(#variants),* - } - )) - } -} - -pub struct EasyEBMLVariant { - path: EBMLPath, - ty: Ident, - id: LitInt, -} - -impl EasyEBMLVariant { - pub fn into_variant(self) -> Result { - let EasyEBMLVariant { path, ty, id } = self; - let span = path.span; - let mut path: Vec = path.parts.into_iter().collect(); - let ident: Ident = match path - .pop() - .ok_or_else(|| Error::new(span, "easy_ebml enum variant must be at least: `Name: Type = id`"))? - { - PathPart::Ident(id) => Ok(id), - PathPart::Global(_) => Err(Error::new(span, "easy_ebml enum variant cannot end in global path")), - }?; - let mut attrs = vec![]; - attrs.push(Attribute { - pound_token: Default::default(), - style: AttrStyle::Outer, - bracket_token: Default::default(), - path: Ident::new("id", proc_macro2::Span::call_site()).into(), - tokens: quote!((#id)), - }); - attrs.push(Attribute { - pound_token: Default::default(), - style: AttrStyle::Outer, - bracket_token: Default::default(), - path: Ident::new("data_type", proc_macro2::Span::call_site()).into(), - tokens: quote!((TagDataType::#ty)), - }); - - if !path.is_empty() { - let mut tokens: Punctuated = Punctuated::new(); - for part in path { - tokens.push(part); - } - attrs.push(Attribute { - pound_token: Default::default(), - style: AttrStyle::Outer, - bracket_token: Default::default(), - path: Ident::new("doc_path", proc_macro2::Span::call_site()).into(), - tokens: quote!((#tokens)), - }); - } - - Ok(Variant { - attrs, - ident, - fields: syn::Fields::Unit, - discriminant: None, - }) - } -} - -impl Parse for EasyEBMLVariant { - fn parse(input: ParseStream) -> syn::Result { - let path = input.parse()?; - input.parse::()?; - let ty: Ident = input.parse()?; - input.parse::()?; - let id: LitInt = input.parse()?; - Ok(Self { path, ty, id }) - } -} diff --git a/crates/ebml-iterable/specification-derive/src/lib.rs b/crates/ebml-iterable/specification-derive/src/lib.rs deleted file mode 100644 index 8c7449866..000000000 --- a/crates/ebml-iterable/specification-derive/src/lib.rs +++ /dev/null @@ -1,143 +0,0 @@ -extern crate proc_macro; - -mod ast; -mod attr; -mod easy_ebml; -mod pathing; - -use proc_macro::TokenStream; -use syn::{Error, ItemEnum}; - -use crate::easy_ebml::EasyEBML; - -/// -/// Attribute that derives implementations of [`EbmlSpecification`][spec] and [`EbmlTag`][tag] for an enum. -/// -/// This macro is intended to make implementing the traits in ebml-iterable-specification easier to manage. Rather than requiring handwritten implementations for [`EbmlSpecification`][spec] and [`EbmlTag`][tag] methods, this macro understands attributes assigned to enum members and generates an implementation accordingly. -/// -/// When deriving `EbmlSpecification` for an enum, the following attributes are required for each variant: -/// * __#[id(`u64`)]__ - This attribute specifies the "id" of the tag. e.g. `0x1a45dfa3` -/// * __#[data_type(`TagDataType`)]__ - This attribute specifies the type of data contained in the tag. e.g. `TagDataType::UnsignedInt` -/// -/// The following attribute is optional for each variant: -/// * __#[doc_path(Path/To/Element)]__ - This attribute specifies the document path of the current element. If this attribute is not present, the variant is treated as a Root element. Global elements can be defined with wildcard paths, e.g. #[doc_path(Segment/(1-)/)]. -/// -/// # Note -/// -/// This attribute modifies the variants in the enumeration by adding fields to them. It also will add the following variants to the enum: -/// - `Crc32(Vec)` - global tag defined in the EBML spec -/// - `Void(Vec)` - global tag defined in the EBML spec -/// - `RawTag(u64, Vec)` - used to support reading "unknown" tags that aren't in the spec -/// -/// [spec]: ebml_iterable_specification::EbmlSpecification -/// [tag]: ebml_iterable_specification::EbmlTag - -#[proc_macro_attribute] -pub fn ebml_specification(_args: TokenStream, input: TokenStream) -> TokenStream { - let mut input = match syn::parse::(input) { - Ok(syntax_tree) => syntax_tree, - Err(err) => { - return TokenStream::from( - Error::new( - err.span(), - "#[ebml_specification] attribute can only be applied to enums", - ) - .to_compile_error(), - ) - } - }; - - attr::impl_ebml_specification(&mut input) - .unwrap_or_else(|err| err.to_compile_error()) - .into() -} - -/// -/// Macro that makes writing an EBML spec easy. -/// -/// This provides an even easier alternative to create implementations of the [`EbmlSpecification`][spec] and [`EbmlTag`][tag] traits than using the [`[#ebml_specification]`][macro] attribute. As a bonus, your spec will be more legible and maintainable! -/// -/// As an example, compare the following equivalent definitions: -/// ``` -/// # use ebml_iterable_specification_derive::ebml_specification; -/// # use ebml_iterable_specification::TagDataType::{Master, UnsignedInt}; -/// # pub mod ebml_iterable { pub mod specs { -/// # pub use ebml_iterable_specification_derive::ebml_specification as ebml_specification; -/// # pub use ebml_iterable_specification::EbmlSpecification as EbmlSpecification; -/// # pub use ebml_iterable_specification::EbmlTag as EbmlTag; -/// # pub use ebml_iterable_specification::TagDataType as TagDataType; -/// # pub use ebml_iterable_specification::Master as Master; -/// # pub use ebml_iterable_specification::PathPart as PathPart; -/// # }} -/// #[ebml_specification] -/// #[derive(Clone)] -/// enum Example { -/// #[id(0x01)] -/// #[data_type(Master)] -/// Root, -/// -/// #[id(0x02)] -/// #[data_type(Master)] -/// #[doc_path(Root)] -/// Parent, -/// -/// #[id(0x100)] -/// #[data_type(UnsignedInt)] -/// #[doc_path(Root/Parent)] -/// Data, -/// } -/// ``` -/// vs -/// ``` -/// # use ebml_iterable_specification_derive::easy_ebml; -/// # use ebml_iterable_specification::TagDataType; -/// # use ebml_iterable_specification::TagDataType::{Master, UnsignedInt}; -/// # pub mod ebml_iterable { pub mod specs { -/// # pub use ebml_iterable_specification_derive::ebml_specification as ebml_specification; -/// # pub use ebml_iterable_specification::EbmlSpecification as EbmlSpecification; -/// # pub use ebml_iterable_specification::EbmlTag as EbmlTag; -/// # pub use ebml_iterable_specification::TagDataType as TagDataType; -/// # pub use ebml_iterable_specification::Master as Master; -/// # pub use ebml_iterable_specification::PathPart as PathPart; -/// # }} -/// easy_ebml! { -/// #[derive(Clone)] -/// enum Example { -/// Root : Master = 0x01, -/// Root/Parent : Master = 0x02, -/// Root/Parent/Data : UnsignedInt = 0x100, -/// } -/// } -/// ``` -/// -/// Behind the scenes `easy_ebml!` still uses the existing [`[#ebml_specification]`][macro] attribute macro, so the final output of this macro will remain identical. -/// -/// [spec]: ebml_iterable_specification::EbmlSpecification -/// [tag]: ebml_iterable_specification::EbmlTag -/// [macro]: macro@crate::ebml_specification - -#[proc_macro] -pub fn easy_ebml(input: TokenStream) -> TokenStream { - let input = match syn::parse::(input) { - Ok(syntax_tree) => syntax_tree, - Err(err) => { - return TokenStream::from( - Error::new( - err.span(), - "easy_ebml! {} content must be of format: enum Name {\ - Root: Type = id,\ - Path/Of/Component: Type = id,\ - // example\ - Ebml: Master = 0x1a45dfa3,\ - Ebml/EbmlVersion: UnsignedInt = 0x4286,\ - // global elements can be used in paths, example:\ - (1-)/Crc32: Binary = 0xbf,\ - }", - ) - .to_compile_error(), - ) - } - }; - - input.implement().unwrap_or_else(|err| err.to_compile_error()).into() -} diff --git a/crates/ebml-iterable/specification-derive/src/pathing.rs b/crates/ebml-iterable/specification-derive/src/pathing.rs deleted file mode 100644 index 201e703b6..000000000 --- a/crates/ebml-iterable/specification-derive/src/pathing.rs +++ /dev/null @@ -1,86 +0,0 @@ -use proc_macro2::Span; -use quote::{quote, TokenStreamExt}; -use syn::parse::{Parse, ParseStream}; -use syn::punctuated::Punctuated; -use syn::{Ident, LitInt, Result, Token}; - -pub struct EBMLPath { - pub span: Span, - pub parts: Punctuated, -} - -impl Parse for EBMLPath { - fn parse(input: ParseStream) -> Result { - let parts: Punctuated = Punctuated::parse_separated_nonempty(input)?; - Ok(Self { - parts, - span: input.span(), - }) - } -} - -#[derive(PartialEq)] -pub enum PathPart { - Ident(Ident), - Global((Option, Option)), -} - -impl Parse for PathPart { - fn parse(input: ParseStream) -> Result { - if input.lookahead1().peek(syn::token::Paren) { - let content; - syn::parenthesized!(content in input); - let min: Option = { - if content.lookahead1().peek(Token![-]) { - None - } else { - Some(content.parse::()?.base10_parse()?) - } - }; - content.parse::()?; - let max: Option = { - let val: Option = content.parse()?; - if let Some(val) = val { - Some(val.base10_parse()?) - } else { - None - } - }; - - Ok(PathPart::Global((min, max))) - } else { - let id: Ident = input.parse()?; - Ok(PathPart::Ident(id)) - } - } -} - -impl std::fmt::Display for PathPart { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - PathPart::Ident(id) => write!(f, "{id}"), - PathPart::Global((min, max)) => { - let min = if let Some(min) = min { - min.to_string() - } else { - String::new() - }; - let max = if let Some(max) = max { - max.to_string() - } else { - String::new() - }; - write!(f, "({min}-{max})") - } - } - } -} - -impl quote::ToTokens for PathPart { - fn to_tokens(&self, tokens: &mut proc_macro2::TokenStream) { - match self { - PathPart::Ident(id) => tokens.append(id.clone()), - PathPart::Global((min, max)) => tokens.extend(quote! {(#min-#max)}), - } - } -} diff --git a/crates/ebml-iterable/specification/Cargo.toml b/crates/ebml-iterable/specification/Cargo.toml deleted file mode 100644 index db0b5bbbe..000000000 --- a/crates/ebml-iterable/specification/Cargo.toml +++ /dev/null @@ -1,11 +0,0 @@ -[package] -name = "ebml-iterable-specification" -version = "0.4.0" -authors = ["Austin Blake "] -edition = "2018" -description = "Provides the base `EbmlSpecification` used by the `ebml-iterable` and `ebml-iterable-specification-derive` crates." -license = "MIT" -homepage = "https://github.com/austinleroy/ebml-iterable" -repository = "https://github.com/austinleroy/ebml-iterable" - -[dependencies] \ No newline at end of file diff --git a/crates/ebml-iterable/specification/src/empty_spec.rs b/crates/ebml-iterable/specification/src/empty_spec.rs deleted file mode 100644 index 3f32b27f7..000000000 --- a/crates/ebml-iterable/specification/src/empty_spec.rs +++ /dev/null @@ -1,104 +0,0 @@ -use super::{EbmlSpecification, EbmlTag, Master, PathPart, TagDataType}; - -/// -/// An empty specification for use with examples or testing. -/// -/// This struct isn't intended for production use and should only be used for examples or PoCs. Use at your own risk - may change in the future without warning. -/// -/// # NOT SUITABLE FOR PRODUCTION -/// -#[derive(Debug, Clone, PartialEq, Eq, Ord, PartialOrd)] -pub struct EmptySpec { - id: u64, - children: Option>, - data: Option>, -} - -impl EmptySpec { - pub fn with_children(id: u64, children: Vec) -> Self { - EmptySpec::get_master_tag(id, Master::Full(children)).unwrap() - } - - pub fn with_data(id: u64, data: &[u8]) -> Self { - EmptySpec::get_binary_tag(id, data).unwrap() - } -} - -impl EbmlSpecification for EmptySpec { - fn get_tag_data_type(_id: u64) -> Option { - Some(TagDataType::Binary) - } - - fn get_path_by_id(_id: u64) -> &'static [PathPart] { - &[] - } - - fn get_unsigned_int_tag(_id: u64, _data: u64) -> Option { - None - } - - fn get_signed_int_tag(_id: u64, _data: i64) -> Option { - None - } - - fn get_utf8_tag(_id: u64, _data: String) -> Option { - None - } - - fn get_binary_tag(id: u64, data: &[u8]) -> Option { - Some(EmptySpec { - id, - children: None, - data: Some(data.to_vec()), - }) - } - - fn get_float_tag(_id: u64, _data: f64) -> Option { - None - } - - fn get_master_tag(id: u64, data: Master) -> Option { - Some(EmptySpec { - id, - children: Some(data), - data: None, - }) - } - - fn get_raw_tag(id: u64, data: &[u8]) -> EmptySpec { - EmptySpec::get_binary_tag(id, data).expect("get binary tag for EmptySpec should always return Some") - } -} - -impl EbmlTag for EmptySpec { - fn get_id(&self) -> u64 { - self.id - } - - fn as_unsigned_int(&self) -> Option<&u64> { - None - } - - fn as_signed_int(&self) -> Option<&i64> { - None - } - - fn as_utf8(&self) -> Option<&str> { - None - } - - fn as_binary(&self) -> Option<&[u8]> { - self.data.as_deref() - } - - fn as_float(&self) -> Option<&f64> { - None - } - - fn as_master(&self) -> Option<&Master> { - match &self.children { - Some(children) => Some(children), - None => None, - } - } -} diff --git a/crates/ebml-iterable/specification/src/lib.rs b/crates/ebml-iterable/specification/src/lib.rs deleted file mode 100644 index 7da368a93..000000000 --- a/crates/ebml-iterable/specification/src/lib.rs +++ /dev/null @@ -1,233 +0,0 @@ -//! This crate provides a core ebml specification that is used by the ebml-iterable crate. -//! -//! The related ebml-iterable-specification-derive crate can be used to simplify implementation of this spec. -//! - -/// -/// Contains an empty specification for use with examples or very basic testing. -/// -pub mod empty_spec; - -/// -/// Different data types defined in the EBML specification. -/// -/// # Notes -/// -/// This library made a concious decision to not work with "Date" elements from EBML due to lack of built-in support for dates in Rust. Specification implementations should treat Date elements as Binary so that consumers have the option of parsing the unaltered data using their library of choice, if needed. -/// - -// Possible future feature flag to enable Date functionality by having `chrono` as an optional dependency? -#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug)] -pub enum TagDataType { - Master, - UnsignedInt, - Integer, - Utf8, - Binary, - Float, -} - -#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug)] -pub enum PathPart { - Id(u64), - Global((Option, Option)), -} - -/// -/// This trait, along with [`EbmlTag`], should be implemented to define a specification so that EBML can be parsed correctly. Typically implemented on an Enum of tag variants. -/// -/// Any specification using EBML can take advantage of this library to parse or write binary data. As stated in the docs, [`TagWriter`](https://docs.rs/ebml-iterable/latest/ebml_iterable/struct.TagWriter.html) needs nothing special if you stick with the `write_raw` method, but [`TagIterator`](https://docs.rs/ebml-iterable/latest/ebml_iterable/struct.TagIterator.html) requires a struct implementing this trait. Custom specification implementations can refer to [webm-iterable](https://crates.io/crates/webm_iterable) as an example. -/// -/// This trait and [`EbmlTag`] are typically implemented simultaneously. They are separate traits as they have primarily different uses - [`EbmlSpecification`] should be brought into scope when dealing with the specification as a whole, whereas [`EbmlTag`] should be brought into scope when dealing with specific tags. -pub trait EbmlSpecification + EbmlTag + Clone> { - /// - /// Pulls the data type for a tag from the spec, based on the tag id. - /// - /// This function *must* return [`None`] if the input id is not in the specification. Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. - /// - fn get_tag_data_type(id: u64) -> Option; - - /// - /// Gets the id of a specific tag variant. - /// - /// Default implementation uses the [`EbmlTag`] implementation. Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. - /// - fn get_tag_id(item: &T) -> u64 { - item.get_id() - } - - /// - /// Gets the schema path of a specific tag. - /// - /// This function is used to find the schema defined path of a tag. If the tag is a root element, this function should return an empty array. - /// - fn get_path_by_id(id: u64) -> &'static [PathPart]; - - /// - /// Gets the schema path of a specific tag variant. - /// - /// Default implementation uses [`Self::get_path_by_id`] after obtaining the tag id using the [`EbmlTag`] implementation. - /// - fn get_path_by_tag(item: &T) -> &'static [PathPart] { - Self::get_path_by_id(item.get_id()) - } - - /// - /// Creates an unsigned integer type tag from the spec. - /// - /// This function *must* return `None` if the input id is not in the specification or if the input id data type is not [`TagDataType::UnsignedInt`]. Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. - /// - fn get_unsigned_int_tag(id: u64, data: u64) -> Option; - - /// - /// Creates a signed integer type tag from the spec. - /// - /// This function *must* return `None` if the input id is not in the specification or if the input id data type is not [`TagDataType::Integer`]. Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. - /// - fn get_signed_int_tag(id: u64, data: i64) -> Option; - - /// - /// Creates a utf8 type tag from the spec. - /// - /// This function *must* return `None` if the input id is not in the specification or if the input id data type is not [`TagDataType::Utf8`]. Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. - /// - fn get_utf8_tag(id: u64, data: String) -> Option; - - /// - /// Creates a binary type tag from the spec. - /// - /// This function *must* return `None` if the input id is not in the specification or if the input id data type is not [`TagDataType::Binary`]. Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. - /// - fn get_binary_tag(id: u64, data: &[u8]) -> Option; - - /// - /// Creates a float type tag from the spec. - /// - /// This function *must* return `None` if the input id is not in the specification or if the input id data type is not [`TagDataType::Float`]. Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. - /// - fn get_float_tag(id: u64, data: f64) -> Option; - - /// - /// Creates a master type tag from the spec. - /// - /// This function *must* return `None` if the input id is not in the specification or if the input id data type is not [`TagDataType::Master`]. Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. - /// - fn get_master_tag(id: u64, data: Master) -> Option; - - /// - /// Creates a tag that does not conform to the spec. - /// - /// This function should return a "RawTag" variant that contains the tag id and tag data. Tag data should only be retrievable as binary data. Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. - /// - fn get_raw_tag(id: u64, data: &[u8]) -> T; -} - -/// -/// This trait, along with [`EbmlSpecification`], should be implemented to define a specification so that EBML can be parsed correctly. Typically implemented on an Enum of tag variants. -/// -/// Any specification using EBML can take advantage of this library to parse or write binary data. As stated in the docs, [`TagWriter`](https://docs.rs/ebml-iterable/latest/ebml_iterable/struct.TagWriter.html) needs nothing special if you stick with the `write_raw` method, but [`TagIterator`](https://docs.rs/ebml-iterable/latest/ebml_iterable/struct.TagIterator.html) requires a struct implementing this trait. Custom specification implementations can refer to [webm-iterable](https://crates.io/crates/webm_iterable) as an example. -/// -/// This trait and [`EbmlSpecification`] are typically implemented simultaneously. They are separate traits as they have primarily different uses - [`EbmlSpecification`] should be brought into scope when dealing with the specification as a whole, whereas [`EbmlTag`] should be brought into scope when dealing with specific tags. -pub trait EbmlTag { - /// - /// Gets the id of `self`. - /// - /// Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. - /// - fn get_id(&self) -> u64; - - /// - /// Gets a reference to the data contained in `self` as an unsigned integer. - /// - /// This function *must* return `None` if the associated data type of `self` is not [`TagDataType::UnsignedInt`]. Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. - /// - fn as_unsigned_int(&self) -> Option<&u64>; - - /// - /// Gets a reference to the data contained in `self` as an integer. - /// - /// This function *must* return `None` if the associated data type of `self` is not [`TagDataType::Integer`]. Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. - /// - fn as_signed_int(&self) -> Option<&i64>; - - /// - /// Gets a reference to the data contained in `self` as string slice. - /// - /// This function *must* return `None` if the associated data type of `self` is not [`TagDataType::Utf8`]. Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. - /// - fn as_utf8(&self) -> Option<&str>; - - /// - /// Gets a reference to the data contained in `self` as binary data. - /// - /// This function *must* return `None` if the associated data type of `self` is not [`TagDataType::Binary`]. Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. - /// - fn as_binary(&self) -> Option<&[u8]>; - - /// - /// Gets a reference to the data contained in `self` as float data. - /// - /// This function *must* return `None` if the associated data type of `self` is not [`TagDataType::Float`]. Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. - /// - fn as_float(&self) -> Option<&f64>; - - /// - /// Gets a reference to master data contained in `self`. - /// - /// This function *must* return `None` if the associated data type of `self` is not [`TagDataType::Master`]. Implementors can reference [webm-iterable](https://crates.io/crates/webm_iterable) for an example. - /// - fn as_master(&self) -> Option<&Master>; -} - -/// -/// An enum that defines different possible states of a [`TagDataType::Master`] tag. -/// -/// A "master" tag is a type of tag that contains other tags within it. Because these tags are dynamically sized, the [`TagIterator`](https://docs.rs/ebml-iterable/latest/ebml_iterable/struct.TagIterator.html) emits these tags as [`Master::Start`] and [`Master::End`] variants by default so that the entire tag does not need to be buffered into memory all at once. The [`Master::Full`] variant is a complete "master" tag that includes all child tags within it. -/// -#[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug)] -pub enum Master { - /// - /// Designates the start of a tag. - /// - Start, - - /// - /// Designates the end of a tag. - /// - End, - - /// - /// Designates a full tag. `Vec` contains all child tags contained in this master tag. - /// - Full(Vec), -} - -impl Master { - /// - /// Convenience method to pull children from a master tag. - /// - /// # Panics - /// - /// Panics if `self` is not a `Full` variant. - /// - /// # Examples - /// - /// ``` - /// # use ebml_iterable_specification::empty_spec::EmptySpec; - /// use ebml_iterable_specification::Master; - /// - /// let children = vec![EmptySpec::with_data(0x1253, &[1]), EmptySpec::with_data(0x1234, &[2])]; - /// // Clone children because creating a Master consumes it - /// let tag = Master::Full(children.clone()); - /// let retrieved_children = tag.get_children(); - /// assert_eq!(retrieved_children, children); - /// ``` - /// - pub fn get_children(self) -> Vec { - match self { - Master::Full(data) => data, - Master::Start => panic!("`get_children` called on Master::Start variant"), - Master::End => panic!("`get_children` called on Master::End variant"), - } - } -} diff --git a/crates/ebml-iterable/src/errors.rs b/crates/ebml-iterable/src/errors.rs deleted file mode 100644 index 5cf00a78d..000000000 --- a/crates/ebml-iterable/src/errors.rs +++ /dev/null @@ -1,363 +0,0 @@ -use std::error::Error; -use std::fmt; - -pub mod tool { - use std::string::FromUtf8Error; - - use super::{fmt, Error}; - - #[derive(Debug)] - pub enum ToolError { - ReadVintOverflow, - WriteVintOverflow(u64), - WriteSignedVintOverflow(i64), - ReadU64Overflow(Vec), - ReadI64Overflow(Vec), - ReadF64Mismatch(Vec), - FromUtf8Error(Vec, FromUtf8Error), - } - - impl fmt::Display for ToolError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - ToolError::ReadVintOverflow => write!(f, "Unrepresentable Vint size encountered."), - ToolError::WriteVintOverflow(val) => write!(f, "Value too large to be written as a vint: {val}"), - ToolError::WriteSignedVintOverflow(val) => { - write!(f, "Value outside range to be written as a vint: {val}") - } - ToolError::ReadU64Overflow(arr) => write!(f, "Could not read unsigned int from array: {arr:?}"), - ToolError::ReadI64Overflow(arr) => write!(f, "Could not read int from array: {arr:?}"), - ToolError::ReadF64Mismatch(arr) => write!(f, "Could not read float from array: {arr:?}"), - ToolError::FromUtf8Error(arr, _source) => write!(f, "Could not read utf8 data: {arr:?}"), - } - } - } - - impl Error for ToolError { - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - ToolError::FromUtf8Error(_arr, source) => Some(source), - _ => None, - } - } - } -} - -pub mod tag_iterator { - use std::io; - - use super::tool::ToolError; - use super::{fmt, Error}; - - /// - /// Errors that indicate file data is corrupted. - /// - #[derive(Debug)] - pub enum CorruptedFileError { - /// - /// An error indicating the reader found an ebml tag id not defined in the current specification. - /// - InvalidTagId { - /// - /// The position of the element. - /// - position: usize, - - /// - /// The id of the tag that was found. - /// - tag_id: u64, - }, - - /// - /// An error indicating the reader could not parse a valid tag due to corrupted tag data (size/contents). - /// - InvalidTagData { - /// - /// The position of the element. - /// - position: usize, - - /// - /// The id of the tag that was found. - /// - tag_id: u64, - }, - - /// - /// An error indicating the reader found an element outside of its expected hierarchy. - /// - HierarchyError { - /// - /// The id of the tag that was found. - /// - found_tag_id: u64, - - /// - /// The id of the current "master" element that contains the tag that was found. - /// - current_parent_id: Option, - }, - - /// - /// An error indicating the reader found a child element with incorrect sizing. - /// - OversizedChildElement { - /// - /// The position of the element. - /// - position: usize, - - /// - /// The id of the tag that was found. - /// - tag_id: u64, - - /// - /// The size of the tag that was found. - /// - size: usize, - }, - - /// - /// An error indicating the reader found a tag with an invalid size. - /// - InvalidTagSize { - /// - /// The position of the element. - /// - position: usize, - - /// - /// The id of the tag that was found. - /// - tag_id: u64, - - /// - /// The size of the tag that was found. - /// - size: usize, - }, - } - - impl fmt::Display for CorruptedFileError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - CorruptedFileError::InvalidTagId { - position, - tag_id - } => write!(f, "Encountered invalid tag id [0x{tag_id:x?}] at position {position}"), - CorruptedFileError::InvalidTagData { - position, - tag_id - } => write!(f, "Encountered invalid tag data for tag id [0x{tag_id:x?}] at position {position}"), - CorruptedFileError::HierarchyError { - found_tag_id, - current_parent_id, - } => write!(f, "Found child tag [{found_tag_id:x?}] when processing parent [{current_parent_id:x?}]"), - CorruptedFileError::OversizedChildElement { - position, - tag_id, - size : _ - } => write!(f, "Found an oversized tag [0x{tag_id:x?}] at position {position}"), - CorruptedFileError::InvalidTagSize { - position, - tag_id, - size, - } => write!(f, "Found an oversized tag [0x{tag_id:x?}] at position {position} with size {size}. Max supported size is 8GB."), - } - } - } - - /// - /// Errors that can occur when reading ebml data. - /// - #[derive(Debug)] - pub enum TagIteratorError { - /// - /// An error indicating that data in the file being read is not valid. - /// - CorruptedFileData(CorruptedFileError), - - /// - /// An error indicating that the iterator reached the end of the input stream unexpectedly while reading a tag. - /// - /// This error will occur if the iterator is expecting more data (either due to expecting a size after reading a tag id or based on a tag size) but nothing is available in the input stream. - /// - UnexpectedEOF { - /// - /// The start position of the tag that was being read when EOF was reached. - /// - tag_start: usize, - - /// - /// The id of the partially read tag, if available. - /// - tag_id: Option, - - /// - /// The size of the partially read tag, if available. - /// - tag_size: Option, - - /// - /// Any available data that was read for the tag before reaching EOF. - /// - partial_data: Option>, - }, - - /// - /// An error indicating that tag data appears to be corrupted. - /// - /// This error typically occurs if tag data cannot be read as its expected data type (e.g. trying to read `[32,42,8]` as float data, since floats require either 4 or 8 bytes). - /// - CorruptedTagData { - /// - /// The id of the corrupted tag. - /// - tag_id: u64, - - /// - /// An error describing why the data is corrupted. - /// - problem: ToolError, - }, - - /// - /// An error that wraps an IO error when reading from the underlying source. - /// - ReadError { - /// - /// The [`io::Error`] that caused this problem. - /// - source: io::Error, - }, - } - - impl fmt::Display for TagIteratorError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - TagIteratorError::CorruptedFileData(err) => write!(f, "Encountered corrupted data. Message: {err}"), - TagIteratorError::UnexpectedEOF { - tag_start, - tag_id, - tag_size, - partial_data: _ - } => write!(f, "Reached EOF unexpectedly. Partial tag data: {{tag offset:{tag_start}}} {{id:{tag_id:x?}}} {{size:{tag_size:?}}}"), - TagIteratorError::CorruptedTagData { - tag_id, - problem, - } => write!(f, "Error reading data for tag id (0x{tag_id:x?}). {problem}"), - TagIteratorError::ReadError { source: _ } => write!(f, "Error reading from source."), - } - } - } - - impl Error for TagIteratorError { - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - TagIteratorError::CorruptedFileData(_) => None, - TagIteratorError::UnexpectedEOF { - tag_start: _, - tag_id: _, - tag_size: _, - partial_data: _, - } => None, - TagIteratorError::CorruptedTagData { tag_id: _, problem } => problem.source(), - TagIteratorError::ReadError { source } => Some(source), - } - } - } -} - -pub mod tag_writer { - use std::io; - - use super::{fmt, Error}; - - /// - /// Errors that can occur when writing ebml data. - /// - #[derive(Debug)] - pub enum TagWriterError { - /// - /// An error indicating the tag to be written doesn't conform to the current specification. - /// - /// This error occurs if you attempt to write a tag outside of a valid document path. See the [EBML RFC](https://www.rfc-editor.org/rfc/rfc8794.html#section-11.1.6.2) for details on element paths. - /// - UnexpectedTag { tag_id: u64, current_path: Vec }, - - /// - /// An error with a tag id. - /// - /// This error should only occur if writing "RawTag" variants, and only if the input id is not a valid vint. - /// - TagIdError(u64), - - /// - /// An error with the size of a tag. - /// - /// Can occur if the tag size overflows the max value representable by a vint (`2^57 - 1`, or `144,115,188,075,855,871`). - /// - /// This can also occur if a non-[`Master`][`crate::specs::TagDataType::Master`] tag is sent to be written with an unknown size. - /// - TagSizeError(String), - - /// - /// An error indicating a tag was closed unexpectedly. - /// - /// Can occur if a [`Master::End`][`crate::specs::Master::End`] variant is passed to the [`TagWriter`][`crate::TagWriter`] but the id doesn't match the currently open tag. - /// - UnexpectedClosingTag { - /// - /// The id of the tag being closed. - /// - tag_id: u64, - - /// - /// The id of the currently open tag. - /// - expected_id: Option, - }, - - /// - /// An error that wraps an IO error when writing to the underlying destination. - /// - WriteError { source: io::Error }, - } - - impl fmt::Display for TagWriterError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - TagWriterError::UnexpectedTag { tag_id, current_path } => { - write!(f, "Unexpected tag 0x{tag_id:x?} when writing to {current_path:x?}") - } - TagWriterError::TagIdError(id) => write!(f, "Tag id 0x{id:x?} is not a valid vint"), - TagWriterError::TagSizeError(message) => write!(f, "Problem writing data tag size. {message}"), - TagWriterError::UnexpectedClosingTag { tag_id, expected_id } => match expected_id { - Some(expected) => write!(f, "Unexpected closing tag 0x'{tag_id:x?}'. Expected 0x'{expected:x?}'"), - None => write!(f, "Unexpected closing tag 0x'{tag_id:x?}'"), - }, - TagWriterError::WriteError { source: _ } => write!(f, "Error writing to destination."), - } - } - } - - impl Error for TagWriterError { - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - TagWriterError::UnexpectedTag { - tag_id: _, - current_path: _, - } => None, - TagWriterError::TagIdError(_) => None, - TagWriterError::TagSizeError(_) => None, - TagWriterError::UnexpectedClosingTag { - tag_id: _, - expected_id: _, - } => None, - TagWriterError::WriteError { source } => Some(source), - } - } - } -} diff --git a/crates/ebml-iterable/src/lib.rs b/crates/ebml-iterable/src/lib.rs deleted file mode 100644 index 33156c6c4..000000000 --- a/crates/ebml-iterable/src/lib.rs +++ /dev/null @@ -1,58 +0,0 @@ -//! This crate provides an iterator and a serializer for [EBML][EBML] files. Its primary goal is to provide typed iteration and serialization as lightly and quickly as possible. -//! -//! [EBML][EBML] stands for Extensible Binary Meta-Language and is somewhat of a -//! binary version of XML. It's used for container formats like [WebM][webm] or -//! [MKV][mkv]. -//! -//! # Important - Specifications -//! The iterator contained in this crate is spec-agnostic and requires a specification implementing the [`specs::EbmlSpecification`] and [`specs::EbmlTag`] traits to read files. Typically, you would only use this crate to implement a custom specification - most often you would prefer a crate providing an existing specification, like [webm-iterable][webm-iterable]. -//! -//! Implementing custom specifications can be made less painful and safer by enabling the `"derive-spec"` feature flag in this crate and using the [`#[ebml_specification]`](https://docs.rs/ebml-iterable-specification-derive/latest/ebml_iterable_specification_derive/attr.ebml_specification.html) macro. -//! -//! # Features -//! -//! There is currently only one optional feature in this crate, but that may change over time as needs arise. -//! -//! * **derive-spec** - -//! When enabled, this provides the [`#[ebml_specification]`](https://docs.rs/ebml-iterable-specification-derive/latest/ebml_iterable_specification_derive/attr.ebml_specification.html) attribute macro to simplify implementation of the [`EbmlSpecification`][`specs::EbmlSpecification`] and [`EbmlTag`][`specs::EbmlTag`] traits. This introduces dependencies on [`syn`](https://crates.io/crates/syn), [`quote`](https://crates.io/crates/quote), and [`proc-macro2`](https://crates.io/crates/proc-macro2), so expect compile times to increase a little. -//! -//! [EBML]: http://ebml.sourceforge.net/ -//! [webm]: https://www.webmproject.org/ -//! [mkv]: http://www.matroska.org/technical/specs/index.html -//! [rfc8794]: https://datatracker.ietf.org/doc/rfc8794/ -//! [webm-iterable]: https://crates.io/crates/webm_iterable -//! - -mod errors; -mod spec_util; -pub mod specs; -mod tag_decoder; -mod tag_iterator; -mod tag_iterator_util; -mod tag_parse; -mod tag_writer; -pub mod tools; - -#[cfg(feature = "futures")] -pub mod nonblocking; - -pub use self::tag_decoder::{PositionedTag, TagDecoder}; -pub use self::tag_iterator::TagIterator; -pub use self::tag_writer::{TagWriter, WriteOptions}; - -pub mod iterator { - pub use super::tag_iterator_util::AllowableErrors; -} - -pub mod error { - - //! - //! Potential errors that can occur when reading or writing EBML data. - //! - pub use super::errors::tag_iterator::{CorruptedFileError, TagIteratorError}; - pub use super::errors::tag_writer::TagWriterError; - /// - /// Error details that may be included in some thrown errors - /// - pub use super::errors::tool::ToolError; -} diff --git a/crates/ebml-iterable/src/nonblocking.rs b/crates/ebml-iterable/src/nonblocking.rs deleted file mode 100644 index a7eb8f6ce..000000000 --- a/crates/ebml-iterable/src/nonblocking.rs +++ /dev/null @@ -1,59 +0,0 @@ -use std::io::Cursor; - -use ebml_iterable_specification::{EbmlSpecification, EbmlTag}; -use futures::{AsyncRead, AsyncReadExt, Stream}; - -use crate::error::TagIteratorError; -use crate::TagIterator; - -/// -/// This can be transformed into a [`Stream`] using [`into_stream`][TagIteratorAsync::into_stream], or consumed directly by calling [`.next().await`] in a loop. -/// -/// The struct can be created with the [`new()`][TagIteratorAsync::new] function on any source that implements the [`futures::AsyncRead`] trait. -/// -pub struct TagIteratorAsync -where - TSpec: EbmlSpecification + EbmlTag + Clone, -{ - source: R, - buffer: Box<[u8]>, - iterator: TagIterator>, TSpec>, -} - -impl TagIteratorAsync -where - TSpec: EbmlSpecification + EbmlTag + Clone, -{ - pub fn new(source: R, tags_to_buffer: &[TSpec]) -> Self { - let buffer = vec![0u8; 1024 * 64]; - Self { - source, - buffer: buffer.into_boxed_slice(), - iterator: TagIterator::new(Cursor::new(Vec::new()), tags_to_buffer), - } - } - - pub async fn next(&mut self) -> Option> { - match self.source.read(&mut self.buffer).await { - Ok(len) => { - self.iterator - .get_mut() - .get_mut() - .append(&mut self.buffer[..len].to_vec()); - self.iterator.next() - } - Err(e) => Some(Err(TagIteratorError::ReadError { source: e })), - } - } - - pub fn into_stream(self) -> impl Stream> { - futures::stream::unfold(self, |mut read| async { - let next = read.next().await; - next.map(move |it| (it, read)) - }) - } - - pub fn last_emitted_tag_offset(&self) -> usize { - self.iterator.last_emitted_tag_offset() - } -} diff --git a/crates/ebml-iterable/src/spec_util.rs b/crates/ebml-iterable/src/spec_util.rs deleted file mode 100644 index 32f6f48ba..000000000 --- a/crates/ebml-iterable/src/spec_util.rs +++ /dev/null @@ -1,89 +0,0 @@ -use ebml_iterable_specification::{EbmlSpecification, EbmlTag, PathPart}; - -use crate::tag_iterator_util::EBMLSize; - -/// -/// Returns whether or not the a `test_id` is a parent of `current_id`. -/// -pub fn is_parent + EbmlTag + Clone>(current_id: u64, test_id: u64) -> bool { - let path = ::get_path_by_id(current_id); - path.iter().any(|p| matches!(p, PathPart::Id(p) if p == &test_id)) -} - -/// -/// Returns whether or not the `test_id` is a sibling of `current_id`. -/// -/// A sibling tag is one which shares the same direct parent. A separate instance of the current tag counts as a sibling. -/// -pub fn is_sibling + EbmlTag + Clone>(current_id: u64, test_id: u64) -> bool { - ::get_path_by_id(current_id) == ::get_path_by_id(test_id) -} - -/// -/// Returns whether or not the `test_id` would end this "Unknown" sized `current_id`. -/// -/// Regarding this method, unknown sized tags can be ended if we reach an element that is: -/// - A parent of the tag -/// - A direct sibling of the tag -/// - A Root element -/// -/// There are a couple of other cases where an Unknown sized tag can end, but they rely on knowing document position and tag sizes. More details can be found in the [EBML RFC](https://www.rfc-editor.org/rfc/rfc8794.html#name-unknown-data-size). -/// -pub fn is_ended_by + EbmlTag + Clone>(current_id: u64, test_id: u64) -> bool { - is_parent::(current_id, test_id) || // parent - is_sibling::(current_id, test_id) || // sibling - ( // Root element - ::get_tag_data_type(test_id).is_some() && - ::get_path_by_id(test_id).is_empty() - ) -} - -#[inline(always)] -pub fn validate_tag_path + EbmlTag + Clone>( - tag_id: u64, - doc_path: impl Iterator, -) -> bool { - let path = ::get_path_by_id(tag_id); - let mut path_marker = 0; - let mut global_counter = 0; - for item in doc_path { - let current_node_id = item.0; - - if !item.1.is_known() && is_ended_by::(current_node_id, tag_id) { - return true; - } - - if path_marker >= path.len() { - return false; - } - - match path[path_marker] { - PathPart::Id(id) => { - if id != current_node_id { - return false; - } - path_marker += 1; - } - PathPart::Global((min, max)) => { - global_counter += 1; - if max.is_some() && global_counter > max.unwrap_or_default() { - return false; - } - if path.len() > (path_marker + 1) - && matches!(path[path_marker + 1], PathPart::Id(id) if id == current_node_id) - { - if min.is_some() && global_counter < min.unwrap_or_default() { - return false; - } - path_marker += 2; - global_counter = 0; - } - } - } - } - - // Validate that we compared ALL parents in the path - path.len() == path_marker || - // or that the last parent was a global whose minimum was met - ((path.len() - 1) == path_marker && matches!(path[path_marker], PathPart::Global((min, _)) if global_counter >= min.unwrap_or(0))) -} diff --git a/crates/ebml-iterable/src/specs.rs b/crates/ebml-iterable/src/specs.rs deleted file mode 100644 index b7458bd9d..000000000 --- a/crates/ebml-iterable/src/specs.rs +++ /dev/null @@ -1,11 +0,0 @@ -//! -//! Provides the EBML specification types. -//! -//! Typically won't be used unless you are implementing a custom specification that uses EBML. You can enable the `"derive-spec"` feature to obtain a macro to make implementation easier. -//! - -pub use ebml_iterable_specification::{EbmlSpecification, EbmlTag, Master, PathPart, TagDataType}; -#[cfg(feature = "derive-spec")] -pub use ebml_iterable_specification_derive::easy_ebml; -#[cfg(feature = "derive-spec")] -pub use ebml_iterable_specification_derive::ebml_specification; diff --git a/crates/ebml-iterable/src/tag_decoder.rs b/crates/ebml-iterable/src/tag_decoder.rs deleted file mode 100644 index 5c2889f2c..000000000 --- a/crates/ebml-iterable/src/tag_decoder.rs +++ /dev/null @@ -1,406 +0,0 @@ -// This decoder extends Austin Blake's MIT-licensed ebml-iterable 0.6.3. -// Gateway vendors it because growing recordings must keep partial tags across appends. - -use std::collections::{HashSet, VecDeque}; - -use bytes::{Buf, BytesMut}; - -use crate::errors::tag_iterator::{CorruptedFileError, TagIteratorError}; -use crate::spec_util::{is_ended_by, validate_tag_path}; -use crate::specs::{EbmlSpecification, EbmlTag, Master, PathPart, TagDataType}; -use crate::tag_iterator_util::EBMLSize::{Known, Unknown}; -use crate::tag_iterator_util::{AllowableErrors, EBMLSize}; -use crate::tag_parse::{self, TagHeader}; -use crate::tools; - -const INVALID_TAG_ID_ERROR: u8 = 0x01; -const INVALID_HIERARCHY_ERROR: u8 = 0x02; -const OVERSIZED_CHILD_ERROR: u8 = 0x04; - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct PositionedTag { - pub tag: TSpec, - pub offset: usize, -} - -struct OpenTag { - id: u64, - size: EBMLSize, - tag_start: usize, - data_start: usize, - capture: bool, - children: Vec, -} - -pub struct TagDecoder -where - TSpec: EbmlSpecification + EbmlTag + Clone, -{ - tag_ids_to_buffer: HashSet, - allowed_errors: u8, - max_allowed_tag_size: Option, - position: usize, - tag_stack: Vec>, - emission_queue: VecDeque>, - has_determined_doc_path: bool, - finished: bool, -} - -impl TagDecoder -where - TSpec: EbmlSpecification + EbmlTag + Clone, -{ - pub fn new(tags_to_buffer: &[TSpec]) -> Self { - Self { - tag_ids_to_buffer: tags_to_buffer.iter().map(EbmlTag::get_id).collect(), - allowed_errors: 0, - max_allowed_tag_size: Some(4 * usize::pow(1000, 3)), - position: 0, - tag_stack: Vec::new(), - emission_queue: VecDeque::new(), - has_determined_doc_path: false, - finished: false, - } - } - - pub fn allow_errors(&mut self, errors: &[AllowableErrors]) { - self.allowed_errors = errors.iter().fold(0u8, |allowed, error| match error { - AllowableErrors::InvalidTagIds => allowed | INVALID_TAG_ID_ERROR, - AllowableErrors::HierarchyProblems => allowed | INVALID_HIERARCHY_ERROR, - AllowableErrors::OversizedTags => allowed | OVERSIZED_CHILD_ERROR, - }); - } - - pub fn set_max_allowable_tag_size(&mut self, size: Option) { - self.max_allowed_tag_size = size; - } - - pub fn position(&self) -> usize { - self.position - } - - pub fn decode(&mut self, input: &mut BytesMut) -> Result>, TagIteratorError> { - assert!( - !self.finished || input.is_empty(), - "cannot append EBML data after end of input" - ); - if let Some(tag) = self.emission_queue.pop_front() { - return Ok(Some(tag)); - } - if self.finished { - return Ok(None); - } - - loop { - self.close_completed_tags()?; - if let Some(tag) = self.emission_queue.pop_front() { - return Ok(Some(tag)); - } - - if input.is_empty() { - return Ok(None); - } - - let Some(header) = tag_parse::read_header::(input, self.position)? else { - return Ok(None); - }; - - self.close_unknown_tags(header.id)?; - if let Some(tag) = self.emission_queue.pop_front() { - return Ok(Some(tag)); - } - - self.validate_header(&header)?; - if matches!(header.data_type, Some(TagDataType::Master)) { - self.open_master(input, header)?; - } else { - let Some(tag) = self.read_data_tag(input, &header)? else { - return Ok(None); - }; - self.deliver(tag, self.position); - self.advance(input, header.len + header.size.value()); - } - - if let Some(tag) = self.emission_queue.pop_front() { - return Ok(Some(tag)); - } - } - } - - pub fn decode_eof(&mut self, input: &mut BytesMut) -> Result>, TagIteratorError> { - if let Some(tag) = self.decode(input)? { - return Ok(Some(tag)); - } - - if !input.is_empty() { - return Err(self.unexpected_eof(input)); - } - - if let Some(open_tag) = self - .tag_stack - .iter() - .find(|tag| matches!(tag.size, Known(size) if self.position < tag.data_start.saturating_add(size))) - { - return Err(TagIteratorError::UnexpectedEOF { - tag_start: open_tag.tag_start, - tag_id: Some(open_tag.id), - tag_size: match open_tag.size { - Known(size) => Some(size), - Unknown => None, - }, - partial_data: None, - }); - } - - while !self.tag_stack.is_empty() { - self.close_top()?; - } - self.finished = true; - - Ok(self.emission_queue.pop_front()) - } - - pub fn is_finished(&self) -> bool { - self.finished && self.emission_queue.is_empty() - } - - fn validate_header(&mut self, header: &TagHeader) -> Result<(), TagIteratorError> { - if self.allowed_errors & INVALID_TAG_ID_ERROR == 0 && header.data_type.is_none() { - return Err(TagIteratorError::CorruptedFileData(CorruptedFileError::InvalidTagId { - position: self.position, - tag_id: header.id, - })); - } - - self.determine_doc_path(header.id); - if self.allowed_errors & INVALID_HIERARCHY_ERROR == 0 - && header.data_type.is_some() - && self.has_determined_doc_path - && !validate_tag_path::(header.id, self.tag_stack.iter().map(|tag| (tag.id, tag.size, 0))) - { - return Err(TagIteratorError::CorruptedFileData( - CorruptedFileError::HierarchyError { - found_tag_id: header.id, - current_parent_id: self.tag_stack.last().map(|tag| tag.id), - }, - )); - } - - if let Known(size) = header.size { - let total_size = header.len.checked_add(size).ok_or({ - TagIteratorError::CorruptedFileData(CorruptedFileError::InvalidTagSize { - position: self.position, - tag_id: header.id, - size, - }) - })?; - let element_end = self.position.checked_add(total_size).ok_or({ - TagIteratorError::CorruptedFileData(CorruptedFileError::InvalidTagSize { - position: self.position, - tag_id: header.id, - size, - }) - })?; - - if self.allowed_errors & OVERSIZED_CHILD_ERROR == 0 - && self.tag_stack.iter().any(|tag| { - matches!(tag.size, Known(parent_size) if tag.data_start.saturating_add(parent_size) < element_end) - }) - { - return Err(TagIteratorError::CorruptedFileData( - CorruptedFileError::OversizedChildElement { - position: self.position, - tag_id: header.id, - size, - }, - )); - } - - if self.max_allowed_tag_size.is_some_and(|max_size| size > max_size) { - return Err(TagIteratorError::CorruptedFileData( - CorruptedFileError::InvalidTagSize { - position: self.position, - tag_id: header.id, - size, - }, - )); - } - } - - Ok(()) - } - - fn determine_doc_path(&mut self, tag_id: u64) { - if self.has_determined_doc_path { - return; - } - - let path = TSpec::get_path_by_id(tag_id); - if !path.iter().all(|part| matches!(part, PathPart::Id(_))) { - return; - } - - self.tag_stack = path - .iter() - .map(|part| match part { - PathPart::Id(id) => OpenTag { - id: *id, - size: Unknown, - tag_start: 0, - data_start: 0, - capture: false, - children: Vec::new(), - }, - PathPart::Global(_) => unreachable!(), - }) - .collect(); - self.has_determined_doc_path = true; - } - - fn open_master(&mut self, input: &mut BytesMut, header: TagHeader) -> Result<(), TagIteratorError> { - let tag_start = self.position; - let data_start = self - .position - .checked_add(header.len) - .expect("validated tag header length should fit in usize"); - let capture = - self.tag_ids_to_buffer.contains(&header.id) || self.tag_stack.last().is_some_and(|parent| parent.capture); - let start = TSpec::get_master_tag(header.id, Master::Start).unwrap_or_else(|| { - panic!( - "Bad specification implementation: Tag id 0x{:x?} type was master, but could not get tag!", - header.id - ) - }); - - self.advance(input, header.len); - self.tag_stack.push(OpenTag { - id: header.id, - size: header.size, - tag_start, - data_start, - capture, - children: Vec::new(), - }); - if !capture { - self.emission_queue.push_back(PositionedTag { - tag: start, - offset: tag_start, - }); - } - - Ok(()) - } - - fn read_data_tag(&self, input: &[u8], header: &TagHeader) -> Result, TagIteratorError> { - let Known(size) = header.size else { - return Err(TagIteratorError::CorruptedFileData( - CorruptedFileError::InvalidTagData { - tag_id: header.id, - position: self.position, - }, - )); - }; - let total_size = header - .len - .checked_add(size) - .expect("validated tag size should fit in usize"); - if input.len() < total_size { - return Ok(None); - } - - let raw_data = &input[header.len..total_size]; - tag_parse::read_data_tag(header.id, header.data_type, raw_data).map(Some) - } - - fn close_completed_tags(&mut self) -> Result<(), TagIteratorError> { - let ended_index = self - .tag_stack - .iter() - .position(|tag| matches!(tag.size, Known(size) if self.position >= tag.data_start.saturating_add(size))); - if let Some(index) = ended_index { - while self.tag_stack.len() > index { - self.close_top()?; - } - } - Ok(()) - } - - fn close_unknown_tags(&mut self, next_id: u64) -> Result<(), TagIteratorError> { - while self - .tag_stack - .last() - .is_some_and(|tag| tag.size == Unknown && is_ended_by::(tag.id, next_id)) - { - self.close_top()?; - } - Ok(()) - } - - fn close_top(&mut self) -> Result<(), TagIteratorError> { - let open_tag = self - .tag_stack - .pop() - .expect("an open tag should exist before it is closed"); - let id = open_tag.id; - let tag_start = open_tag.tag_start; - let tag = if open_tag.capture { - TSpec::get_master_tag(id, Master::Full(open_tag.children)) - } else { - TSpec::get_master_tag(id, Master::End) - } - .unwrap_or_else(|| { - panic!( - "Bad specification implementation: Tag id 0x{:x?} type was master, but could not get tag!", - id - ) - }); - self.deliver(tag, tag_start); - Ok(()) - } - - fn deliver(&mut self, tag: TSpec, offset: usize) { - if let Some(parent) = self.tag_stack.iter_mut().rev().find(|parent| parent.capture) { - parent.children.push(tag); - } else { - self.emission_queue.push_back(PositionedTag { tag, offset }); - } - } - - fn advance(&mut self, input: &mut BytesMut, count: usize) { - input.advance(count); - self.position = self - .position - .checked_add(count) - .expect("validated tag size should keep the decoder position in usize"); - } - - fn unexpected_eof(&self, input: &[u8]) -> TagIteratorError { - let first = input.first().copied(); - let id_len = first.map_or(0, |byte| if byte == 0 { 1 } else { 8 - byte.ilog2() as usize }); - let tag_id = (id_len > 0 && input.len() >= id_len).then(|| { - input[..id_len] - .iter() - .fold(0u64, |value, byte| (value << 8) + u64::from(*byte)) - }); - let size = tag_id.and_then(|_| { - tools::read_vint(input.get(id_len..).unwrap_or_default()) - .ok() - .flatten() - .and_then(|(size, size_len)| match EBMLSize::new(size, size_len) { - Known(size) => Some((size, id_len + size_len)), - Unknown => None, - }) - }); - let partial_data = size.map_or_else( - || Some(input.to_vec()), - |(_, header_len)| Some(input.get(header_len..).unwrap_or_default().to_vec()), - ); - - TagIteratorError::UnexpectedEOF { - tag_start: self.position, - tag_id, - tag_size: size.map(|(size, _)| size), - partial_data, - } - } -} diff --git a/crates/ebml-iterable/src/tag_iterator.rs b/crates/ebml-iterable/src/tag_iterator.rs deleted file mode 100644 index d35b5f517..000000000 --- a/crates/ebml-iterable/src/tag_iterator.rs +++ /dev/null @@ -1,597 +0,0 @@ -use std::collections::{HashSet, VecDeque}; -use std::io::Read; - -use super::errors::tag_iterator::{CorruptedFileError, TagIteratorError}; -use super::specs::{EbmlSpecification, EbmlTag, Master, PathPart, TagDataType}; -use crate::spec_util::validate_tag_path; -use crate::tag_iterator_util::EBMLSize::{Known, Unknown}; -use crate::tag_iterator_util::{AllowableErrors, EBMLSize, ProcessingTag, DEFAULT_BUFFER_LEN}; -use crate::tag_parse; - -const INVALID_TAG_ID_ERROR: u8 = 0x01; -const INVALID_HIERARCHY_ERROR: u8 = 0x02; -const OVERSIZED_CHILD_ERROR: u8 = 0x04; - -/// -/// Provides an iterator over EBML files (read from a source implementing the [`std::io::Read`] trait). Can be configured to read specific "Master" tags as complete objects rather than just emitting when they start and end. -/// -/// This is a generic struct that requires a specification implementing [`EbmlSpecification`] and [`EbmlTag`]. No specifications are included in this crate - you will need to either use another crate providing a spec (such as the Matroska spec implemented in the [webm-iterable](https://crates.io/crates/webm_iterable) or write your own spec if you want to iterate over a custom EBML file. The iterator outputs `TSpec` variants representing the type of tag (defined by the specification) and the accompanying tag data. "Master" tags (defined by the specification) usually will be read as `Start` and `End` variants, but the iterator can be configured to buffer Master tags into a `Full` variant using the `tags_to_buffer` parameter. -/// -/// Note: The [`Self::with_capacity()`] method can be used to construct a `TagIterator` with a specified default buffer size. This is only useful as a microoptimization to memory management if you know the maximum tag size of the file you're reading. -/// -/// ## Example -/// -/// ```no_run -/// use std::fs::File; -/// use ebml_iterable::TagIterator; -/// # -/// # use ebml_iterable::specs::{EbmlSpecification, TagDataType}; -/// # use ebml_iterable_specification::empty_spec::EmptySpec; -/// -/// # fn main() -> Result<(), Box> { -/// let file = File::open("my_ebml_file.ebml")?; -/// let mut my_iterator: TagIterator<_, EmptySpec> = TagIterator::new(file, &[]); -/// for tag in my_iterator { -/// println!("{:?}", tag?); -/// } -/// # Ok(()) -/// # } -/// ``` -/// -/// ## Errors -/// -/// The `Item` type for the associated [`Iterator`] implementation is a [`Result`], meaning each `next()` call has the potential to fail. This is because the source data is not parsed all at once - it is incrementally parsed as the iterator progresses. If the iterator runs into an error (such as corrupted data or an unexpected end-of-file), it needs to be propagated to the logic trying to read the tags. The different possible error states are enumerated in [`TagIteratorError`]. -/// -/// ## Panics -/// -/// The iterator can panic if `` is an internally inconsistent specification (i.e. it claims that a specific tag id has a specific data type but fails to produce a tag variant using data of that type). This won't happen if the specification being used was created using the [`#[ebml_specification]`](https://docs.rs/ebml-iterable-specification-derive/latest/ebml_iterable_specification_derive/attr.ebml_specification.html) attribute macro. -/// -pub struct TagIterator -where - TSpec: EbmlSpecification + EbmlTag + Clone, -{ - source: R, - tag_ids_to_buffer: HashSet, - allowed_errors: u8, - max_allowed_tag_size: Option, - - buffer: Box<[u8]>, - buffer_offset: Option, - buffered_byte_length: usize, - internal_buffer_position: usize, - tag_stack: Vec>, - emission_queue: VecDeque>, - last_emitted_tag_offset: usize, - has_determined_doc_path: bool, - - emit_master_end_when_eof: bool, -} - -impl TagIterator -where - TSpec: EbmlSpecification + EbmlTag + Clone, -{ - /// - /// Returns a new [`TagIterator`] instance. - /// - /// The `source` parameter must implement [`std::io::Read`]. The second argument, `tags_to_buffer`, specifies which "Master" tags should be read as [`Master::Full`]s rather than as [`Master::Start`] and [`Master::End`]s. Refer to the documentation on [`TagIterator`] for more explanation of how to use the returned instance. - /// - pub fn new(source: R, tags_to_buffer: &[TSpec]) -> Self { - TagIterator::with_capacity(source, tags_to_buffer, DEFAULT_BUFFER_LEN) - } - - /// - /// Returns a new [`TagIterator`] instance with the specified internal buffer capacity. - /// - /// This initializes the [`TagIterator`] with a specific byte capacity. The iterator will still reallocate if necessary. (Reallocation occurs if the iterator comes across a tag that should be output as a [`Master::Full`] and its size in bytes is greater than the iterator's current buffer capacity.) - /// - pub fn with_capacity(source: R, tags_to_buffer: &[TSpec], capacity: usize) -> Self { - let buffer = vec![0; capacity]; - - TagIterator { - source, - tag_ids_to_buffer: tags_to_buffer.iter().map(|tag| tag.get_id()).collect(), - allowed_errors: 0, - max_allowed_tag_size: Some(4 * usize::pow(1000, 3)), // 4GB - buffer: buffer.into_boxed_slice(), - buffered_byte_length: 0, - buffer_offset: None, - internal_buffer_position: 0, - tag_stack: Vec::new(), - emission_queue: VecDeque::new(), - last_emitted_tag_offset: 0, - has_determined_doc_path: false, - emit_master_end_when_eof: true, - } - } - - /// - /// Configures how strictly the iterator abides ``. - /// - /// By default (as of v0.5.0), the iterator assumes `` is complete and that any tags that do not conform to `` are due to corrupted file data. This method can be used to relax some of these checks so that fewer [`TagIteratorError::CorruptedFileData`] errors occur. - /// - /// # Important - /// - /// Relaxing these checks do not necessarily make the iterator more robust. If all errors are allowed, the iterator will assume any incoming tag id and tag data size are valid, and it will produce "RawTag"s containing binary contents for any tag ids not in ``. However, if the file truly has corrupted data, the "size" of these elements will likely be corrupt as well. This can typically result in requests for massive allocations, causing delays and eventual crashing. By eagerly returning errors (the default), applications can decide how to handle corrupted elements more quickly and efficiently. - /// - /// tldr; allow errors at your own risk - /// - /// > Note: TagIterators returned by [`Self::new()`] and [`Self::with_capacity()`] allow no errors by default. - /// - pub fn allow_errors(&mut self, errors: &[AllowableErrors]) { - self.allowed_errors = errors.iter().fold(0u8, |a, c| match c { - AllowableErrors::InvalidTagIds => a | INVALID_TAG_ID_ERROR, - AllowableErrors::HierarchyProblems => a | INVALID_HIERARCHY_ERROR, - AllowableErrors::OversizedTags => a | OVERSIZED_CHILD_ERROR, - }); - } - - /// - /// Configures the maximum size a tag is allowed to be before the iterator considers it invalid. - /// - /// By default (as of v0.6.1), the iterator will throw an [`CorruptedFileError::InvalidTagSize`] error if it comes across any tags that declare their data to be more than 4GB. This method can be used to change (and optionally remove) this behavior. Note that increasing this size can potentially result in massive allocations, causing delays and panics. - /// - pub fn set_max_allowable_tag_size(&mut self, size: Option) { - self.max_allowed_tag_size = size; - } - - /// - /// Instructs the iterator to attempt to recover after reaching corrupted file data. - /// - /// This method can be used to skip over corrupted sections of a read stream without recreating a new iterator. The iterator will seek forward from its current internal position until it reaches either a valid EBML tag id or EOF. After recovery, [`Iterator::next()`] *should* return an [`Ok`] result. - /// - pub fn try_recover(&mut self) -> Result<(), TagIteratorError> { - let original_position = self.current_offset(); - loop { - if !self.ensure_data_read(1)? { - return Err(TagIteratorError::UnexpectedEOF { - tag_start: self.current_offset(), - tag_id: None, - tag_size: None, - partial_data: None, - }); - } - - self.internal_buffer_position += 1; - if self.peek_valid_tag_header().is_ok() { - break; - } - } - - // As part of recovery, update internal tag stack sizes so that we don't get "oversized children" errors after skipping corrupted data - let diff = self.current_offset() - original_position; - for tag in self.tag_stack.iter_mut() { - if let EBMLSize::Known(size) = &tag.size { - tag.size = EBMLSize::Known(size + diff); - } - } - - Ok(()) - } - - /// - /// Consumes self and returns the underlying read stream. - /// - /// Note that any leftover tags in the internal emission queue are lost, and any data read into [`TagIterator`]'s internal buffer is dropped. Therefore, constructing a new [`TagIterator`] using the returned stream may lead to data loss unless it is rewound. - /// - pub fn into_inner(self) -> R { - self.source - } - - /// - /// Gets a mutable reference to the underlying read stream. - /// - /// It is inadvisable to directly read from the underlying stream. - /// - pub fn get_mut(&mut self) -> &mut R { - &mut self.source - } - - /// - /// Gets a reference to the underlying read stream. - /// - /// It is inadvisable to directly read from the underlying stream. - /// - pub fn get_ref(&self) -> &R { - &self.source - } - - /// - /// Returns the byte offset of the last emitted tag. - /// - /// This function returns a byte index specifying the start of the last emitted tag in the context of the [`TagIterator`]'s source read stream. This value is *not guaranteed to always increase as the file is read*. Whenever the iterator emits a [`Master::End`] variant, [`Self::last_emitted_tag_offset()`] will reflect the start index of the "Master" tag, which will be before previous values that were obtainable when any children of the master were emitted. - /// - pub fn last_emitted_tag_offset(&self) -> usize { - self.last_emitted_tag_offset - } - - /// - /// Control whether the iterator should emit closing tags when it reaches EOF. - /// - /// By default, the iterator will emit [`Master::End`] items for all currently open tags when it reaches the end of the file. You may override this behavior by passing `false` to this method. - /// - /// This is recommended if you supply a [`std::io::Read`] source that can supply more data after reaching EOF, as in some streaming scenarios. - /// - pub fn emit_master_end_when_eof(&mut self, emit: bool) { - self.emit_master_end_when_eof = emit; - } - - #[inline(always)] - fn current_offset(&self) -> usize { - self.buffer_offset.unwrap_or(0) + self.internal_buffer_position - } - - fn private_read(&mut self, internal_buffer_start: usize) -> Result { - let bytes_read = self - .source - .read(&mut self.buffer[internal_buffer_start..]) - .map_err(|source| TagIteratorError::ReadError { source })?; - if bytes_read == 0 { - Ok(false) - } else { - self.buffered_byte_length += bytes_read; - Ok(true) - } - } - - fn ensure_capacity(&mut self, required_capacity: usize) { - if required_capacity > self.buffer.len() { - let mut new_buffer = Vec::from(&self.buffer[..]); - new_buffer.resize(required_capacity, 0); - self.buffer = new_buffer.into_boxed_slice(); - } - } - - fn ensure_data_read(&mut self, length: usize) -> Result { - if self.internal_buffer_position + length <= self.buffered_byte_length { - return Ok(true); - } - - if self.buffer_offset.is_none() { - if !self.private_read(0)? { - return Ok(false); - } - self.buffer_offset = Some(0); - self.internal_buffer_position = 0; - } else { - while self.internal_buffer_position + length > self.buffered_byte_length { - self.buffer - .copy_within(self.internal_buffer_position..self.buffered_byte_length, 0); - self.buffered_byte_length -= self.internal_buffer_position; - self.buffer_offset = Some(self.current_offset()); - self.internal_buffer_position = 0; - if !self.private_read(self.buffered_byte_length)? { - return Ok(false); - } - } - } - Ok(true) - } - - #[inline] - fn peek_valid_tag_header(&mut self) -> Result<(u64, Option, EBMLSize, usize), TagIteratorError> { - self.ensure_data_read(16)?; - let current_offset = self.current_offset(); - let available = &self.buffer[self.internal_buffer_position..self.buffered_byte_length]; - let header = - tag_parse::read_header::(available, current_offset)?.ok_or(TagIteratorError::UnexpectedEOF { - tag_start: current_offset, - tag_id: None, - tag_size: None, - partial_data: None, - })?; - let tag_id = header.id; - let spec_tag_type = header.data_type; - let size = header.size; - let header_len = header.len; - - if (self.allowed_errors & INVALID_TAG_ID_ERROR == 0) && spec_tag_type.is_none() { - return Err(TagIteratorError::CorruptedFileData(CorruptedFileError::InvalidTagId { - tag_id, - position: self.current_offset(), - })); - } - - if (self.allowed_errors & INVALID_HIERARCHY_ERROR == 0) && spec_tag_type.is_some() { - // Do not run check for raw tags ^^^^^^^^^^^^^^^^^^^^^^^ - if !self.has_determined_doc_path { - //Trust that the first tag in the stream is valid (like if the read stream was seeked to this location) - let path = ::get_path_by_id(tag_id); - if path.iter().all(|p| matches!(p, PathPart::Id(_))) { - //We only know the current path if we read a tag that is non-global - self.tag_stack = path.iter().map(|id| { - match id { - PathPart::Id(id) => { - ProcessingTag { - tag: ::get_master_tag(*id, Master::Start).unwrap_or_else(|| panic!("Bad specification implementation: Tag id 0x{:x?} type was in path, but could not get master tag!", id)), - size: EBMLSize::Unknown, - tag_start: 0, - data_start: 0, - } - }, - PathPart::Global(_) => unreachable!() - } - }).collect(); - self.has_determined_doc_path = true; - } - } - if self.has_determined_doc_path && !self.validate_tag_path(tag_id) { - return Err(TagIteratorError::CorruptedFileData( - CorruptedFileError::HierarchyError { - found_tag_id: tag_id, - current_parent_id: self.tag_stack.last().map(|tag| tag.tag.get_id()), - }, - )); - } - } - - if (self.allowed_errors & OVERSIZED_CHILD_ERROR == 0) - && size.is_known() - && self.is_invalid_tag_size(header_len + size.value()) - { - return Err(TagIteratorError::CorruptedFileData( - CorruptedFileError::OversizedChildElement { - position: self.current_offset(), - tag_id, - size: size.value(), - }, - )); - } - - if let Some(max_size) = self.max_allowed_tag_size { - if size.is_known() && size.value() > max_size { - return Err(TagIteratorError::CorruptedFileData( - CorruptedFileError::InvalidTagSize { - position: self.current_offset(), - tag_id, - size: size.value(), - }, - )); - } - } - - Ok((tag_id, spec_tag_type, size, header_len)) - } - - #[inline(always)] - fn read_valid_tag_header(&mut self) -> Result<(u64, Option, EBMLSize), TagIteratorError> { - let (tag_id, spec_tag_type, size, header_len) = self.peek_valid_tag_header()?; - - self.internal_buffer_position += header_len; - Ok((tag_id, spec_tag_type, size)) - } - - fn read_tag_data(&mut self, size: usize) -> Result, TagIteratorError> { - self.ensure_capacity(size); - if !self.ensure_data_read(size)? { - return Ok(None); - } - - self.internal_buffer_position += size; - Ok(Some( - &self.buffer[(self.internal_buffer_position - size)..self.internal_buffer_position], - )) - } - - fn read_tag(&mut self) -> Result, TagIteratorError> { - let tag_start = self.current_offset(); - - let (tag_id, spec_tag_type, size) = self.read_valid_tag_header()?; - - let data_start = self.current_offset(); - let raw_data = if matches!(spec_tag_type, Some(TagDataType::Master)) { - &[] - } else if let Known(size) = size { - if let Some(data) = self.read_tag_data(size)? { - data - } else { - return Err(TagIteratorError::UnexpectedEOF { - tag_start, - tag_id: Some(tag_id), - tag_size: Some(size), - partial_data: Some(self.buffer[self.internal_buffer_position..].to_vec()), - }); - } - } else { - return Err(TagIteratorError::CorruptedFileData( - CorruptedFileError::InvalidTagData { - tag_id, - position: tag_start, - }, - )); - }; - - let tag = if matches!(spec_tag_type, Some(TagDataType::Master)) { - TSpec::get_master_tag(tag_id, Master::Start).unwrap_or_else(|| { - panic!( - "Bad specification implementation: Tag id 0x{:x?} type was master, but could not get tag!", - tag_id - ) - }) - } else { - tag_parse::read_data_tag(tag_id, spec_tag_type, raw_data)? - }; - - Ok(ProcessingTag { - tag, - size, - tag_start, - data_start, - }) - } - - fn read_tag_checked(&mut self) -> Option, TagIteratorError>> { - if self.internal_buffer_position == self.buffered_byte_length { - //If we've already consumed the entire internal buffer - //ensure there is nothing else in the data source before returning `None` - let read_result = self.ensure_data_read(1); - match read_result { - Err(err) => return Some(Err(err)), - Ok(data_remaining) => { - if !data_remaining { - return None; - } - } - } - } - - if self.internal_buffer_position > self.buffered_byte_length { - panic!("read position exceeded buffer length"); - } - - Some(self.read_tag()) - } - - fn read_next(&mut self) { - //If we have reached the known end of any open master tags, queue that tag and all children to emit ends - let ended_tag_index = self - .tag_stack - .iter() - .position(|tag| matches!(tag.size, Known(size) if self.current_offset() >= tag.data_start + size)); - if let Some(index) = ended_tag_index { - self.emission_queue - .extend(self.tag_stack.drain(index..).map(|t| Ok((t.tag, t.tag_start))).rev()); - } - - if let Some(next_read) = self.read_tag_checked() { - if let Ok(next_tag) = &next_read { - while matches!(self.tag_stack.last(), Some(open_tag) if open_tag.size == Unknown) { - let open_tag = self.tag_stack.last().unwrap(); - let previous_tag_ended = open_tag.is_ended_by(next_tag.tag.get_id()); - - if previous_tag_ended { - let t = self.tag_stack.pop().unwrap(); - self.emission_queue.push_back(Ok((t.tag, t.tag_start))); - } else { - break; - } - } - - if let Some(Master::Start) = next_tag.tag.as_master() { - let tag_id = next_tag.tag.get_id(); - - self.tag_stack.push(ProcessingTag { - tag: TSpec::get_master_tag(tag_id, Master::End).unwrap(), - size: next_tag.size, - tag_start: next_tag.tag_start, - data_start: next_tag.data_start, - }); - - if self.tag_ids_to_buffer.contains(&tag_id) { - self.buffer_master(tag_id); - return; - } - } - } - - self.emission_queue.push_back(next_read.map(|r| (r.tag, r.tag_start))); - } else if self.emit_master_end_when_eof { - while let Some(tag) = self.tag_stack.pop() { - self.emission_queue.push_back(Ok((tag.tag, tag.tag_start))); - } - } - } - - fn buffer_master(&mut self, tag_id: u64) { - let tag_start = self.current_offset(); - let pre_queue_len = self.emission_queue.len(); - - let mut position = pre_queue_len; - 'endTagSearch: loop { - if position >= self.emission_queue.len() { - self.read_next(); - - if position >= self.emission_queue.len() { - self.emission_queue.push_back(Err(TagIteratorError::UnexpectedEOF { - tag_start, - tag_id: Some(tag_id), - tag_size: None, - partial_data: None, - })); - return; - } - } - - while position < self.emission_queue.len() { - if let Some(r) = self.emission_queue.get(position) { - match r { - Err(_) => break 'endTagSearch, - Ok(t) => { - if t.0.get_id() == tag_id && matches!(t.0.as_master(), Some(Master::End)) { - break 'endTagSearch; - } - } - } - } - position += 1; - } - } - - let mut children = self.emission_queue.split_off(pre_queue_len); - let split_to = position - pre_queue_len; - if children.get(split_to).unwrap().is_ok() { - let remaining = children.split_off(split_to).into_iter().skip(1); - let full_tag = Self::roll_up_children(tag_id, children.into_iter().map(|c| c.unwrap().0).collect()); - self.emission_queue.push_back(Ok((full_tag, tag_start))); - self.emission_queue.extend(remaining); - } else { - self.emission_queue.extend(children.drain(split_to..).take(1)); - } - } - - fn roll_up_children(tag_id: u64, children: Vec) -> TSpec { - let mut rolled_children = Vec::new(); - - let mut iter = children.into_iter(); - while let Some(child) = iter.next() { - if let Some(Master::Start) = child.as_master() { - let child_id = child.get_id(); - let subchildren = iter - .by_ref() - .take_while(|c| !matches!(c.as_master(), Some(Master::End)) || c.get_id() != child_id) - .collect(); - rolled_children.push(Self::roll_up_children(child_id, subchildren)); - } else { - rolled_children.push(child); - } - } - - TSpec::get_master_tag(tag_id, Master::Full(rolled_children)).unwrap_or_else(|| { - panic!( - "Bad specification implementation: Tag id 0x{:x?} type was master, but could not get tag!", - tag_id - ) - }) - } - - #[inline(always)] - fn validate_tag_path(&self, tag_id: u64) -> bool { - validate_tag_path::(tag_id, self.tag_stack.iter().map(|p| (p.tag.get_id(), p.size, 0))) - } - - #[inline(always)] - fn is_invalid_tag_size(&self, size: usize) -> bool { - self.tag_stack - .iter() - .filter(|p| p.size.is_known()) - .any(|t| (t.data_start + t.size.value()) < (self.current_offset() + size)) - } -} - -impl Iterator for TagIterator -where - TSpec: EbmlSpecification + EbmlTag + Clone, -{ - type Item = Result; - - fn next(&mut self) -> Option { - if self.emission_queue.is_empty() { - self.read_next(); - } - let next_item = self.emission_queue.pop_front(); - if let Some(Ok(ref tuple)) = next_item { - self.last_emitted_tag_offset = tuple.1; - } - next_item.map(|r| r.map(|t| t.0)) - } -} diff --git a/crates/ebml-iterable/src/tag_iterator_async.rs b/crates/ebml-iterable/src/tag_iterator_async.rs deleted file mode 100644 index da33d4314..000000000 --- a/crates/ebml-iterable/src/tag_iterator_async.rs +++ /dev/null @@ -1,191 +0,0 @@ -use std::io::ErrorKind; -use std::iter::repeat; -use std::mem; -use ebml_iterable_specification::{EbmlSpecification, EbmlTag, Master, TagDataType}; -use futures::{AsyncRead, AsyncReadExt, Stream}; -use crate::error::{TagIteratorError, ToolError}; -use crate::errors::tag_iterator::CorruptedFileError; -use crate::tag_iterator_util::{EBMLSize, ProcessingTag}; -use crate::tag_iterator_util::EBMLSize::Known; -use crate::tools; - -/// -/// This Can be transformed into a [`Stream`] using [`into_stream`][TagIteratorAsync::into_stream], or consumed directly by calling [`.next().await`] in a loop. -/// -/// The struct can be created with the [`new()`][TagIteratorAsync::new] function on any source that implements the [`futures::AsyncRead`] trait. -/// -pub struct TagIteratorAsync - where - TSpec: EbmlSpecification + EbmlTag + Clone -{ - read: R, - buf: Vec, - offset: usize, - tag_stack: Vec> -} - -impl TagIteratorAsync - where - TSpec: EbmlSpecification + EbmlTag + Clone -{ - - pub fn new(read: R) -> Self { - Self { - read, - buf: Default::default(), - offset: 0, - tag_stack: Default::default() - } - } - - fn current_offset(&self) -> usize { - self.offset - } - - fn advance(&mut self, length: usize) { - self.offset += length; - self.buf.drain(0..length); - } - - fn advance_get(&mut self, length: usize) -> Vec { - self.offset += length; - let upper = self.buf.split_off(length); - mem::replace(&mut self.buf, upper) - } - - async fn ensure_data_read(&mut self, len: usize) -> Result { - let size = self.buf.len(); - if size < len { - let remaining = len - size; - self.buf.extend(repeat(0).take(remaining)); - if let Err(source) = self.read.read_exact(&mut self.buf[size..]).await { - return match source.kind() { - ErrorKind::UnexpectedEof => { - Ok(false) - } - _ => Err(TagIteratorError::ReadError { source }) - } - } - } - Ok(true) - } - - async fn read_tag_id(&mut self) -> Result { - self.ensure_data_read(8).await?; - match tools::read_vint(&self.buf).unwrap_or(Some((0, 1))) { - Some((value, length)) => { - self.advance(length); - Ok(value + (1 << (7 * length))) - }, - None => Err(TagIteratorError::UnexpectedEOF{ tag_start: self.current_offset(), tag_id: None, tag_size: None, partial_data: None }), - } - } - - async fn read_tag_size(&mut self) -> Result { - self.ensure_data_read(8).await?; - match tools::read_vint(&self.buf).or(Err(TagIteratorError::CorruptedFileData(CorruptedFileError::InvalidTagData { tag_id: 0, position: self.current_offset() })))? { - Some((value, length)) => { - self.advance(length); - Ok(EBMLSize::new(value, length)) - }, - None => Err(TagIteratorError::UnexpectedEOF{ tag_start: self.current_offset(), tag_id: None, tag_size: None, partial_data: None }), - } - } - - async fn read_tag_data(&mut self, size: usize) -> Result, TagIteratorError> { - if !self.ensure_data_read(size).await? { - return Err(TagIteratorError::UnexpectedEOF{ tag_start: self.current_offset(), tag_id: None, tag_size: None, partial_data: None }); - } - Ok(self.advance_get(size)) - } - - async fn read_tag(&mut self) -> Result { - let tag_id = self.read_tag_id().await?; - let spec_tag_type = TSpec::get_tag_data_type(tag_id); - let size = self.read_tag_size().await?; - let current_offset = self.current_offset(); - - let is_master = matches!(spec_tag_type, Some(TagDataType::Master)); - if is_master { - self.tag_stack.push(ProcessingTag { - tag: TSpec::get_master_tag(tag_id, Master::End).unwrap_or_else(|| panic!("Bad specification implementation: Tag id {} type was master, but could not get tag!", tag_id)), - size, - data_start: current_offset, - tag_start: 0, //not implemented here - }); - Ok(TSpec::get_master_tag(tag_id, Master::Start).unwrap_or_else(|| panic!("Bad specification implementation: Tag id {} type was master, but could not get tag!", tag_id))) - } else { - let size = if let Known(size) = size { - size - } else { - return Err(TagIteratorError::CorruptedFileData(CorruptedFileError::InvalidTagData { tag_id, position: current_offset })); - }; - - let raw_data = self.read_tag_data(size).await?; - let tag = match spec_tag_type { - Some(TagDataType::Master) => { unreachable!("Master should have been handled before querying data") }, - Some(TagDataType::UnsignedInt) => { - let val = tools::arr_to_u64(&raw_data).map_err(|e| TagIteratorError::CorruptedTagData{ tag_id, problem: e })?; - TSpec::get_unsigned_int_tag(tag_id, val).unwrap_or_else(|| panic!("Bad specification implementation: Tag id {} type was unsigned int, but could not get tag!", tag_id)) - }, - Some(TagDataType::Integer) => { - let val = tools::arr_to_i64(&raw_data).map_err(|e| TagIteratorError::CorruptedTagData{ tag_id, problem: e })?; - TSpec::get_signed_int_tag(tag_id, val).unwrap_or_else(|| panic!("Bad specification implementation: Tag id {} type was integer, but could not get tag!", tag_id)) - }, - Some(TagDataType::Utf8) => { - let val = String::from_utf8(raw_data.to_vec()).map_err(|e| TagIteratorError::CorruptedTagData{ tag_id, problem: ToolError::FromUtf8Error(raw_data, e) })?; - TSpec::get_utf8_tag(tag_id, val).unwrap_or_else(|| panic!("Bad specification implementation: Tag id {} type was utf8, but could not get tag!", tag_id)) - }, - Some(TagDataType::Binary) | None => { - TSpec::get_binary_tag(tag_id, &raw_data).unwrap_or_else(|| TSpec::get_raw_tag(tag_id, &raw_data)) - }, - Some(TagDataType::Float) => { - let val = tools::arr_to_f64(&raw_data).map_err(|e| TagIteratorError::CorruptedTagData{ tag_id, problem: e })?; - TSpec::get_float_tag(tag_id, val).unwrap_or_else(|| panic!("Bad specification implementation: Tag id {} type was float, but could not get tag!", tag_id)) - }, - }; - - match self.tag_stack.last() { - None => Ok(tag), - Some(previous_tag) => { - let previous_tag_ended = previous_tag.is_ended_by(tag_id); - - if previous_tag_ended { - Ok(mem::replace(self.tag_stack.last_mut().unwrap(), ProcessingTag { tag, size: Known(size), data_start: current_offset, tag_start: 0 }).into_inner()) - } else { - Ok(tag) - } - } - } - } - } - - /// can be consumed - pub async fn next(&mut self) -> Option> { - if let Some(tag) = self.tag_stack.pop() { - if let Known(size) = tag.size { - if self.current_offset() >= tag.data_start + size { - return Some(Ok(tag.tag)); - } - } - self.tag_stack.push(tag); - } - - match self.ensure_data_read(1).await { - Err(err) => return Some(Err(err)), - Ok(data_remaining) => { - if !data_remaining { - return self.tag_stack.pop().map(|tag| Ok(tag.into_inner())); - } - } - } - Some(self.read_tag().await) - } - - pub fn into_stream(self) -> impl Stream> { - futures::stream::unfold(self, |mut read| async { - let next = read.next().await; - next.map(move |it| (it, read)) - }) - } -} diff --git a/crates/ebml-iterable/src/tag_iterator_util.rs b/crates/ebml-iterable/src/tag_iterator_util.rs deleted file mode 100644 index 329d542f4..000000000 --- a/crates/ebml-iterable/src/tag_iterator_util.rs +++ /dev/null @@ -1,125 +0,0 @@ -use std::convert::TryInto; - -use ebml_iterable_specification::{EbmlSpecification, EbmlTag}; - -use crate::spec_util::is_ended_by; -use crate::tag_iterator_util::EBMLSize::{Known, Unknown}; - -#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] -pub enum EBMLSize { - Known(usize), - Unknown, -} - -impl EBMLSize { - pub fn new(size: u64, vint_length: usize) -> Self { - match vint_length { - 1 => { - if size == ((1 << (7)) - 1) { - return Unknown; - } - } - 2 => { - if size == ((1 << (7 * 2)) - 1) { - return Unknown; - } - } - 3 => { - if size == ((1 << (7 * 3)) - 1) { - return Unknown; - } - } - 4 => { - if size == ((1 << (7 * 4)) - 1) { - return Unknown; - } - } - 5 => { - if size == ((1 << (7 * 5)) - 1) { - return Unknown; - } - } - 6 => { - if size == ((1 << (7 * 6)) - 1) { - return Unknown; - } - } - 7 => { - if size == ((1 << (7 * 7)) - 1) { - return Unknown; - } - } - 8 => { - if size == ((1 << (7 * 8)) - 1) { - return Unknown; - } - } - _ => {} - } - - match size.try_into() { - Ok(value) => Known(value), - Err(_) => Unknown, - } - } - - #[inline(always)] - pub fn is_known(&self) -> bool { - matches!(&self, &EBMLSize::Known(_)) - } - - /// - /// # Panics - /// - /// Panics if the current variant is not EBMLSize::Known - /// - #[inline(always)] - pub fn value(&self) -> usize { - match &self { - EBMLSize::Known(val) => *val, - _ => panic!("Called EBMLSize::value() on an unknown size!"), - } - } -} - -#[derive(Copy, Clone, Debug)] -pub struct ProcessingTag -where - TSpec: EbmlSpecification + EbmlTag + Clone, -{ - pub tag: TSpec, - pub size: EBMLSize, - pub tag_start: usize, - pub data_start: usize, -} - -impl ProcessingTag -where - TSpec: EbmlSpecification + EbmlTag + Clone, -{ - pub fn is_ended_by(&self, id: u64) -> bool { - is_ended_by::(self.tag.get_id(), id) - } -} - -pub const DEFAULT_BUFFER_LEN: usize = 1024 * 64; - -/// -/// Used to relax rules on how strictly a [`TagIterator`](crate::TagIterator) should validate the read stream. -/// -pub enum AllowableErrors { - /// - /// Causes the [`TagIterator`](crate::TagIterator) to produce "RawTag" binary variants for any unknown tag ids rather than throwing an error. - /// - InvalidTagIds, - - /// - /// Causes the [`TagIterator`](crate::TagIterator) to emit tags even if they appear outside of their defined parent element. - /// - HierarchyProblems, - - /// - /// Causes the [`TagIterator`](crate::TagIterator) to emit tags even if they exceed the length of a parent element. - /// - OversizedTags, -} diff --git a/crates/ebml-iterable/src/tag_parse.rs b/crates/ebml-iterable/src/tag_parse.rs deleted file mode 100644 index 5e1c5b8fe..000000000 --- a/crates/ebml-iterable/src/tag_parse.rs +++ /dev/null @@ -1,110 +0,0 @@ -use crate::errors::tag_iterator::{CorruptedFileError, TagIteratorError}; -use crate::errors::tool::ToolError; -use crate::specs::{EbmlSpecification, EbmlTag, TagDataType}; -use crate::tag_iterator_util::EBMLSize; -use crate::tools; - -pub(crate) struct TagHeader { - pub id: u64, - pub data_type: Option, - pub size: EBMLSize, - pub len: usize, -} - -pub(crate) fn read_header(input: &[u8], position: usize) -> Result, TagIteratorError> -where - TSpec: EbmlSpecification + EbmlTag + Clone, -{ - let Some(first) = input.first().copied() else { - return Ok(None); - }; - let id_len = if first == 0 { 1 } else { 8 - first.ilog2() as usize }; - if input.len() < id_len { - return Ok(None); - } - - let id = input[..id_len] - .iter() - .fold(0u64, |value, byte| (value << 8) + u64::from(*byte)); - let Some((size, size_len)) = tools::read_vint(&input[id_len..]).map_err(|_| { - TagIteratorError::CorruptedFileData(CorruptedFileError::InvalidTagData { tag_id: id, position }) - })? - else { - return Ok(None); - }; - let data_type = TSpec::get_tag_data_type(id); - - if matches!( - data_type, - Some(TagDataType::UnsignedInt | TagDataType::Integer | TagDataType::Float) - ) && size > 8 - { - return Err(TagIteratorError::CorruptedFileData( - CorruptedFileError::InvalidTagData { tag_id: id, position }, - )); - } - - Ok(Some(TagHeader { - id, - data_type, - size: EBMLSize::new(size, size_len), - len: id_len + size_len, - })) -} - -pub(crate) fn read_data_tag( - id: u64, - data_type: Option, - raw_data: &[u8], -) -> Result -where - TSpec: EbmlSpecification + EbmlTag + Clone, -{ - let tag = match data_type { - Some(TagDataType::Master) => unreachable!(), - Some(TagDataType::UnsignedInt) => { - let value = if raw_data.is_empty() { - 0 - } else { - tools::arr_to_u64(raw_data) - .map_err(|problem| TagIteratorError::CorruptedTagData { tag_id: id, problem })? - }; - TSpec::get_unsigned_int_tag(id, value) - } - Some(TagDataType::Integer) => { - let value = if raw_data.is_empty() { - 0 - } else { - tools::arr_to_i64(raw_data) - .map_err(|problem| TagIteratorError::CorruptedTagData { tag_id: id, problem })? - }; - TSpec::get_signed_int_tag(id, value) - } - Some(TagDataType::Utf8) => { - let value = String::from_utf8(raw_data.to_vec()).map_err(|error| TagIteratorError::CorruptedTagData { - tag_id: id, - problem: ToolError::FromUtf8Error(raw_data.to_vec(), error), - })?; - TSpec::get_utf8_tag(id, value) - } - Some(TagDataType::Binary) => TSpec::get_binary_tag(id, raw_data), - Some(TagDataType::Float) => { - let value = if raw_data.is_empty() { - 0.0 - } else { - tools::arr_to_f64(raw_data) - .map_err(|problem| TagIteratorError::CorruptedTagData { tag_id: id, problem })? - }; - TSpec::get_float_tag(id, value) - } - None => return Ok(TSpec::get_raw_tag(id, raw_data)), - } - .unwrap_or_else(|| { - panic!( - "Bad specification implementation: Tag id 0x{:x?} had an incompatible data type!", - id - ) - }); - - Ok(tag) -} diff --git a/crates/ebml-iterable/src/tag_writer.rs b/crates/ebml-iterable/src/tag_writer.rs deleted file mode 100644 index 35e3ad537..000000000 --- a/crates/ebml-iterable/src/tag_writer.rs +++ /dev/null @@ -1,760 +0,0 @@ -use std::convert::{TryFrom, TryInto}; -use std::io::Write; - -use super::errors::tag_writer::TagWriterError; -use super::specs::{EbmlSpecification, EbmlTag, Master, TagDataType}; -use super::tag_iterator_util::EBMLSize::{self, Known, Unknown}; -use super::tools::{is_vint, Vint}; -use crate::errors::tool::ToolError; -use crate::spec_util::validate_tag_path; - -/// -/// Options that can be passed to the writer to customize written output -/// -pub struct WriteOptions { - size_byte_length: Option, - unknown_sized_element: bool, -} - -impl WriteOptions { - /// - /// Specifies the byte length for the element's "size" - /// - /// This function generates [`WriteOptions`] that will force the Element Data Size to be a specific number of bytes for the written tag. - /// - /// ## Panics - /// - /// This method asserts that `len` is within 1-8 (inclusive). Values outside this range will cause a panic. - /// - pub fn set_size_byte_count(len: usize) -> Self { - assert!( - len > 0 && len < 9, - "Size byte count for written vints must be within 1-8 (inclusive)" - ); - Self { - size_byte_length: Some(len), - unknown_sized_element: false, - } - } - - /// - /// Specifies that the element has an Unknown Data Size. - /// - /// The [`WriteOptions`] generated by this function allow you to start a tag that doesn't have a known size. Useful for streaming, or when the data is expected to be too large to fit into memory. This should *only* be used with Master type tags. - /// - pub fn is_unknown_sized_element() -> Self { - Self { - size_byte_length: None, - unknown_sized_element: true, - } - } -} - -/// -/// Provides a tool to write EBML files based on Tags. Writes to a destination that implements [`std::io::Write`]. -/// -/// Unlike the [`TagIterator`][`super::TagIterator`], this does not require a specification to write data. This writer provides the [`write_raw()`](#method.write_raw) method which can be used to write data that is outside of any specification. The regular [`write()`](#method.write) method can be used to write any `TSpec` objects regardless of whether they came from a [`TagIterator`][`super::TagIterator`] or not. -/// -pub struct TagWriter { - dest: W, - open_tags: Vec<(u64, EBMLSize, usize)>, - working_buffer: Vec, -} - -impl TagWriter { - /// - /// Returns a new [`TagWriter`] instance. - /// - /// The `dest` parameter can be anything that implements [`std::io::Write`]. - /// - pub fn new(dest: W) -> Self { - TagWriter { - dest, - open_tags: Vec::new(), - working_buffer: Vec::new(), - } - } - - /// - /// Consumes self and returns the underlying write stream. - /// - /// Any incomplete tags are written out before returning the stream. - /// - pub fn into_inner(mut self) -> Result { - self.flush()?; - Ok(self.dest) - } - - /// - /// Gets a mutable reference to the underlying write stream. - /// - pub fn get_mut(&mut self) -> &mut W { - &mut self.dest - } - - /// - /// Gets a reference to the underlying write stream. - /// - pub fn get_ref(&self) -> &W { - &self.dest - } - - fn start_tag(&mut self, id: u64, size_length: usize) { - self.open_tags.push((id, Known(self.working_buffer.len()), size_length)); - } - - fn start_unknown_size_tag(&mut self, id: u64) { - self.working_buffer - .extend(id.to_be_bytes().iter().skip_while(|&v| *v == 0u8)); - self.working_buffer.extend_from_slice(&(u64::MAX >> 7).to_be_bytes()); - self.open_tags.push((id, Unknown, 0)); - } - - fn end_tag(&mut self, id: u64) -> Result<(), TagWriterError> { - match self.open_tags.pop() { - Some(open_tag) => { - if open_tag.0 == id { - if let Known(start) = open_tag.1 { - let size: u64 = self - .working_buffer - .len() - .checked_sub(start) - .expect("overflow subtracting tag size from working buffer length") - .try_into() - .expect("couldn't convert usize to u64"); - - match open_tag.2 { - 1 => { - let size_vint = size - .as_vint_with_length::<1>() - .map_err(|e| TagWriterError::TagSizeError(e.to_string()))?; - self.working_buffer.splice( - start..start, - open_tag - .0 - .to_be_bytes() - .iter() - .skip_while(|&v| *v == 0u8) - .chain(size_vint.iter()) - .copied(), - ); - } - 2 => { - let size_vint = size - .as_vint_with_length::<2>() - .map_err(|e| TagWriterError::TagSizeError(e.to_string()))?; - self.working_buffer.splice( - start..start, - open_tag - .0 - .to_be_bytes() - .iter() - .skip_while(|&v| *v == 0u8) - .chain(size_vint.iter()) - .copied(), - ); - } - 3 => { - let size_vint = size - .as_vint_with_length::<3>() - .map_err(|e| TagWriterError::TagSizeError(e.to_string()))?; - self.working_buffer.splice( - start..start, - open_tag - .0 - .to_be_bytes() - .iter() - .skip_while(|&v| *v == 0u8) - .chain(size_vint.iter()) - .copied(), - ); - } - 4 => { - let size_vint = size - .as_vint_with_length::<4>() - .map_err(|e| TagWriterError::TagSizeError(e.to_string()))?; - self.working_buffer.splice( - start..start, - open_tag - .0 - .to_be_bytes() - .iter() - .skip_while(|&v| *v == 0u8) - .chain(size_vint.iter()) - .copied(), - ); - } - 5 => { - let size_vint = size - .as_vint_with_length::<5>() - .map_err(|e| TagWriterError::TagSizeError(e.to_string()))?; - self.working_buffer.splice( - start..start, - open_tag - .0 - .to_be_bytes() - .iter() - .skip_while(|&v| *v == 0u8) - .chain(size_vint.iter()) - .copied(), - ); - } - 6 => { - let size_vint = size - .as_vint_with_length::<6>() - .map_err(|e| TagWriterError::TagSizeError(e.to_string()))?; - self.working_buffer.splice( - start..start, - open_tag - .0 - .to_be_bytes() - .iter() - .skip_while(|&v| *v == 0u8) - .chain(size_vint.iter()) - .copied(), - ); - } - 7 => { - let size_vint = size - .as_vint_with_length::<7>() - .map_err(|e| TagWriterError::TagSizeError(e.to_string()))?; - self.working_buffer.splice( - start..start, - open_tag - .0 - .to_be_bytes() - .iter() - .skip_while(|&v| *v == 0u8) - .chain(size_vint.iter()) - .copied(), - ); - } - 8 => { - let size_vint = size - .as_vint_with_length::<8>() - .map_err(|e| TagWriterError::TagSizeError(e.to_string()))?; - self.working_buffer.splice( - start..start, - open_tag - .0 - .to_be_bytes() - .iter() - .skip_while(|&v| *v == 0u8) - .chain(size_vint.iter()) - .copied(), - ); - } - _ => { - let size_vint = size - .as_vint() - .map_err(|e| TagWriterError::TagSizeError(e.to_string()))?; - self.working_buffer.splice( - start..start, - open_tag - .0 - .to_be_bytes() - .iter() - .skip_while(|&v| *v == 0u8) - .chain(size_vint.iter()) - .copied(), - ); - } - }; - } - Ok(()) - } else { - Err(TagWriterError::UnexpectedClosingTag { - tag_id: id, - expected_id: Some(open_tag.0), - }) - } - } - None => Err(TagWriterError::UnexpectedClosingTag { - tag_id: id, - expected_id: None, - }), - } - } - - fn private_flush(&mut self) -> Result<(), TagWriterError> { - self.dest - .write_all(self.working_buffer.drain(..).as_slice()) - .map_err(|source| TagWriterError::WriteError { source })?; - self.dest - .flush() - .map_err(|source| TagWriterError::WriteError { source }) - } - - fn write_unsigned_int_tag(&mut self, id: u64, data: &u64) -> Result<(), TagWriterError> { - self.working_buffer - .extend(id.to_be_bytes().iter().skip_while(|&v| *v == 0u8)); - let data = *data; - - u8::try_from(data) - .map(|n| { - if SIZE_LENGTH == 0 { - self.working_buffer.push(0x81); // vint representation of "1" - self.working_buffer.extend_from_slice(&n.to_be_bytes()); - } else { - self.working_buffer - .extend_from_slice(&1u8.as_vint_with_length::()?); - self.working_buffer.extend_from_slice(&n.to_be_bytes()); - } - Ok(()) - }) - .or_else(|_| { - u16::try_from(data).map(|n| { - if SIZE_LENGTH == 0 { - self.working_buffer.push(0x82); // vint representation of "2" - self.working_buffer.extend_from_slice(&n.to_be_bytes()); - } else { - self.working_buffer - .extend_from_slice(&2u8.as_vint_with_length::()?); - self.working_buffer.extend_from_slice(&n.to_be_bytes()); - } - Ok(()) - }) - }) - .or_else(|_| { - u32::try_from(data).map(|n| { - if SIZE_LENGTH == 0 { - self.working_buffer.push(0x84); // vint representation of "4" - self.working_buffer.extend_from_slice(&n.to_be_bytes()); - } else { - self.working_buffer - .extend_from_slice(&4u8.as_vint_with_length::()?); - self.working_buffer.extend_from_slice(&n.to_be_bytes()); - } - Ok(()) - }) - }) - .unwrap_or_else(|_| { - if SIZE_LENGTH == 0 { - self.working_buffer.push(0x88); // vint representation of "8" - self.working_buffer.extend_from_slice(&data.to_be_bytes()); - } else { - self.working_buffer - .extend_from_slice(&8u8.as_vint_with_length::()?); - self.working_buffer.extend_from_slice(&data.to_be_bytes()); - } - Ok(()) - }) - .map_err(|err: ToolError| TagWriterError::TagSizeError(err.to_string())) - } - - fn write_signed_int_tag(&mut self, id: u64, data: &i64) -> Result<(), TagWriterError> { - self.working_buffer - .extend(id.to_be_bytes().iter().skip_while(|&v| *v == 0u8)); - let data = *data; - i8::try_from(data) - .map(|n| { - if SIZE_LENGTH == 0 { - self.working_buffer.push(0x81); // vint representation of "1" - self.working_buffer.extend_from_slice(&n.to_be_bytes()); - } else { - self.working_buffer - .extend_from_slice(&1u8.as_vint_with_length::()?); - self.working_buffer.extend_from_slice(&n.to_be_bytes()); - } - Ok(()) - }) - .or_else(|_| { - i16::try_from(data).map(|n| { - if SIZE_LENGTH == 0 { - self.working_buffer.push(0x82); // vint representation of "2" - self.working_buffer.extend_from_slice(&n.to_be_bytes()); - } else { - self.working_buffer - .extend_from_slice(&2u8.as_vint_with_length::()?); - self.working_buffer.extend_from_slice(&n.to_be_bytes()); - } - Ok(()) - }) - }) - .or_else(|_| { - i32::try_from(data).map(|n| { - if SIZE_LENGTH == 0 { - self.working_buffer.push(0x84); // vint representation of "4" - self.working_buffer.extend_from_slice(&n.to_be_bytes()); - } else { - self.working_buffer - .extend_from_slice(&4u8.as_vint_with_length::()?); - self.working_buffer.extend_from_slice(&n.to_be_bytes()); - } - Ok(()) - }) - }) - .unwrap_or_else(|_| { - if SIZE_LENGTH == 0 { - self.working_buffer.push(0x88); // vint representation of "8" - self.working_buffer.extend_from_slice(&data.to_be_bytes()); - } else { - self.working_buffer - .extend_from_slice(&8u8.as_vint_with_length::()?); - self.working_buffer.extend_from_slice(&data.to_be_bytes()); - } - Ok(()) - }) - .map_err(|err: ToolError| TagWriterError::TagSizeError(err.to_string())) - } - - fn write_utf8_tag(&mut self, id: u64, data: &str) -> Result<(), TagWriterError> { - self.working_buffer - .extend(id.to_be_bytes().iter().skip_while(|&v| *v == 0u8)); - - let slice: &[u8] = data.as_bytes(); - let size: u64 = slice.len().try_into().expect("couldn't convert usize to u64"); - if SIZE_LENGTH == 0 { - let size_vint = size - .as_vint() - .map_err(|e| TagWriterError::TagSizeError(e.to_string()))?; - self.working_buffer.extend_from_slice(&size_vint); - } else { - let size_vint = size - .as_vint_with_length::() - .map_err(|e| TagWriterError::TagSizeError(e.to_string()))?; - self.working_buffer.extend_from_slice(&size_vint); - }; - - self.working_buffer.extend_from_slice(slice); - Ok(()) - } - - fn write_binary_tag(&mut self, id: u64, data: &[u8]) -> Result<(), TagWriterError> { - self.working_buffer - .extend(id.to_be_bytes().iter().skip_while(|&v| *v == 0u8)); - - let size: u64 = data.len().try_into().expect("couldn't convert usize to u64"); - if SIZE_LENGTH == 0 { - let size_vint = size - .as_vint() - .map_err(|e| TagWriterError::TagSizeError(e.to_string()))?; - self.working_buffer.extend_from_slice(&size_vint); - } else { - let size_vint = size - .as_vint_with_length::() - .map_err(|e| TagWriterError::TagSizeError(e.to_string()))?; - self.working_buffer.extend_from_slice(&size_vint); - } - - self.working_buffer.extend_from_slice(data); - Ok(()) - } - - fn write_float_tag(&mut self, id: u64, data: &f64) -> Result<(), TagWriterError> { - self.working_buffer - .extend(id.to_be_bytes().iter().skip_while(|&v| *v == 0u8)); - if SIZE_LENGTH == 0 { - self.working_buffer.push(0x88); // vint representation of "8" - } else { - let size_vint = 8u8 - .as_vint_with_length::() - .map_err(|e| TagWriterError::TagSizeError(e.to_string()))?; - self.working_buffer.extend_from_slice(&size_vint); - } - self.working_buffer.extend_from_slice(&data.to_be_bytes()); - Ok(()) - } - - /// - /// Write a tag to this instance's destination. - /// - /// This method writes a tag from any specification. There are no restrictions on the type of specification being written - it simply needs to implement the [`EbmlSpecification`] and [`EbmlTag`] traits. - /// - /// ## Errors - /// - /// This method can error if there is a problem writing the input tag. The different possible error states are enumerated in [`TagWriterError`]. - /// - /// ## Panics - /// - /// This method can panic if `` is an internally inconsistent specification (i.e. it claims that a specific tag variant is a specific data type but it is not). This won't happen if the specification being used was created using the [`#[ebml_specification]`](https://docs.rs/ebml-iterable-specification-derive/latest/ebml_iterable_specification_derive/attr.ebml_specification.html) attribute macro. - /// - /// ## Examples - /// - /// ```no_run - /// use std::fs::File; - /// use ebml_iterable::TagWriter; - /// use ebml_iterable::specs::Master; - /// # use ebml_iterable_specification::empty_spec::EmptySpec; - /// - /// # fn main() -> Result<(), Box> { - /// let mut file = File::create("my_ebml_file.ebml")?; - /// let mut my_writer = TagWriter::new(&mut file); - /// my_writer.write(&EmptySpec::with_children( - /// 0x1a45dfa3, - /// vec![EmptySpec::with_data(0x18538067, &[0x01])]) - /// )?; - /// # Ok(()) - /// # } - /// ``` - /// - pub fn write + EbmlTag + Clone>( - &mut self, - tag: &TSpec, - ) -> Result<(), TagWriterError> { - self.write_advanced( - tag, - WriteOptions { - size_byte_length: None, - unknown_sized_element: false, - }, - ) - } - - /// - /// Write a tag to this instance's destination using advanced options. - /// - /// This method is just like the normal [`write()`](#method.write) method, but allows for tailoring the output binary to better suit your needs. See [`WriteOptions`] for more detail on available options. - /// - /// ## Errors - /// - /// This method can error if there is a problem writing the input tag. The different possible error states are enumerated in [`TagWriterError`]. - /// - /// ## Panics - /// - /// This method can panic if `` is an internally inconsistent specification (i.e. it claims that a specific tag variant is a specific data type but it is not). This won't happen if the specification being used was created using the [`#[ebml_specification]`](https://docs.rs/ebml-iterable-specification-derive/latest/ebml_iterable_specification_derive/attr.ebml_specification.html) attribute macro. - /// - pub fn write_advanced + EbmlTag + Clone>( - &mut self, - tag: &TSpec, - options: WriteOptions, - ) -> Result<(), TagWriterError> { - let tag_id = tag.get_id(); - let tag_type = TSpec::get_tag_data_type(tag_id); - - if options.unknown_sized_element { - match tag_type { - Some(TagDataType::Master) => {} - _ => { - return Err(TagWriterError::TagSizeError(format!( - "Cannot write an unknown size for tag of type {tag_type:?}" - ))) - } - }; - self.start_unknown_size_tag(tag_id); - } else { - let should_validate = tag_type.is_some() - && (!matches!(tag_type, Some(TagDataType::Master)) - || !matches!( - tag.as_master().unwrap_or_else(|| panic!( - "Bad specification implementation: Tag id {} type was master, but could not get tag!", - tag_id - )), - Master::End - )); - if should_validate && !validate_tag_path::(tag_id, self.open_tags.iter().copied()) { - return Err(TagWriterError::UnexpectedTag { - tag_id, - current_path: self.open_tags.iter().map(|t| t.0).collect(), - }); - } - - match options.size_byte_length { - Some(1) => self.write_explicit_sized::(tag, tag_id, tag_type)?, - Some(2) => self.write_explicit_sized::(tag, tag_id, tag_type)?, - Some(3) => self.write_explicit_sized::(tag, tag_id, tag_type)?, - Some(4) => self.write_explicit_sized::(tag, tag_id, tag_type)?, - Some(5) => self.write_explicit_sized::(tag, tag_id, tag_type)?, - Some(6) => self.write_explicit_sized::(tag, tag_id, tag_type)?, - Some(7) => self.write_explicit_sized::(tag, tag_id, tag_type)?, - Some(8) => self.write_explicit_sized::(tag, tag_id, tag_type)?, - _ => self.write_explicit_sized::(tag, tag_id, tag_type)?, - } - } - - Ok(()) - } - - fn write_explicit_sized + EbmlTag + Clone, const SIZE_LENGTH: usize>( - &mut self, - tag: &TSpec, - tag_id: u64, - tag_type: Option, - ) -> Result<(), TagWriterError> { - assert!(SIZE_LENGTH < 9, "Vint length must be less than 9 bytes"); - match tag_type { - Some(TagDataType::UnsignedInt) => { - let val = tag.as_unsigned_int().unwrap_or_else(|| { - panic!( - "Bad specification implementation: Tag id {} type was unsigned int, but could not get tag!", - tag_id - ) - }); - self.write_unsigned_int_tag::(tag_id, val)? - } - Some(TagDataType::Integer) => { - let val = tag.as_signed_int().unwrap_or_else(|| { - panic!( - "Bad specification implementation: Tag id {} type was integer, but could not get tag!", - tag_id - ) - }); - self.write_signed_int_tag::(tag_id, val)? - } - Some(TagDataType::Utf8) => { - let val = tag.as_utf8().unwrap_or_else(|| { - panic!( - "Bad specification implementation: Tag id {} type was utf8, but could not get tag!", - tag_id - ) - }); - self.write_utf8_tag::(tag_id, val)? - } - Some(TagDataType::Binary) => { - let val = tag.as_binary().unwrap_or_else(|| { - panic!( - "Bad specification implementation: Tag id {} type was binary, but could not get tag!", - tag_id - ) - }); - self.write_binary_tag::(tag_id, val)? - } - Some(TagDataType::Float) => { - let val = tag.as_float().unwrap_or_else(|| { - panic!( - "Bad specification implementation: Tag id {} type was float, but could not get tag!", - tag_id - ) - }); - self.write_float_tag::(tag_id, val)? - } - Some(TagDataType::Master) => { - let position = tag.as_master().unwrap_or_else(|| { - panic!( - "Bad specification implementation: Tag id {} type was master, but could not get tag!", - tag_id - ) - }); - - match position { - Master::Start => self.start_tag(tag_id, SIZE_LENGTH), - Master::End => self.end_tag(tag_id)?, - Master::Full(children) => { - self.start_tag(tag_id, SIZE_LENGTH); - for child in children { - self.write(child)?; - } - self.end_tag(tag_id)?; - } - } - } - None => { - // Should be a "raw tag" - if !is_vint(tag_id) { - return Err(TagWriterError::TagIdError(tag_id)); - } else { - let val = tag.as_binary().unwrap_or_else(|| panic!("Bad specification implementation: Tag id {} type was raw tag, but could not get binary data!", tag_id)); - self.write_binary_tag::(tag_id, val)? - } - } - } - - if !self.open_tags.iter().any(|t| matches!(t.1, Known(_))) { - self.private_flush() - } else { - Ok(()) - } - } - - /// - /// Write a tag with an unknown size to this instance's destination. - /// - /// DEPRECATED - Prefer using the [`write_advanced()`](#method.write_advanced) method with [`WriteOptions`] obtained from their [`is_unknown_sized_element()`](struct.WriteOptions.html#method.is_unknown_sized_element) instead. - /// - /// This method allows you to start a tag that doesn't have a known size. Useful for streaming, or when the data is expected to be too large to fit into memory. This method can *only* be used on Master type tags. - /// - /// ## Errors - /// - /// This method will return an error if the input tag is not a Master type tag, as those are the only types allowed to be of unknown size. - /// - #[deprecated( - since = "0.6.0", - note = "Please use 'write_advanced' with WriteOptions obtained using 'is_unknown_sized_element' instead" - )] - pub fn write_unknown_size + EbmlTag + Clone>( - &mut self, - tag: &TSpec, - ) -> Result<(), TagWriterError> { - let tag_id = tag.get_id(); - let tag_type = TSpec::get_tag_data_type(tag_id); - match tag_type { - Some(TagDataType::Master) => {} - _ => { - return Err(TagWriterError::TagSizeError(format!( - "Cannot write an unknown size for tag of type {tag_type:?}" - ))) - } - }; - self.start_unknown_size_tag(tag_id); - Ok(()) - } - - /// - /// Write raw tag data to this instance's destination. - /// - /// This method allows writing any tag id with any arbitrary data without using a specification. Specifications should generally provide an `Unknown` variant to handle arbitrary unknown data which can be written through the regular [`write()`](#method.write) method, so use of this method is typically discouraged. - /// - /// ## Errors - /// - /// This method can error if there is a problem writing the input tag. The different possible error states are enumerated in [`TagWriterError`]. - /// - /// ## Examples - /// - /// ```no_run - /// use std::fs::File; - /// use ebml_iterable::TagWriter; - /// - /// # fn main() -> Result<(), Box> { - /// let mut file = File::create("my_ebml_file.ebml")?; - /// let mut my_writer = TagWriter::new(&mut file); - /// my_writer.write_raw(0x1a45dfa3, &[0x18, 0x53, 0x80, 0x67, 0x81, 0x01])?; - /// # Ok(()) - /// # } - /// ``` - /// - pub fn write_raw(&mut self, tag_id: u64, data: &[u8]) -> Result<(), TagWriterError> { - self.write_binary_tag::<0>(tag_id, data)?; - - if !self.open_tags.iter().any(|t| matches!(t.1, Known(_))) { - self.private_flush() - } else { - Ok(()) - } - } - - /// - /// Attempts to flush all unwritten tags to the underlying destination. - /// - /// This method can be used to finalize any open [`Master`] type tags that have not been ended. The writer makes an attempt to close every open tag and write all bytes to the instance's destination. - /// - /// ## Errors - /// - /// This method can error if there is a problem writing to the destination. - /// - pub fn flush(&mut self) -> Result<(), TagWriterError> { - while let Some(id) = self.open_tags.last().map(|t| t.0) { - self.end_tag(id)?; - } - self.private_flush() - } - - //TODO: panic on drop if there is an open tag that hasn't been written. Or maybe flush stream of any open tags? -} - -#[cfg(test)] -mod tests { - use std::io::Cursor; - - use super::super::tools::Vint; - use super::TagWriter; - - #[test] - fn write_ebml_tag() { - let mut dest = Cursor::new(Vec::new()); - let mut writer = TagWriter::new(&mut dest); - writer.write_raw(0x1a45dfa3, &[]).expect("Error writing tag"); - - let zero_size = 0u64.as_vint().expect("Error converting [0] to vint")[0]; - assert_eq!(vec![0x1a, 0x45, 0xdf, 0xa3, zero_size], dest.get_ref().to_vec()); - } -} diff --git a/crates/ebml-iterable/src/tools.rs b/crates/ebml-iterable/src/tools.rs deleted file mode 100644 index 882910aa7..000000000 --- a/crates/ebml-iterable/src/tools.rs +++ /dev/null @@ -1,523 +0,0 @@ -//! -//! Contains a number of tools that are useful when working with EBML encoded files. -//! - -use std::convert::TryInto; - -use super::errors::tool::ToolError; - -/// -/// Trait to enable easy serialization to a vint. -/// -/// This is only available for types that can be cast as `u64`. -/// -pub trait Vint: Into + Copy { - /// - /// Returns a representation of the current value as a vint array. - /// - /// # Errors - /// - /// This can return an error if the value is too large to be representable as a vint. - /// - fn as_vint(self) -> Result, ToolError> { - let val: u64 = self.into(); - check_size_u64(val, 8)?; - - if val < (1 << 7) { - Ok(as_vint_no_check_u64::<1>(val).to_vec()) - } else if val < (1 << (7 * 2)) { - Ok(as_vint_no_check_u64::<2>(val).to_vec()) - } else if val < (1 << (7 * 3)) { - Ok(as_vint_no_check_u64::<3>(val).to_vec()) - } else if val < (1 << (7 * 4)) { - Ok(as_vint_no_check_u64::<4>(val).to_vec()) - } else if val < (1 << (7 * 5)) { - Ok(as_vint_no_check_u64::<5>(val).to_vec()) - } else if val < (1 << (7 * 6)) { - Ok(as_vint_no_check_u64::<6>(val).to_vec()) - } else if val < (1 << (7 * 7)) { - Ok(as_vint_no_check_u64::<7>(val).to_vec()) - } else { - Ok(as_vint_no_check_u64::<8>(val).to_vec()) - } - } - - /// - /// Returns a representation of the current value as a vint array with a specified length. - /// - /// # Errors - /// - /// This can return an error if the value is too large to be representable as a vint. - /// - fn as_vint_with_length(&self) -> Result<[u8; LENGTH], ToolError> { - let val: u64 = (*self).into(); - check_size_u64(val, LENGTH)?; - Ok(as_vint_no_check_u64::(val)) - } -} - -impl Vint for u64 {} -impl Vint for u32 {} -impl Vint for u16 {} -impl Vint for u8 {} - -#[inline] -fn check_size_u64(val: u64, max_length: usize) -> Result<(), ToolError> { - if val >= 1 << (max_length * 7) { - Err(ToolError::WriteVintOverflow(val)) - } else { - Ok(()) - } -} - -#[inline] -fn as_vint_no_check_u64(val: u64) -> [u8; LENGTH] { - let mut bytes: [u8; 8] = val.to_be_bytes(); - bytes[8 - LENGTH] |= 1 << (8 - LENGTH); - bytes[8 - LENGTH..].try_into().expect("8 - (8-length) != length !?!?") -} - -/// -/// Reads a vint from the beginning of the input array slice. -/// -/// This method returns an option with the `None` variant used to indicate there was not enough data in the buffer to completely read a vint. -/// -/// The returned tuple contains the value of the vint (`u64`) and the length of the vint (`usize`). The length will be less than or equal to the length of the input slice. -/// -/// # Errors -/// -/// This method can return a `ToolError` if the input array cannot be read as a vint. -/// -pub fn read_vint(buffer: &[u8]) -> Result, ToolError> { - if buffer.is_empty() { - return Ok(None); - } - - if buffer[0] == 0 { - return Err(ToolError::ReadVintOverflow); - } - - let length = 8 - buffer[0].ilog2() as usize; - - if length > buffer.len() { - // Not enough data in the buffer to read out the vint value - return Ok(None); - } - - let mut value = buffer[0] as u64; - value -= 1 << (8 - length); - - for item in buffer.iter().take(length).skip(1) { - value <<= 8; - value += *item as u64; - } - - Ok(Some((value, length))) -} - -pub fn is_vint(val: u64) -> bool { - if val == 0 { - return false; - } - - val.ilog2().is_multiple_of(7) -} - -/// -/// Trait to enable easy serialization to a signed vint. -/// -/// This is only available for types that can be cast as `i64`. A signed vint can be written as a variable number of bytes just like a regular vint, but the value portion of the vint is expressed in two's complement notation. -/// -/// For example, the decimal number "-33" would be written as [0xDF = 1101 1111]. This value is determined by first taking the two's complement of 33 [0x21 = 0010 0001] **but only using the bits available for the vint value**. In this case, that is 7 bits (because the vint marker takes up the 8th bit). The two's complement is [101 1111]. A handy calculator for two's complement can be found [here](https://www.omnicalculator.com/math/twos-complement). Once the two's complement has been found, simply prepend the vint marker as usual to get [1101 1111 = 0xDF]. -/// -/// Some more examples: -/// ``` -/// use ebml_iterable::tools::SignedVint; -/// -/// # fn main() -> Result<(), Box> { -/// assert_eq!(vec![0xDF], (-33i64).as_signed_vint().unwrap()); -/// assert_eq!(vec![0x40, 0xC8], (200i64).as_signed_vint().unwrap()); -/// assert_eq!(vec![0x7F, 0x38], (-200i64).as_signed_vint().unwrap()); -/// assert_eq!(vec![0xFF], (-1i64).as_signed_vint().unwrap()); -/// # Ok(()) -/// # } -/// ``` -pub trait SignedVint: Into + Copy { - /// - /// Returns a representation of the current value as a vint array. - /// - /// # Errors - /// - /// This can return an error if the value is outside of the range that can be represented as a vint. - /// - fn as_signed_vint(&self) -> Result, ToolError> { - let val: i64 = (*self).into(); - check_size_i64(val, 8)?; - let mut length = 1; - while length <= 8 { - if val >= -(1 << (7 * length - 1)) && val < (1 << (7 * length - 1)) { - break; - } - length += 1; - } - - Ok(as_vint_no_check_i64(val, length)) - } - - /// - /// Returns a representation of the current value as a vint array with a specified length. - /// - /// # Errors - /// - /// This can return an error if the value is outside of the range that can be represented as a vint. - /// - fn as_signed_vint_with_length(&self, length: usize) -> Result, ToolError> { - let val: i64 = (*self).into(); - check_size_i64(val, length)?; - Ok(as_vint_no_check_i64(val, length)) - } -} - -impl SignedVint for i64 {} -impl SignedVint for i32 {} -impl SignedVint for i16 {} -impl SignedVint for i8 {} - -#[inline] -fn check_size_i64(val: i64, max_length: usize) -> Result<(), ToolError> { - if val <= -(1 << (max_length * 7 - 1)) || val >= (1 << (max_length * 7 - 1)) { - Err(ToolError::WriteSignedVintOverflow(val)) - } else { - Ok(()) - } -} - -#[inline] -fn as_vint_no_check_i64(val: i64, length: usize) -> Vec { - let bytes: [u8; 8] = val.to_be_bytes(); - let mut result: Vec = Vec::from(&bytes[(8 - length)..]); - if val < 0 { - result[0] &= 0xFF >> (length - 1); - } else { - result[0] |= 1 << (8 - length); - } - result -} - -/// -/// Reads a signed vint from the beginning of the input array slice. -/// -/// This method returns an option with the `None` variant used to indicate there was not enough data in the buffer to completely read a vint. -/// -/// The returned tuple contains the value of the vint (`i64`) and the length of the vint (`usize`). The length will be less than or equal to the length of the input slice. -/// -/// # Errors -/// -/// This method can return a `ToolError` if the input array cannot be read as a vint. -/// -pub fn read_signed_vint(buffer: &[u8]) -> Result, ToolError> { - if buffer.is_empty() { - return Ok(None); - } - - if buffer[0] == 0 { - return Err(ToolError::ReadVintOverflow); - } - - let length = 8 - buffer[0].ilog2() as usize; - - if length > buffer.len() { - // Not enough data in the buffer to read out the vint value - return Ok(None); - } - - let is_negative = if length == 8 { - buffer[1] & 0x80 - } else { - buffer[0] & (0x80 >> length) - } > 0; - - let mut value = if is_negative { - (buffer[0] as i64) | (!0i64 << (8 - length)) - } else { - (buffer[0] & (0xFF >> length)) as i64 - }; - - for item in buffer.iter().take(length).skip(1) { - value <<= 8; - value += *item as i64; - } - - Ok(Some((value, length))) -} - -/// -/// Reads a `u64` value from any length array slice. -/// -/// Rather than forcing the input to be a `[u8; 8]` like standard library methods, this can interpret a `u64` from a slice of any length < 8. Bytes are assumed to be least significant when reading the value - i.e. an array of `[4, 0]` would return a value of `1024`. -/// -/// # Errors -/// -/// This method will return an error if the input slice has a length > 8. -/// -/// ## Example -/// -/// ``` -/// # use ebml_iterable::tools::arr_to_u64; -/// # fn main() -> Result<(), Box> { -/// let result = arr_to_u64(&[16,0])?; -/// assert_eq!(result, 4096); -/// # Ok(()) -/// # } -/// ``` -/// -pub fn arr_to_u64(arr: &[u8]) -> Result { - if arr.len() > 8 { - return Err(ToolError::ReadU64Overflow(Vec::from(arr))); - } - - let mut val = 0u64; - for byte in arr { - val *= 256; - val += *byte as u64; - } - Ok(val) -} - -/// -/// Reads an `i64` value from any length array slice. -/// -/// Rather than forcing the input to be a `[u8; 8]` like standard library methods, this can interpret an `i64` from a slice of any length < 8. Bytes are assumed to be least significant when reading the value - i.e. an array of `[4, 0]` would return a value of `1024`. -/// -/// # Errors -/// -/// This method will return an error if the input slice has a length > 8. -/// -/// ## Example -/// -/// ``` -/// # use ebml_iterable::tools::arr_to_i64; -/// # fn main() -> Result<(), Box> { -/// let result = arr_to_i64(&[4,0])?; -/// assert_eq!(result, 1024); -/// # Ok(()) -/// # } -/// ``` -/// -pub fn arr_to_i64(arr: &[u8]) -> Result { - if arr.len() > 8 { - return Err(ToolError::ReadI64Overflow(Vec::from(arr))); - } - - if arr[0] > 127 { - if arr.len() == 8 { - Ok(i64::from_be_bytes( - arr.try_into().expect("[u8;8] should be convertible to i64"), - )) - } else { - Ok(-((1 << (arr.len() * 8)) - - (arr_to_u64(arr).expect("arr_to_u64 shouldn't error if length is <= 8") as i64))) - } - } else { - Ok(arr_to_u64(arr).expect("arr_to_u64 shouldn't error if length is <= 8") as i64) - } -} - -/// -/// Reads an `f64` value from an array slice of length 4 or 8. -/// -/// This method wraps `f32` and `f64` conversions from big endian byte arrays and casts the result as an `f64`. -/// -/// # Errors -/// -/// This method will throw an error if the input slice length is not 4 or 8. -/// -pub fn arr_to_f64(arr: &[u8]) -> Result { - if arr.len() == 4 { - Ok(f32::from_be_bytes(arr.try_into().expect("arr should be [u8;4]")) as f64) - } else if arr.len() == 8 { - Ok(f64::from_be_bytes(arr.try_into().expect("arr should be [u8;8]"))) - } else { - Err(ToolError::ReadF64Mismatch(Vec::from(arr))) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn read_vint_sixteen() { - let buffer = [144]; - let result = read_vint(&buffer).unwrap().expect("Reading vint failed"); - - assert_eq!(16, result.0); - assert_eq!(1, result.1); - } - - #[test] - fn write_vint_sixteen() { - let result = 16u64.as_vint().expect("Writing vint failed"); - assert_eq!(vec![144u8], result); - } - - #[test] - fn read_vint_one_twenty_seven() { - let buffer = [255u8]; - let result = read_vint(&buffer).unwrap().expect("Reading vint failed"); - - assert_eq!(127, result.0); - assert_eq!(1, result.1); - } - - #[test] - fn write_vint_one_twenty_seven() { - let result = 127u64.as_vint().expect("Writing vint failed"); - assert_eq!(vec![255u8], result); - } - - #[test] - fn read_vint_two_hundred() { - let buffer = [64, 200]; - let result = read_vint(&buffer).unwrap().expect("Reading vint failed"); - - assert_eq!(200, result.0); - assert_eq!(2, result.1); - } - - #[test] - fn write_vint_two_hundred() { - let result = 200u64.as_vint().expect("Writing vint failed"); - assert_eq!(vec![64u8, 200u8], result); - } - - #[test] - fn read_vint_for_ebml_tag() { - let buffer = [0x1a, 0x45, 0xdf, 0xa3]; - let result = read_vint(&buffer).unwrap().expect("Reading vint failed"); - - assert_eq!(0x0a45dfa3, result.0); - assert_eq!(4, result.1); - } - - #[test] - fn read_vint_very_long() { - let buffer = [1, 0, 0, 0, 0, 0, 0, 1]; - let result = read_vint(&buffer).unwrap().expect("Reading vint failed"); - - assert_eq!(1, result.0); - assert_eq!(8, result.1); - } - - #[test] - fn write_vint_very_long() { - let result = 1u64.as_vint_with_length::<8>().expect("Writing vint failed"); - assert_eq!(vec![1, 0, 0, 0, 0, 0, 0, 1], result); - } - - #[test] - fn read_vint_overflow() { - let buffer = [1, 0, 0, 0]; - let result = read_vint(&buffer).expect("Reading vint failed"); - - assert_eq!(true, result.is_none()); - } - - #[test] - #[should_panic] - fn too_big_for_vint() { - (1u64 << 56).as_vint().expect("Writing vint failed"); - } - - #[test] - fn vint_encode_decode_range() { - for val in 0..500_000 { - let bytes = val.as_vint().unwrap(); - let result = read_vint(bytes.as_slice()).unwrap().unwrap().0; - assert_eq!(val, result); - } - } - - #[test] - fn signed_vint_encode_decode_range() { - for val in -500_000..500_000 { - let bytes = val.as_signed_vint().unwrap(); - let result = read_signed_vint(bytes.as_slice()).unwrap().unwrap().0; - assert_eq!(val, result); - } - } - - #[test] - fn read_u64_values() { - let mut buffer = vec![]; - let mut expected = 0; - for _ in 0..8 { - buffer.push(0x25); - expected = (expected << 8) + 0x25; - - let result = arr_to_u64(&buffer).unwrap(); - assert_eq!(expected, result); - } - } - - #[test] - fn read_i64_values() { - let mut buffer = vec![]; - let mut expected = 0; - for _ in 0..8 { - buffer.push(0x0a); - expected = (expected << 8) + 0x0a; - - let result = arr_to_i64(&buffer).unwrap(); - assert_eq!(expected, result); - - let neg_result = arr_to_i64(&(buffer.iter().map(|b| !b).collect::>())).unwrap() + 1; - assert_eq!(-expected, neg_result); - } - } - - #[test] - fn valid_vints() { - assert!(is_vint(0x1F43B675)); - assert!(is_vint(0xA0)); - assert!(is_vint(0xA1)); - assert!(is_vint(0x75A1)); - assert!(is_vint(0xA6)); - assert!(is_vint(0xEE)); - assert!(is_vint(0xA5)); - assert!(is_vint(0x9B)); - assert!(is_vint(0xA2)); - assert!(is_vint(0xA4)); - assert!(is_vint(0x75A2)); - assert!(is_vint(0xFB)); - assert!(is_vint(0xC8)); - assert!(is_vint(0xC9)); - assert!(is_vint(0xCA)); - assert!(is_vint(0xFA)); - assert!(is_vint(0xFD)); - assert!(is_vint(0x8E)); - assert!(is_vint(0xE8)); - assert!(is_vint(0xCB)); - assert!(is_vint(0xCE)); - assert!(is_vint(0xCD)); - assert!(is_vint(0xCC)); - assert!(is_vint(0xCF)); - assert!(is_vint(0xAF)); - assert!(is_vint(0xA7)); - assert!(is_vint(0xAB)); - assert!(is_vint(0x5854)); - assert!(is_vint(0x58D7)); - assert!(is_vint(0xA3)); - assert!(is_vint(0xE7)); - assert!(is_vint(0x3E83BB)); - assert!(is_vint(0x3EB923)); - assert!(is_vint(0x3C83AB)); - assert!(is_vint(0x3CB923)); - - assert!(!is_vint(1234)); - assert!(!is_vint(0x11)); - assert!(!is_vint(0x7a)); - assert!(!is_vint(0xfa4c)); - assert!(!is_vint(0x1a5d)); - } -} diff --git a/crates/ebml-iterable/tests/corrupt_data_tests.rs b/crates/ebml-iterable/tests/corrupt_data_tests.rs deleted file mode 100644 index 5b6d6ac82..000000000 --- a/crates/ebml-iterable/tests/corrupt_data_tests.rs +++ /dev/null @@ -1,239 +0,0 @@ -mod test_spec; - -pub mod corrupt_data_tests { - use std::io::Cursor; - - use ebml_iterable::error::{CorruptedFileError, TagIteratorError}; - use ebml_iterable::iterator::AllowableErrors; - use ebml_iterable::specs::Master; - use ebml_iterable::{TagIterator, TagWriter, WriteOptions}; - - use super::test_spec::TestSpec; - - fn get_data_with_invalid_ids() -> Cursor> { - let tags: Vec = vec![ - TestSpec::Segment(Master::Start), - TestSpec::TrackType(0x01), - TestSpec::RawTag(0xf2, vec![0x01]), - TestSpec::Segment(Master::End), - ]; - - let mut dest = Cursor::new(Vec::new()); - let mut writer = TagWriter::new(&mut dest); - - for tag in tags.iter() { - writer.write(tag).expect("Test shouldn't error"); - } - - println!("dest {:x?}", dest); - dest.set_position(0); - dest - } - - #[test] - pub fn error_on_invalid_ids() { - let mut cursor = get_data_with_invalid_ids(); - let mut reader: TagIterator<_, TestSpec> = TagIterator::new(&mut cursor, &[]); - assert!(reader.next().unwrap().is_ok()); - assert!(reader.next().unwrap().is_ok()); - assert!(matches!( - reader.next().unwrap(), - Err(TagIteratorError::CorruptedFileData(CorruptedFileError::InvalidTagId { - tag_id: _, - position: _ - })) - )); - } - - #[test] - pub fn allow_errors_invalid_ids() { - let mut cursor = get_data_with_invalid_ids(); - let mut reader: TagIterator<_, TestSpec> = TagIterator::new(&mut cursor, &[]); - reader.allow_errors(&[AllowableErrors::InvalidTagIds]); - reader.for_each(|t| assert!(t.is_ok())); - } - - fn get_data_with_hierarchy_problems() -> Cursor> { - // Test based on this structure. Can't use the writer for this test since it checks hierarchy when writing. - - // let tags: Vec = vec![ - // TestSpec::Segment(Master::Start), - // TestSpec::Count(1), - // TestSpec::Segment(Master::End), - // ]; - - let mut dest = Cursor::new(vec![0x18, 0x53, 0x80, 0x67, 0x84, 0x41, 0x00, 0x81, 0x01]); - - println!("dest {:x?}", dest); - dest.set_position(0); - dest - } - - #[test] - pub fn error_on_hierarchy_problems() { - let mut cursor = get_data_with_hierarchy_problems(); - let mut reader: TagIterator<_, TestSpec> = TagIterator::new(&mut cursor, &[]); - assert!(reader.next().unwrap().is_ok()); - assert!(matches!( - reader.next().unwrap(), - Err(TagIteratorError::CorruptedFileData( - CorruptedFileError::HierarchyError { - found_tag_id: _, - current_parent_id: _ - } - )) - )); - } - - #[test] - pub fn allow_errors_hierarchy_problems() { - let mut cursor = get_data_with_hierarchy_problems(); - let mut reader: TagIterator<_, TestSpec> = TagIterator::new(&mut cursor, &[]); - reader.allow_errors(&[AllowableErrors::HierarchyProblems]); - reader.for_each(|t| assert!(t.is_ok())); - } - - fn get_data_with_oversized_child() -> Cursor> { - let tags: Vec = vec![ - TestSpec::Segment(Master::Start), - TestSpec::Cluster(Master::Start), - TestSpec::Block(vec![0x01, 0x02, 0x03, 0x04]), - TestSpec::Cluster(Master::End), - TestSpec::Segment(Master::End), - ]; - - let mut dest = Cursor::new(Vec::new()); - let mut writer = TagWriter::new(&mut dest); - - for tag in tags.iter() { - writer.write(tag).expect("Test shouldn't error"); - } - - // Extend size of block element without resizing parents - dest.get_mut()[11] = 0x86; - dest.get_mut().push(0x0a); - dest.get_mut().push(0x0a); - - println!("dest {:x?}", dest); - dest.set_position(0); - dest - } - - #[test] - pub fn error_on_oversized_child() { - let mut cursor = get_data_with_oversized_child(); - let mut reader: TagIterator<_, TestSpec> = TagIterator::new(&mut cursor, &[]); - assert!(reader.next().unwrap().is_ok()); - assert!(reader.next().unwrap().is_ok()); - assert!(matches!( - reader.next().unwrap(), - Err(TagIteratorError::CorruptedFileData( - CorruptedFileError::OversizedChildElement { - position: _, - tag_id: _, - size: _ - } - )) - )); - } - - #[test] - pub fn allow_errors_oversized_child() { - let mut cursor = get_data_with_oversized_child(); - let mut reader: TagIterator<_, TestSpec> = TagIterator::new(&mut cursor, &[]); - reader.allow_errors(&[AllowableErrors::OversizedTags]); - reader.for_each(|t| assert!(t.is_ok())); - } - - fn get_data_with_6_byte_tag() -> Cursor> { - let tags: Vec = vec![ - TestSpec::Segment(Master::Start), - TestSpec::Cluster(Master::Start), - TestSpec::Block(vec![0x01, 0x02, 0x03, 0x04, 0x05, 0x06]), - TestSpec::Cluster(Master::End), - TestSpec::Segment(Master::End), - ]; - - let mut dest = Cursor::new(Vec::new()); - let mut writer = TagWriter::new(&mut dest); - - for tag in tags.iter() { - if matches!(tag, TestSpec::Segment(_)) || matches!(tag, TestSpec::Cluster(_)) { - writer - .write_advanced(tag, WriteOptions::is_unknown_sized_element()) - .expect("Test shouldn't error"); - } else { - writer.write(tag).expect("Test shouldn't error"); - } - } - - // // Rewrite size of block element - // dest.get_mut()[25] = 0x09; - // dest.get_mut()[26] = 0x65; - // dest.get_mut()[27] = 0xa0; - // dest.get_mut()[28] = 0xbc; - // dest.get_mut()[29] = 0x00; - - println!("dest {:x?}", dest); - dest.set_position(0); - dest - } - - #[test] - pub fn error_on_oversized_tag() { - let mut cursor = get_data_with_6_byte_tag(); - let mut reader: TagIterator<_, TestSpec> = TagIterator::new(&mut cursor, &[]); - reader.set_max_allowable_tag_size(Some(5)); - assert!(reader.next().unwrap().is_ok()); - assert!(reader.next().unwrap().is_ok()); - assert!(matches!( - reader.next().unwrap(), - Err(TagIteratorError::CorruptedFileData( - CorruptedFileError::InvalidTagSize { - position: _, - tag_id: _, - size: _ - } - )) - )); - } - - #[test] - pub fn recover_on_global_element() { - let tags: Vec = vec![ - TestSpec::Segment(Master::Start), - TestSpec::Cluster(Master::Start), - TestSpec::Crc32(vec![0x01]), - TestSpec::Count(1), - TestSpec::Cluster(Master::End), - TestSpec::Segment(Master::End), - ]; - - let mut dest = Cursor::new(Vec::new()); - let mut writer = TagWriter::new(&mut dest); - - for tag in tags.iter() { - writer.write(tag).expect("Test shouldn't error"); - } - - // Inserting some junk data to skip - dest.get_mut().insert(10, 0x0a); - dest.get_mut().insert(10, 0x0a); - dest.get_mut().insert(10, 0x0a); - dest.set_position(0); - - println!("dest {:x?}", dest); - - let mut reader: TagIterator<_, TestSpec> = TagIterator::new(&mut dest, &[]); - assert!(matches!(reader.next(), Some(t) if t.is_ok())); - assert!(matches!(reader.next(), Some(t) if t.is_ok())); - assert!(matches!(reader.next(), Some(t) if t.is_err())); - assert!(reader.try_recover().is_ok()); - reader.for_each(|t| { - if let Err(err) = t { - println!("{err:?}"); - assert!(false); - } - }); - } -} diff --git a/crates/ebml-iterable/tests/derive_spec_compile.rs b/crates/ebml-iterable/tests/derive_spec_compile.rs deleted file mode 100644 index d1e7006f6..000000000 --- a/crates/ebml-iterable/tests/derive_spec_compile.rs +++ /dev/null @@ -1,45 +0,0 @@ -#[cfg(feature = "derive-spec")] -pub mod derive_spec_compile { - use ebml_iterable::specs::{ebml_specification, EbmlSpecification, Master, TagDataType}; - - #[ebml_specification] - #[derive(Clone, Debug, PartialEq)] - pub enum Trial { - #[id(0x01)] - #[data_type(TagDataType::Master)] - Root, - - #[id(0x02)] - #[data_type(TagDataType::Master)] - Parent, - - #[id(0x100)] - #[data_type(TagDataType::UnsignedInt)] - Count, - - #[id(0x200)] - #[data_type(TagDataType::Binary)] - Data, - - #[id(0x201)] - #[data_type(TagDataType::Utf8)] - Name, - - #[id(0x102)] - #[data_type(TagDataType::Float)] - Amount, - - #[id(0x101)] - #[data_type(TagDataType::Integer)] - Id, - } - - #[test] - pub fn compile_worked() { - let data_type = Trial::get_tag_data_type(0x01); - assert_eq!(Some(TagDataType::Master), data_type); - - let tag = Trial::get_master_tag(0x01, Master::Start).unwrap(); - assert_eq!(Trial::Root(Master::Start), tag); - } -} diff --git a/crates/ebml-iterable/tests/derive_spec_compile_with_hierarchy.rs b/crates/ebml-iterable/tests/derive_spec_compile_with_hierarchy.rs deleted file mode 100644 index 170f6f6f9..000000000 --- a/crates/ebml-iterable/tests/derive_spec_compile_with_hierarchy.rs +++ /dev/null @@ -1,51 +0,0 @@ -#[cfg(feature = "derive-spec")] -pub mod derive_spec_compile { - use ebml_iterable::specs::{ebml_specification, EbmlSpecification, Master, TagDataType}; - - #[ebml_specification] - #[derive(Clone, Debug, PartialEq)] - pub enum Trial { - #[id(0x01)] - #[data_type(TagDataType::Master)] - Root, - - #[id(0x02)] - #[data_type(TagDataType::Master)] - #[doc_path(Root)] - Parent, - - #[id(0x100)] - #[data_type(TagDataType::UnsignedInt)] - #[doc_path(Root/Parent)] - Count, - - #[id(0x200)] - #[data_type(TagDataType::Binary)] - #[doc_path(Root/Parent)] - Data, - - #[id(0x201)] - #[data_type(TagDataType::Utf8)] - #[doc_path(Root/Parent)] - Name, - - #[id(0x102)] - #[data_type(TagDataType::Float)] - #[doc_path(Root/Parent)] - Amount, - - #[id(0x101)] - #[data_type(TagDataType::Integer)] - #[doc_path(Root/Parent)] - Id, - } - - #[test] - pub fn compile_worked() { - let data_type = Trial::get_tag_data_type(0x01); - assert_eq!(Some(TagDataType::Master), data_type); - - let tag = Trial::get_master_tag(0x01, Master::Start).unwrap(); - assert_eq!(Trial::Root(Master::Start), tag); - } -} diff --git a/crates/ebml-iterable/tests/spec_write_read.rs b/crates/ebml-iterable/tests/spec_write_read.rs deleted file mode 100644 index a4d974745..000000000 --- a/crates/ebml-iterable/tests/spec_write_read.rs +++ /dev/null @@ -1,375 +0,0 @@ -mod test_spec; - -pub mod spec_write_read { - use std::io::Cursor; - - use ebml_iterable::error::TagIteratorError; - use ebml_iterable::specs::{EbmlTag, Master}; - use ebml_iterable::{TagIterator, TagWriter, WriteOptions}; - - use super::test_spec::TestSpec; - - #[test] - pub fn simple_read_write() { - let tags: Vec = vec![ - TestSpec::Ebml(Master::Start), - TestSpec::Ebml(Master::End), - TestSpec::Segment(Master::Start), - TestSpec::TrackType(0x01), - TestSpec::Segment(Master::End), - ]; - - let mut dest = Cursor::new(Vec::new()); - let mut writer = TagWriter::new(&mut dest); - - for tag in tags.iter() { - writer.write(tag).expect("Test shouldn't error"); - } - - println!("dest {:?}", dest); - - let mut src = Cursor::new(dest.get_ref().to_vec()); - let reader = TagIterator::new(&mut src, &[]); - let read_tags: Vec = reader.into_iter().map(|t| t.unwrap()).collect(); - - println!("tags {:?}", read_tags); - - for i in 0..read_tags.len() { - assert_eq!(tags[i], read_tags[i]); - } - } - - #[test] - pub fn read_write_buffered_tag() { - let tags: Vec = vec![ - TestSpec::Segment(Master::Start), - TestSpec::Cluster(Master::Full(vec![TestSpec::CueRefCluster(0x02)])), - TestSpec::Segment(Master::End), - ]; - - let mut dest = Cursor::new(Vec::new()); - let mut writer = TagWriter::new(&mut dest); - - for tag in tags.iter() { - writer.write(tag).expect("Test shouldn't error"); - } - - println!("dest {:?}", dest); - - let mut src = Cursor::new(dest.get_ref().to_vec()); - let reader = TagIterator::new(&mut src, &[TestSpec::Cluster(Master::Start)]); - let read_tags: Vec = reader.into_iter().map(|t| t.unwrap()).collect(); - - println!("tags {:?}", read_tags); - - for i in 0..read_tags.len() { - assert_eq!(tags[i], read_tags[i]); - } - } - - #[test] - pub fn oversized_tag() { - let mut dest = Cursor::new(Vec::new()); - let mut writer = TagWriter::new(&mut dest); - - writer - .write(&TestSpec::Segment(Master::Start)) - .expect("Error writing tag"); - writer - .write(&TestSpec::Cluster(Master::Start)) - .expect("Error writing tag"); - // Why 0x10001 specifically? This exceeds the default buffer length (0x10000)?! - writer.write_raw(0xa1, &[0x00; 0x10001]).expect("Error writing tag"); - writer.write(&TestSpec::Count(0x00)).expect("Error writing tag"); - writer - .write(&TestSpec::Cluster(Master::End)) - .expect("Error writing tag"); - writer - .write(&TestSpec::Segment(Master::End)) - .expect("Error writing tag"); - drop(writer); - - dest.set_position(0); - let iter = TagIterator::<_, TestSpec>::new(dest, &[]); - - let tags: Vec<_> = iter.into_iter().collect(); - assert_eq!(tags.len(), 4 + 2, "Reading every tag that was written"); - } - - #[test] - pub fn write_unknown_size() { - let mut dest = Cursor::new(Vec::new()); - let mut writer = TagWriter::new(&mut dest); - - writer.write(&TestSpec::Root(Master::Start)).unwrap(); - writer - .write_advanced( - &TestSpec::Parent(Master::Start), - WriteOptions::is_unknown_sized_element(), - ) - .unwrap(); - writer.write(&TestSpec::Child(1)).unwrap(); - writer.write(&TestSpec::Child(2)).unwrap(); - writer.write(&TestSpec::Parent(Master::End)).unwrap(); - writer.write(&TestSpec::Root(Master::End)).unwrap(); - - dest.set_position(0); - - let iter = TagIterator::<_, TestSpec>::new(dest, &[]); - let tags: Vec<_> = iter.into_iter().collect(); - assert_eq!(tags.len(), 6, "Reading every tag that was written"); - } - - #[test] - pub fn buffer_unknown_size() { - let mut dest = Cursor::new(Vec::new()); - let mut writer = TagWriter::new(&mut dest); - - writer.write(&TestSpec::Root(Master::Start)).unwrap(); - writer - .write_advanced( - &TestSpec::Parent(Master::Start), - WriteOptions::is_unknown_sized_element(), - ) - .unwrap(); - writer.write(&TestSpec::Child(1)).unwrap(); - writer.write(&TestSpec::Child(2)).unwrap(); - writer.write(&TestSpec::Parent(Master::End)).unwrap(); - writer.write(&TestSpec::Root(Master::End)).unwrap(); - - dest.set_position(0); - - let iter = TagIterator::<_, TestSpec>::new(dest, &[TestSpec::Parent(Master::Start)]); - let mut tags: Vec<_> = iter.into_iter().collect(); - assert_eq!(tags.len(), 3, "Buffering 'Parent' into full variant"); - - tags.pop(); - let parent = tags.pop().unwrap().unwrap(); - assert!( - matches!(parent.as_master(), Some(Master::Full(c)) if c.len() == 2), - "Did not buffer tag as master with 2 children" - ); - } - - #[test] - pub fn unknown_size_write_read() { - let mut dest = Cursor::new(Vec::new()); - let mut writer = TagWriter::new(&mut dest); - - writer.write(&TestSpec::Root(Master::Start)).unwrap(); - writer - .write_advanced( - &TestSpec::Parent(Master::Start), - WriteOptions::is_unknown_sized_element(), - ) - .unwrap(); - writer.write(&TestSpec::Child(1)).unwrap(); - writer.write(&TestSpec::Child(2)).unwrap(); - writer.write(&TestSpec::Parent(Master::End)).unwrap(); - writer.write(&TestSpec::Int(2)).unwrap(); - writer.write(&TestSpec::Root(Master::End)).unwrap(); - - println!("{dest:x?}"); - dest.set_position(0); - - let mut iter = TagIterator::<_, TestSpec>::new(dest, &[]); - assert!(matches!(iter.next(), Some(Ok(TestSpec::Root(Master::Start))))); - assert!(matches!(iter.next(), Some(Ok(TestSpec::Parent(Master::Start))))); - assert!(matches!(iter.next(), Some(Ok(TestSpec::Child(1))))); - assert!(matches!(iter.next(), Some(Ok(TestSpec::Child(2))))); - assert!(matches!(iter.next(), Some(Ok(TestSpec::Parent(Master::End))))); - assert!(matches!(iter.next(), Some(Ok(TestSpec::Int(2))))); - assert!(matches!(iter.next(), Some(Ok(TestSpec::Root(Master::End))))); - assert!(matches!(iter.next(), None)); - } - - #[test] - pub fn specific_size_length_write_read() { - let mut dest = Cursor::new(Vec::new()); - let mut writer = TagWriter::new(&mut dest); - - writer.write(&TestSpec::Root(Master::Start)).unwrap(); - writer - .write_advanced(&TestSpec::Parent(Master::Start), WriteOptions::set_size_byte_count(8)) - .unwrap(); - writer.write(&TestSpec::Child(1)).unwrap(); - writer.write(&TestSpec::Child(2)).unwrap(); - writer.write(&TestSpec::Parent(Master::End)).unwrap(); - writer.write(&TestSpec::Int(2)).unwrap(); - writer.write(&TestSpec::Root(Master::End)).unwrap(); - - println!("{dest:x?}"); - dest.set_position(0); - - let mut iter = TagIterator::<_, TestSpec>::new(dest, &[]); - assert!(matches!(iter.next(), Some(Ok(TestSpec::Root(Master::Start))))); - assert!(matches!(iter.next(), Some(Ok(TestSpec::Parent(Master::Start))))); - assert!(matches!(iter.next(), Some(Ok(TestSpec::Child(1))))); - assert!(matches!(iter.next(), Some(Ok(TestSpec::Child(2))))); - assert!(matches!(iter.next(), Some(Ok(TestSpec::Parent(Master::End))))); - assert!(matches!(iter.next(), Some(Ok(TestSpec::Int(2))))); - assert!(matches!(iter.next(), Some(Ok(TestSpec::Root(Master::End))))); - assert!(matches!(iter.next(), None)); - } - - #[test] - pub fn eof_error_is_helpful() { - let tags: Vec = vec![ - TestSpec::Segment(Master::Start), - TestSpec::TrackType(0x01), - TestSpec::Cluster(Master::Start), - TestSpec::CueRefCluster(3), - TestSpec::Count(1), - TestSpec::Block(vec![0, 1, 2, 3, 4, 5, 6, 7, 8]), - TestSpec::Cluster(Master::End), - TestSpec::Segment(Master::End), - ]; - - let mut dest = Cursor::new(Vec::new()); - let mut writer = TagWriter::new(&mut dest); - - for tag in tags.iter() { - writer.write(tag).expect("Test shouldn't error"); - } - - println!("dest {:x?}", dest); - - let mut src = Cursor::new(dest.get_ref()[0..26].to_vec()); - let reader = TagIterator::new(&mut src, &[]); - let mut iter = reader - .into_iter() - .skip_while(|x: &Result| x.is_ok()); - - let err = iter.next().expect("Shouldn't have reached end of data"); - - match err.expect_err("Should be an error") { - TagIteratorError::UnexpectedEOF { - tag_start, - tag_id, - tag_size, - partial_data: _, - } => { - assert_eq!(tag_start, 20); - assert_eq!(tag_id, Some(TestSpec::Block(vec![]).get_id())); - assert_eq!(tag_size, Some(9)); - } - other => { - println!("{other:?}"); - assert!(false); - } - } - } - - #[test] - pub fn eof_on_tag_size() { - let tags: Vec = vec![ - TestSpec::Segment(Master::Start), - TestSpec::TrackType(0x01), - TestSpec::Cluster(Master::Start), - TestSpec::CueRefCluster(3), - TestSpec::Count(1), - TestSpec::Block(vec![0, 1, 2, 3, 4, 5, 6, 7]), - TestSpec::Block(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]), - TestSpec::Cluster(Master::End), - TestSpec::Segment(Master::End), - ]; - - let mut dest = Cursor::new(Vec::new()); - let mut writer = TagWriter::new(&mut dest); - - for tag in tags.iter() { - writer.write(tag).expect("Test shouldn't error"); - } - - println!("dest {:x?}", dest); - - let mut src = Cursor::new(dest.get_ref()[0..31].to_vec()); - let reader = TagIterator::with_capacity(&mut src, &[], 10); - let mut iter = reader - .into_iter() - .skip_while(|x: &Result| x.is_ok()); - - let err = iter.next().expect("Shouldn't have reached end of data"); - - match err.expect_err("Should be an error") { - TagIteratorError::UnexpectedEOF { - tag_start, - tag_id, - tag_size, - partial_data: _, - } => { - println!("got error - {tag_start}, {tag_id:?}, {tag_size:?}"); - assert_eq!(tag_start, 30); - assert_eq!(tag_id, Some(TestSpec::Block(vec![]).get_id())); - assert_eq!(tag_size, None); - } - other => { - println!("{other:?}"); - assert!(false); - } - } - } - - #[test] - pub fn allow_start_reading_not_at_root() { - let tags: Vec = vec![ - TestSpec::Segment(Master::Start), - TestSpec::TrackType(0x01), - TestSpec::Cluster(Master::Start), - TestSpec::CueRefCluster(3), - TestSpec::Count(1), - TestSpec::Block(vec![0, 1, 2, 3, 4, 5, 6, 7, 8]), - TestSpec::Cluster(Master::End), - TestSpec::Segment(Master::End), - ]; - - let mut dest = Cursor::new(Vec::new()); - let mut writer = TagWriter::new(&mut dest); - - for tag in tags.iter() { - writer.write(tag).expect("Test shouldn't error"); - } - - println!("dest {:x?}", dest); - - let mut src = Cursor::new(dest.get_ref()[8..].to_vec()); - let reader: TagIterator<_, TestSpec> = TagIterator::new(&mut src, &[]); - reader.for_each(|t| assert!(t.is_ok())); - } - - #[test] - pub fn validate_global_hierarchies() { - let tags: Vec = vec![ - TestSpec::Ebml(Master::Start), - TestSpec::Ebml(Master::End), - TestSpec::Void(vec![0xa0]), - TestSpec::Segment(Master::Start), - TestSpec::Crc32(vec![0x01]), - TestSpec::TrackType(0x01), - TestSpec::Cluster(Master::Start), - TestSpec::Crc32(vec![0x02]), - TestSpec::Count(1), - TestSpec::Cluster(Master::End), - TestSpec::Segment(Master::End), - ]; - - let mut dest = Cursor::new(Vec::new()); - let mut writer = TagWriter::new(&mut dest); - - for tag in tags.iter() { - writer.write(tag).expect("Test shouldn't error"); - } - - println!("dest {:?}", dest); - - let mut src = Cursor::new(dest.get_ref().to_vec()); - let reader = TagIterator::new(&mut src, &[]); - let read_tags: Vec = reader.into_iter().map(|t| t.unwrap()).collect(); - - println!("tags {:?}", read_tags); - - for i in 0..read_tags.len() { - assert_eq!(tags[i], read_tags[i]); - } - } -} diff --git a/crates/ebml-iterable/tests/test_spec.rs b/crates/ebml-iterable/tests/test_spec.rs deleted file mode 100644 index 41ac96f7d..000000000 --- a/crates/ebml-iterable/tests/test_spec.rs +++ /dev/null @@ -1,214 +0,0 @@ -// use ebml_iterable_specification_derive::easy_ebml; -// easy_ebml!( -// pub enum TestSpec { -// Root : Master = 0x81, -// Root/Int : UnsignedInt = 0x4101, -// Root/String : Utf8 = 0x4102, -// Root/Parent : Master = 0x4103, -// Root/Parent/Child: UnsignedInt = 0x210301, - -// Ebml : Master = 0x1a45dfa3, -// Segment : Master = 0x18538067, -// Segment/TrackType : UnsignedInt = 0x83, -// Segment/Cluster : Master = 0x1F43B675, -// Segment/Cluster/CueRefCluster : UnsignedInt = 0x97, -// Segment/Cluster/Count : UnsignedInt = 0x4100, -// Segment/Cluster/Block : Binary = 0xa1, -// Segment/Cluster/SimpleBlock : Binary = 0xa3, -// } -// ) - -use ebml_iterable::specs::TagDataType; - -#[derive(Clone, Debug, PartialEq)] -// Recursive expansion of ebml_specification! macro -// ================================================= - -pub enum TestSpec { - Root(ebml_iterable::specs::Master), - Int(u64), - String(String), - Parent(ebml_iterable::specs::Master), - Child(u64), - Ebml(ebml_iterable::specs::Master), - Segment(ebml_iterable::specs::Master), - TrackType(u64), - Cluster(ebml_iterable::specs::Master), - CueRefCluster(u64), - Count(u64), - Block(::std::vec::Vec), - SimpleBlock(::std::vec::Vec), - Crc32(::std::vec::Vec), - Void(::std::vec::Vec), - RawTag(u64, ::std::vec::Vec), -} -impl ebml_iterable::specs::EbmlSpecification for TestSpec { - fn get_tag_data_type(id: u64) -> Option { - match id { - 129u64 => Some(TagDataType::Master), - 16641u64 => Some(TagDataType::UnsignedInt), - 16642u64 => Some(TagDataType::Utf8), - 16643u64 => Some(TagDataType::Master), - 2163457u64 => Some(TagDataType::UnsignedInt), - 440786851u64 => Some(TagDataType::Master), - 408125543u64 => Some(TagDataType::Master), - 131u64 => Some(TagDataType::UnsignedInt), - 524531317u64 => Some(TagDataType::Master), - 151u64 => Some(TagDataType::UnsignedInt), - 16640u64 => Some(TagDataType::UnsignedInt), - 161u64 => Some(TagDataType::Binary), - 163u64 => Some(TagDataType::Binary), - 191u64 => Some(ebml_iterable::specs::TagDataType::Binary), - 236u64 => Some(ebml_iterable::specs::TagDataType::Binary), - _ => None, - } - } - fn get_path_by_id(id: u64) -> &'static [ebml_iterable::specs::PathPart] { - match id { - 16641u64 => &[ebml_iterable::specs::PathPart::Id(129u64)], - 16642u64 => &[ebml_iterable::specs::PathPart::Id(129u64)], - 16643u64 => &[ebml_iterable::specs::PathPart::Id(129u64)], - 2163457u64 => &[ - ebml_iterable::specs::PathPart::Id(129u64), - ebml_iterable::specs::PathPart::Id(16643u64), - ], - 131u64 => &[ebml_iterable::specs::PathPart::Id(408125543u64)], - 524531317u64 => &[ebml_iterable::specs::PathPart::Id(408125543u64)], - 151u64 => &[ - ebml_iterable::specs::PathPart::Id(408125543u64), - ebml_iterable::specs::PathPart::Id(524531317u64), - ], - 16640u64 => &[ - ebml_iterable::specs::PathPart::Id(408125543u64), - ebml_iterable::specs::PathPart::Id(524531317u64), - ], - 161u64 => &[ - ebml_iterable::specs::PathPart::Id(408125543u64), - ebml_iterable::specs::PathPart::Id(524531317u64), - ], - 163u64 => &[ - ebml_iterable::specs::PathPart::Id(408125543u64), - ebml_iterable::specs::PathPart::Id(524531317u64), - ], - 191u64 => &[ebml_iterable::specs::PathPart::Global((Some(1u64), None))], - 236u64 => &[ebml_iterable::specs::PathPart::Global((None, None))], - _ => &[], - } - } - fn get_unsigned_int_tag(id: u64, data: u64) -> Option { - match id { - 16641u64 => Some(TestSpec::Int(data)), - 2163457u64 => Some(TestSpec::Child(data)), - 131u64 => Some(TestSpec::TrackType(data)), - 151u64 => Some(TestSpec::CueRefCluster(data)), - 16640u64 => Some(TestSpec::Count(data)), - _ => None, - } - } - fn get_signed_int_tag(id: u64, _data: i64) -> Option { - match id { - _ => None, - } - } - fn get_utf8_tag(id: u64, data: String) -> Option { - match id { - 16642u64 => Some(TestSpec::String(data)), - _ => None, - } - } - fn get_binary_tag(id: u64, data: &[u8]) -> Option { - match id { - 161u64 => Some(TestSpec::Block(data.to_vec())), - 163u64 => Some(TestSpec::SimpleBlock(data.to_vec())), - 191u64 => Some(TestSpec::Crc32(data.to_vec())), - 236u64 => Some(TestSpec::Void(data.to_vec())), - _ => None, - } - } - fn get_float_tag(id: u64, _data: f64) -> Option { - match id { - _ => None, - } - } - fn get_master_tag(id: u64, data: ebml_iterable::specs::Master) -> Option { - match id { - 129u64 => Some(TestSpec::Root(data)), - 16643u64 => Some(TestSpec::Parent(data)), - 440786851u64 => Some(TestSpec::Ebml(data)), - 408125543u64 => Some(TestSpec::Segment(data)), - 524531317u64 => Some(TestSpec::Cluster(data)), - _ => None, - } - } - fn get_raw_tag(id: u64, data: &[u8]) -> TestSpec { - TestSpec::RawTag(id, data.to_vec()) - } -} -impl ebml_iterable::specs::EbmlTag for TestSpec { - fn get_id(&self) -> u64 { - match self { - TestSpec::Root(_) => 129u64, - TestSpec::Int(_) => 16641u64, - TestSpec::String(_) => 16642u64, - TestSpec::Parent(_) => 16643u64, - TestSpec::Child(_) => 2163457u64, - TestSpec::Ebml(_) => 440786851u64, - TestSpec::Segment(_) => 408125543u64, - TestSpec::TrackType(_) => 131u64, - TestSpec::Cluster(_) => 524531317u64, - TestSpec::CueRefCluster(_) => 151u64, - TestSpec::Count(_) => 16640u64, - TestSpec::Block(_) => 161u64, - TestSpec::SimpleBlock(_) => 163u64, - TestSpec::Crc32(_) => 191u64, - TestSpec::Void(_) => 236u64, - TestSpec::RawTag(id, _data) => *id, - } - } - fn as_unsigned_int(&self) -> Option<&u64> { - match self { - TestSpec::Int(val) => Some(val), - TestSpec::Child(val) => Some(val), - TestSpec::TrackType(val) => Some(val), - TestSpec::CueRefCluster(val) => Some(val), - TestSpec::Count(val) => Some(val), - _ => None, - } - } - fn as_signed_int(&self) -> Option<&i64> { - match self { - _ => None, - } - } - fn as_utf8(&self) -> Option<&str> { - match self { - TestSpec::String(val) => Some(val), - _ => None, - } - } - fn as_binary(&self) -> Option<&[u8]> { - match self { - TestSpec::Block(val) => Some(val), - TestSpec::SimpleBlock(val) => Some(val), - TestSpec::Crc32(val) => Some(val), - TestSpec::Void(val) => Some(val), - TestSpec::RawTag(_id, data) => Some(data), - _ => None, - } - } - fn as_float(&self) -> Option<&f64> { - match self { - _ => None, - } - } - fn as_master(&self) -> Option<&ebml_iterable::specs::Master> { - match self { - TestSpec::Root(val) => Some(val), - TestSpec::Parent(val) => Some(val), - TestSpec::Ebml(val) => Some(val), - TestSpec::Segment(val) => Some(val), - TestSpec::Cluster(val) => Some(val), - _ => None, - } - } -}