From 4b57c4a4caf79e766021f356ed7f17621db3ecd8 Mon Sep 17 00:00:00 2001 From: Sirac Ozmen Date: Wed, 4 Nov 2026 06:00:10 +0300 Subject: [PATCH 1/2] feat: refuse an iterator that no init call set up --- .github/workflows/ci.yml | 2 +- CMakeLists.txt | 5 +- docs/internals/overview.md | 2 +- docs/internals/segmentation.md | 4 +- docs/operations/checks.md | 2 +- docs/user/segmentation.md | 2 +- docs/user/shaping.md | 2 +- include/maul-unicode/script.h | 13 +- include/maul-unicode/segment.h | 35 ++-- src/cursor.h | 27 +++ src/grapheme.c | 7 + src/grapheme.h | 4 + src/line.c | 24 +++ src/script_runs.c | 100 ++++++--- src/script_state.h | 35 ++++ src/segment.c | 128 ++++++++++-- src/segmenter.h | 15 ++ src/sentence.c | 9 + src/word.c | 9 + test/fuzz_iterators.c | 204 +++++++++++++++++++ test/seeds/fuzz_iterators/grapheme-more | 1 + test/seeds/fuzz_iterators/line-breaker | 1 + test/seeds/fuzz_iterators/script | 1 + test/seeds/fuzz_iterators/script-overwritten | 1 + test/seeds/fuzz_iterators/sentence | 1 + test/seeds/fuzz_iterators/word-overrides | 1 + test/test_script.c | 30 +++ test/test_segment.c | 48 +++++ tools/modules.txt | 7 +- 29 files changed, 648 insertions(+), 72 deletions(-) create mode 100644 src/script_state.h create mode 100644 test/fuzz_iterators.c create mode 100644 test/seeds/fuzz_iterators/grapheme-more create mode 100644 test/seeds/fuzz_iterators/line-breaker create mode 100644 test/seeds/fuzz_iterators/script create mode 100644 test/seeds/fuzz_iterators/script-overwritten create mode 100644 test/seeds/fuzz_iterators/sentence create mode 100644 test/seeds/fuzz_iterators/word-overrides diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9955fae..e6f17c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -248,7 +248,7 @@ jobs: # # A dispatch's fuzz-seconds must leave time for the minimize and # save, or the timeout kills the job and the corpus is lost: the - # ten targets run in three waves of four, and the night's 50 + # eleven targets run in three waves of four, and the night's 50 # minutes less about 10 for the build, the merge and the save leave # 40, so at most 800 seconds a wave. libFuzzer reads a negative # time as no limit. diff --git a/CMakeLists.txt b/CMakeLists.txt index 181ba17..83ae4bf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -224,7 +224,7 @@ if(MAUL_UNICODE_FUZZ) target_compile_options(maul-unicode PRIVATE -fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=all) target_link_options(maul-unicode PUBLIC -fsanitize=address,undefined -fno-sanitize-recover=all) - set(MUNI_FUZZ_AREAS encoding segment bidi bidi_long script) + set(MUNI_FUZZ_AREAS encoding segment bidi bidi_long script iterators) if(MAUL_UNICODE_NORMALIZATION) list(APPEND MUNI_FUZZ_AREAS normalize) endif() @@ -249,6 +249,9 @@ if(MAUL_UNICODE_FUZZ) target_link_options(fuzz_${area} PRIVATE -fsanitize=fuzzer,address,undefined -fno-sanitize-recover=all) endforeach() + # fuzz_iterators sets the fields of the iterators' state, which only + # src/ declares. + target_include_directories(fuzz_iterators PRIVATE src) # ICU is an oracle for the tests alone, never linked into the library. if(TARGET fuzz_icu) find_package(PkgConfig REQUIRED) diff --git a/docs/internals/overview.md b/docs/internals/overview.md index 5556a68..6362dee 100644 --- a/docs/internals/overview.md +++ b/docs/internals/overview.md @@ -77,7 +77,7 @@ Outside the promise: - `MUNI_MAX_SCRIPT_EXTENSIONS`, `MUNI_MAX_RESOLVED_SCRIPTS` and `MUNI_MAX_DECOMPOSITION` hold for one Unicode version and may rise in a minor release; the calls always report the count they need. - The size of the iterator structs across minor releases. - Everything outside `include/`: `src/`, the generated tables and their sizes, speed, the tools, tests, samples and benchmarks, and the HarfBuzz versions the adapter builds with. -- What a call does when its caller breaks a header's contract, such as a workspace that overlaps its text or an iterator that was never initialized. +- What a call does when its caller breaks a header's contract, such as a workspace that overlaps its text, or an iterator whose bytes were changed by hand while keeping the cookie its init call wrote. An iterator no init call set up is refused with `muni_errorInvalid`, which is promised. The library reads and writes no asset or state format, so it promises none, and its headers name no other Maul library that would have to reach 1.0.0 first. diff --git a/docs/internals/segmentation.md b/docs/internals/segmentation.md index 9c232b4..54eb1a8 100644 --- a/docs/internals/segmentation.md +++ b/docs/internals/segmentation.md @@ -8,7 +8,9 @@ UAX #29 defines three segmentations of the same shape: walk the text and decide, The public struct is opaque storage. [`src/segment.c`](../../src/segment.c) copies the internal `muniSegmenter` in and out of it with `memcpy`, which keeps every access within the C aliasing rules, and dispatches on the kind once per boundary, not per code point. -Its size follows the state: a `static_assert` in [`src/segmenter.h`](../../src/segmenter.h) keeps `muniSegmenter` within it, so a change that grows the state fails to build until the public iterator grows with it. On 64-bit targets `muniSegmenter` fills all 128 bytes, with a line iterator's strictness and writing system packed in the byte after the hold's kind and the class overrides' pointer in the last 8. The next switch or piece of state therefore grows the public iterator, which changes the ABI, so its pull request's title carries `!`. +The caller holds that storage, so every call checks what it copied out before it reads it, as the family's rule on state a caller holds asks ([maul-conventions.md](maul-conventions.md)): the cookie an init call writes in the cursor, every count against the storage it counts (the cursor's pending bytes and position, the override count), every class, state and hold kind against the values its rule set writes, and every flag's byte as a 0 or 1 before the copy is read as a `bool`. It also checks what the assertions downstream rely on: a waiting cursor holds less than a code point, and a hold lies before the cursor. An iterator no init call set up is refused with `muni_errorInvalid`, so no bit pattern makes a call read outside the state; the pointers it keeps (the text, the overrides, the caller's breaker) cannot be checked, and one forged there is outside the contract. The script iterator ([`src/script_runs.c`](../../src/script_runs.c)) checks its state the same way. The checks run once per call, not per code point. `test/fuzz_iterators.c` sets every byte outside those pointers, and the lengths and counts that go with them, from its input and calls every function again. + +Its size follows the state: a `static_assert` in [`src/segmenter.h`](../../src/segmenter.h) keeps `muniSegmenter` within it, so a change that grows the state fails to build until the public iterator grows with it, and on 64-bit targets a second one keeps the iterator from growing a whole `uint64_t` more than the state needs. On 64-bit targets `muniSegmenter` fills all 128 bytes, with the cookie in the two bytes that were the cursor's padding, a line iterator's strictness and writing system packed in the byte after the hold's kind and the class overrides' pointer in the last 8. The next switch or piece of state therefore grows the public iterator, which changes the ABI; from 1.0.0 that is allowed in a minor release, so its pull request is a `feat` with no `!`. ## The engine and its holds diff --git a/docs/operations/checks.md b/docs/operations/checks.md index ca205c0..63fe0f5 100644 --- a/docs/operations/checks.md +++ b/docs/operations/checks.md @@ -45,7 +45,7 @@ A test that fails without a fault in the change under test is quarantined, as [m Every fuzz target (`test/fuzz_.c`) checks properties that must hold for any input rather than known answers: no crash, no broken assertion, piecewise iteration giving the whole-text answer, round trips, idempotence where the standard promises it, agreement between parts (validation and conversion, normalization and its quick check, NFKC_Casefold and the normal forms), and output through a short buffer matching output through a large one. `fuzz_icu` compares with ICU, a test-only oracle, where both Unicode versions agree; its header lists what it leaves out and why. The targets take at most 256 bytes, but `fuzz_bidi_long` repeats its input into a text of 2 KiB to 128 KiB, so work that grows faster than the text runs past libFuzzer's 30-second timeout and is a finding. -Each target's corpus carries over from run to run in one Actions cache entry, `fuzz-corpus-`, holding a directory per target. Every run restores the latest entry, a pull request's included, and adds the committed seeds, which are the floor. After fuzzing, runs on `main` and at night minimize each target's corpus through `tools/fuzz_target.sh --merge` into a fresh directory, swap it in only when the merge finished, and save a new entry. A clean merge prints one line, the target and the size of its merged corpus; an input that crashes the target during the merge fails the step and prints the same report as a fuzzing finding, the target, the kind of finding and the sanitizers' report, never the input. A run whose fuzzing or merge found something saves nothing. A pull request's run reads the corpus too, so a change that breaks a harness's promise fails on its own pull request and not later on `main`. A pull request, or a dispatch on any other branch, never minimizes or saves it, so only `main` writes the corpus. A target whose minimized corpus passes 25 MB fails the step, and the corpora are not saved, so one target cannot crowd the others out of the cache. The `prune-caches` job then deletes the entry the new one replaced. The organization's other libraries run one fuzz job per target, each with its own cache. This repository has ten targets in one job, four at a time, because ten more containers per run would cost more runner time than the targets' corpora need, and the corpora are still kept and minimized separately. +Each target's corpus carries over from run to run in one Actions cache entry, `fuzz-corpus-`, holding a directory per target. Every run restores the latest entry, a pull request's included, and adds the committed seeds, which are the floor. After fuzzing, runs on `main` and at night minimize each target's corpus through `tools/fuzz_target.sh --merge` into a fresh directory, swap it in only when the merge finished, and save a new entry. A clean merge prints one line, the target and the size of its merged corpus; an input that crashes the target during the merge fails the step and prints the same report as a fuzzing finding, the target, the kind of finding and the sanitizers' report, never the input. A run whose fuzzing or merge found something saves nothing. A pull request's run reads the corpus too, so a change that breaks a harness's promise fails on its own pull request and not later on `main`. A pull request, or a dispatch on any other branch, never minimizes or saves it, so only `main` writes the corpus. A target whose minimized corpus passes 25 MB fails the step, and the corpora are not saved, so one target cannot crowd the others out of the cache. The `prune-caches` job then deletes the entry the new one replaced. The organization's other libraries run one fuzz job per target, each with its own cache. This repository has eleven targets in one job, four at a time, because eleven more containers per run would cost more runner time than the targets' corpora need, and the corpora are still kept and minimized separately. A crash a fuzzer finds becomes a test in the suite of its module (see [development.md](development.md#reproducing-a-fuzz-finding)). diff --git a/docs/user/segmentation.md b/docs/user/segmentation.md index 62ab7ee..0042cec 100644 --- a/docs/user/segmentation.md +++ b/docs/user/segmentation.md @@ -1,6 +1,6 @@ # Characters, words, sentences and lines -`segment.h` finds the boundaries of grapheme clusters, words and sentences (UAX #29) and the line break opportunities (UAX #14). All four use one iterator type, `muniSegmentIterator`, a plain struct the caller keeps on the stack that needs no cleanup. +`segment.h` finds the boundaries of grapheme clusters, words and sentences (UAX #29) and the line break opportunities (UAX #14). All four use one iterator type, `muniSegmentIterator`, a plain struct the caller keeps on the stack that needs no cleanup. A copy of an initialized iterator is one too, and every call refuses an iterator no init call set up, such as a zeroed one, with `muni_errorInvalid`. ## Grapheme clusters, words and sentences diff --git a/docs/user/shaping.md b/docs/user/shaping.md index a407112..27b142c 100644 --- a/docs/user/shaping.md +++ b/docs/user/shaping.md @@ -18,7 +18,7 @@ if (muniInitScriptIterator(&scripts, text, length, false) == muni_success) } ``` -Like the segment iterators, it takes text in pieces through `muniScriptIterator_Feed` after `muni_needMoreText`. `muniFindScriptRuns` writes the runs of a whole text into an array. +Like the segment iterators, it takes text in pieces through `muniScriptIterator_Feed` after `muni_needMoreText`, and its calls refuse an iterator no init call set up with `muni_errorInvalid`. `muniFindScriptRuns` writes the runs of a whole text into an array. A text layout pipeline itemizes by paragraph ([bidi](bidi.md)), then by script and bidi run, then by font, and shapes each item. diff --git a/include/maul-unicode/script.h b/include/maul-unicode/script.h index 531fbe4..486e550 100644 --- a/include/maul-unicode/script.h +++ b/include/maul-unicode/script.h @@ -28,7 +28,10 @@ extern "C" #endif // An iterator over script runs. Its contents are private; it is plain - // data the caller keeps anywhere and needs no cleanup. + // data the caller keeps anywhere and needs no cleanup. A copy of an + // iterator an init call set up is one too; every call refuses one no + // init call set up, such as a zeroed one or one copied from other + // bytes, as an invalid argument. typedef struct muniScriptIterator { uint64_t opaque[26]; @@ -65,8 +68,9 @@ extern "C" /// @param moreFollows true when still more pieces will be fed. /// @return `muni_success`; `muni_errorState` for an iterator that did /// not ask for more text, such as one initialized without more - /// text to follow; `muni_errorInvalid` for a NULL argument. - /// `muni_errorInvalid` is reported before `muni_errorState`. + /// text to follow; `muni_errorInvalid` for a NULL argument or + /// an iterator no init call set up. `muni_errorInvalid` is + /// reported before `muni_errorState`. /// @par Thread safety /// Safe from any thread; an iterator is used by one thread at a time. MUNI_NODISCARD MUNI_API muniResult muniScriptIterator_Feed(muniScriptIterator* iterator, @@ -83,7 +87,8 @@ extern "C" /// returns `muni_success`. /// @return `muni_success` with a run; `muni_done` after the last one; /// `muni_needMoreText` when the run's end or script depends on - /// text not yet fed; `muni_errorInvalid` for a NULL argument. + /// text not yet fed; `muni_errorInvalid` for a NULL argument or + /// an iterator no init call set up. /// @par Thread safety /// Safe from any thread; an iterator is used by one thread at a time. MUNI_NODISCARD MUNI_API muniResult muniScriptIterator_NextRun(muniScriptIterator* iterator, diff --git a/include/maul-unicode/segment.h b/include/maul-unicode/segment.h index 8192b7a..3a27577 100644 --- a/include/maul-unicode/segment.h +++ b/include/maul-unicode/segment.h @@ -36,7 +36,9 @@ extern "C" // An iterator over grapheme cluster, word or sentence boundaries or // line break opportunities. Its contents are private; it is plain data // the caller keeps anywhere, usually on the stack, and needs no - // cleanup. + // cleanup. A copy of an iterator an init call set up is one too; every + // call refuses one no init call set up, such as a zeroed one or one + // copied from other bytes, as an invalid argument. typedef struct muniSegmentIterator { uint64_t opaque[16]; @@ -192,7 +194,8 @@ extern "C" /// @return `muni_success`; `muni_errorState` for an iterator already /// asked for a break; `muni_errorBusy` when called from inside /// the iterator's word segmenter; `muni_errorInvalid` for a NULL - /// iterator or another kind of iterator. `muni_errorInvalid` is + /// iterator, one no init call set up or another kind of + /// iterator. `muni_errorInvalid` is /// reported before `muni_errorBusy`, and that before /// `muni_errorState`. /// @par Thread safety @@ -236,7 +239,8 @@ extern "C" /// assigns. /// @return `muni_success`; `muni_errorState` for an iterator already /// asked for a boundary; `muni_errorInvalid` for a NULL - /// iterator, NULL overrides with a nonzero count, another kind + /// iterator or one no init call set up, NULL overrides with a + /// nonzero count, another kind /// of iterator, a count over 0x10F800, a code point past /// U+10FFFF or a surrogate, code points out of order or /// repeated, or a value that is not a class of the iterator's @@ -257,7 +261,8 @@ extern "C" /// @return `muni_success`; `muni_errorState` for an iterator already /// asked for a break; `muni_errorBusy` when called from inside /// the iterator's word segmenter; `muni_errorInvalid` for a NULL - /// iterator, another kind of iterator or an unknown strictness. + /// iterator, one no init call set up, another kind of iterator + /// or an unknown strictness. /// `muni_errorInvalid` is reported before `muni_errorBusy`, and /// that before `muni_errorState`. /// @par Thread safety @@ -270,7 +275,8 @@ extern "C" /// @param iterator The iterator. /// @return The strictness muniSegmentIterator_SetLineBreakStrictness /// set, muni_lineBreakNormal when none was set, and for a NULL - /// iterator or another kind of iterator. + /// iterator, one no init call set up or another kind of + /// iterator. /// @par Thread safety /// Safe from any thread; an iterator is used by one thread at a time. MUNI_API muniLineBreakStrictness @@ -284,7 +290,8 @@ extern "C" /// @return `muni_success`; `muni_errorState` for an iterator already /// asked for a break; `muni_errorBusy` when called from inside /// the iterator's word segmenter; `muni_errorInvalid` for a NULL - /// iterator, another kind of iterator or an unknown writing + /// iterator, one no init call set up, another kind of iterator + /// or an unknown writing /// system. `muni_errorInvalid` is reported before /// `muni_errorBusy`, and that before `muni_errorState`. /// @par Thread safety @@ -296,8 +303,8 @@ extern "C" /// /// @param iterator The iterator. /// @return The writing system muniSegmentIterator_SetWritingSystem set, - /// muni_writingOther when none was set, and for a NULL iterator - /// or another kind of iterator. + /// muni_writingOther when none was set, and for a NULL iterator, + /// one no init call set up or another kind of iterator. /// @par Thread safety /// Safe from any thread; an iterator is used by one thread at a time. MUNI_API muniWritingSystem @@ -314,9 +321,9 @@ extern "C" /// not ask for more text, such as one initialized without more /// text to follow or one fed before it asked; `muni_errorBusy` /// when called from inside the iterator's word segmenter; - /// `muni_errorInvalid` for a NULL argument. `muni_errorInvalid` - /// is reported before `muni_errorBusy`, and that before - /// `muni_errorState`. + /// `muni_errorInvalid` for a NULL argument or an iterator no + /// init call set up. `muni_errorInvalid` is reported before + /// `muni_errorBusy`, and that before `muni_errorState`. /// @par Thread safety /// Safe from any thread; an iterator is used by one thread at a time. MUNI_NODISCARD MUNI_API muniResult muniSegmentIterator_Feed(muniSegmentIterator* iterator, @@ -333,7 +340,8 @@ extern "C" /// one; `muni_needMoreText` when the next boundary depends on /// text not yet fed; `muni_errorBusy` when called from inside /// the iterator's word segmenter, and then the iterator is left - /// as it was; `muni_errorInvalid` for a NULL argument. + /// as it was; `muni_errorInvalid` for a NULL argument or an + /// iterator no init call set up. /// @par Thread safety /// Safe from any thread; an iterator is used by one thread at a time. MUNI_NODISCARD MUNI_API muniResult muniSegmentIterator_NextBreak(muniSegmentIterator* iterator, @@ -390,7 +398,8 @@ extern "C" /// `muni_success`. /// @return `muni_success`; `muni_errorBusy` when called from inside the /// iterator's word segmenter; `muni_errorInvalid` for a NULL - /// argument or an iterator of another kind. + /// argument, an iterator no init call set up or one of another + /// kind. /// @par Thread safety /// Safe from any thread; an iterator is used by one thread at a time. MUNI_NODISCARD MUNI_API muniResult diff --git a/src/cursor.h b/src/cursor.h index 34a9d5f..98d9e0a 100644 --- a/src/cursor.h +++ b/src/cursor.h @@ -23,10 +23,37 @@ typedef struct muniCursor uint8_t pending[4]; // the start of a sequence the previous piece cut off uint8_t pendingCount; // how many bytes of it there are bool moreFollows; // whether more pieces will come + uint16_t cookie; // the iterator's, written by its init call and checked by every call } muniCursor; void muniCursorInit(muniCursor* cursor, const char* text, size_t length, bool moreFollows); +// Whether the byte at offset in raw, state the caller holds, is a flag's +// 0 or 1. The iterators check every flag so before the copy of the state +// that holds it is read: any other byte would be no bool at all. +static inline bool muniIsFlagByte(const void* raw, size_t offset) +{ + return ((const unsigned char*)raw)[offset] <= 1; +} + +// Whether a cursor copied out of raw, from offset at, keeps its counts +// within its piece and its pending bytes, and its flag a 0 or 1; when it +// is waiting for a piece, whether more were announced and less than a +// code point is unread, as muniCursorFeed relies on. +static inline bool muniCursorValid(const muniCursor* cursor, const void* raw, size_t at, + bool waiting) +{ + if (!muniIsFlagByte(raw, at + offsetof(muniCursor, moreFollows)) || + cursor->pendingCount > sizeof(cursor->pending) || cursor->position > cursor->length || + (cursor->text == nullptr && cursor->length != 0)) + { + return false; + } + return !waiting || + (cursor->moreFollows && + cursor->pendingCount + (cursor->length - cursor->position) < sizeof(cursor->pending)); +} + // Hands the cursor its next piece. Bytes still unread in the current // piece are kept as pending. Only a cursor that answered // muni_needMoreText may be fed: more pieces were announced, and at most diff --git a/src/grapheme.c b/src/grapheme.c index 5f4dd9a..7bd98fe 100644 --- a/src/grapheme.c +++ b/src/grapheme.c @@ -122,6 +122,13 @@ void muniGraphemeAdvance(muniGraphemeRules* state, uint8_t value, uint32_t codeP state->previous = value; } +bool muniGraphemeRulesValid(const muniGraphemeRules* state, const void* raw, size_t at) +{ + return state->previous <= muni_gcbLvt && state->emoji <= EmojiJoined && + muniIsFlagByte(raw, at + offsetof(muniGraphemeRules, linker)) && + muniIsFlagByte(raw, at + offsetof(muniGraphemeRules, oddRegional)); +} + static void Absorb(muniSegmenter* segmenter, uint8_t value, uint32_t codePoint) { muniGraphemeAdvance(&segmenter->rules.grapheme, value, codePoint); diff --git a/src/grapheme.h b/src/grapheme.h index 226fa9c..cd36dac 100644 --- a/src/grapheme.h +++ b/src/grapheme.h @@ -16,4 +16,8 @@ bool muniGraphemeIsBoundary(const muniGraphemeRules* state, uint8_t after, uint3 // Moves the state past a code point whose Grapheme_Cluster_Break is value. void muniGraphemeAdvance(muniGraphemeRules* state, uint8_t value, uint32_t codePoint); +// Whether a state copied out of raw, from offset at, holds what Advance +// writes: a Grapheme_Cluster_Break value, an emoji state, flags of 0 or 1. +bool muniGraphemeRulesValid(const muniGraphemeRules* state, const void* raw, size_t at); + #endif // MAUL_UNICODE_SRC_GRAPHEME_H diff --git a/src/line.c b/src/line.c index 4a48b3f..175541f 100644 --- a/src/line.c +++ b/src/line.c @@ -7,6 +7,7 @@ // segment.c sends the other settings to line_tailored.c and // line_anywhere.c, once per call. +#include "grapheme.h" #include "line_rules.h" static uint8_t Lookup(const muniSegmenter* segmenter, uint32_t codePoint) @@ -23,3 +24,26 @@ muniResult muniNextLineSegment(muniSegmenter* segmenter, size_t* offsetOut) { return muniSegmenterNext(segmenter, offsetOut, Lookup, Decide, Resolve, Absorb, nullptr); } + +// The rules of every setting, checked here beside the default ones: +// breaking anywhere keeps the grapheme rules where the others keep the +// classes before the position. +bool muniLineRulesValid(const muniSegmenter* segmenter, const void* raw) +{ + const muniLineRules* state = &segmenter->rules.line; + if (state->actual > muni_lbHh) + { + return false; + } + if (muniLineStrictness(segmenter) == muni_lineBreakAnywhere) + { + return segmenter->holdKind == 0 && + muniGraphemeRulesValid(&state->grapheme, raw, + offsetof(muniSegmenter, rules.line.grapheme)); + } + return state->previous <= muni_lbHh && state->before <= muni_lbHh && state->lead <= muni_lbHh && + state->number <= NumberClosed && segmenter->holdKind <= HoldAksara && + muniIsFlagByte(raw, offsetof(muniSegmenter, rules.line.leadInitialQuote)) && + muniIsFlagByte(raw, offsetof(muniSegmenter, rules.line.sawInfix)) && + muniIsFlagByte(raw, offsetof(muniSegmenter, rules.line.oddRegional)); +} diff --git a/src/script_runs.c b/src/script_runs.c index 496aa16..1ed1fcc 100644 --- a/src/script_runs.c +++ b/src/script_runs.c @@ -10,35 +10,14 @@ // a closing bracket that matches one counts as a code point of that // script, so "(" and ")" around foreign text land in the same script. -#include "cursor.h" #include "invariant.h" +#include "script_state.h" #include "tables.h" #include "maul-unicode/script.h" #include -#define BRACKET_DEPTH 32 -#define NO_SCRIPT 0xFFu - -typedef struct Scripts -{ - uint64_t bits[4]; -} Scripts; - -typedef struct ScriptState -{ - muniCursor cursor; - Scripts candidates; // the scripts the run can still have - uint32_t brackets[BRACKET_DEPTH]; // closing bracket << 8 | Script index or NO_SCRIPT - uint8_t bracketCount; - uint8_t firstScript; // the Script of the run's first code point that has one - bool determined; // a code point with a real script is in the run - bool started; // a code point has been read - bool waiting; // the last call returned muni_needMoreText - bool finished; // the last run has been reported -} ScriptState; - // The public iterator's size follows the state: it holds the state on // every target, and on 64-bit targets, where the state is largest, it // has no whole uint64_t to spare. A change to the state that breaks @@ -265,11 +244,67 @@ static muniResult Next(ScriptState* state, muniScriptRun* runOut) } } -static ScriptState Load(const muniScriptIterator* iterator) +// What an init call writes in the cursor's cookie, in bytes that were its +// padding, so that an iterator no init call set up is refused instead of +// read. Its top bit is set, so a small count or an ASCII byte pattern is +// no cookie; the checks after it keep any other bit pattern safe too. +#define COOKIE 0xB3D0u + +// Whether a script index names one of muniScriptTags, or is NO_SCRIPT +// where that is allowed. +static bool IsScript(uint8_t script, bool none) { - ScriptState state; - memcpy(&state, iterator, sizeof(state)); - return state; + return script < muniScriptTagCount || (none && script == NO_SCRIPT); +} + +// Copies the iterator's state out of caller memory when it is one an +// init call set up and every flag, count and script in it is in range, +// so that no bit pattern makes a call read outside the state, the text +// it names or the script tables; false otherwise. An iterator with the +// cookie whose fields were changed by hand is outside the contract, and +// how its fields agree with each other is not checked again. +static bool Load(const muniScriptIterator* iterator, ScriptState* stateOut) +{ + memcpy(stateOut, iterator, sizeof(*stateOut)); + const ScriptState* state = stateOut; + if (state->cursor.cookie != COOKIE || + !muniIsFlagByte(iterator, offsetof(ScriptState, determined)) || + !muniIsFlagByte(iterator, offsetof(ScriptState, started)) || + !muniIsFlagByte(iterator, offsetof(ScriptState, waiting)) || + !muniIsFlagByte(iterator, offsetof(ScriptState, finished)) || + !muniCursorValid(&state->cursor, iterator, offsetof(ScriptState, cursor), state->waiting) || + state->bracketCount > BRACKET_DEPTH || !IsScript(state->firstScript, true)) + { + return false; + } + for (uint8_t k = 0; k < state->bracketCount; k++) + { + if (!IsScript((uint8_t)(state->brackets[k] & 0xFF), true)) + { + return false; + } + } + // No candidate past the last script, and at least one in a determined + // run, which Best relies on. + bool any = false; + for (unsigned word = 0; word < 4; word++) + { + uint64_t bits = state->candidates.bits[word]; + unsigned first = word * 64; + if (first >= muniScriptTagCount) + { + if (bits != 0) + { + return false; + } + } + else if (muniScriptTagCount - first < 64 && bits >> (muniScriptTagCount - first) != 0) + { + return false; + } + any = any || bits != 0; + } + return any || !state->determined; } static void Store(muniScriptIterator* iterator, const ScriptState* state) @@ -282,6 +317,7 @@ static void Start(ScriptState* state, const char* text, size_t length, bool more memset(state, 0, sizeof(*state)); state->firstScript = NO_SCRIPT; muniCursorInit(&state->cursor, text, length, moreFollows); + state->cursor.cookie = COOKIE; } muniResult muniInitScriptIterator(muniScriptIterator* iterator, const char* text, size_t length, @@ -305,7 +341,11 @@ muniResult muniScriptIterator_Feed(muniScriptIterator* iterator, const char* tex { return muni_errorInvalid; } - ScriptState state = Load(iterator); + ScriptState state; + if (!Load(iterator, &state)) + { + return muni_errorInvalid; + } if (!state.waiting) { return muni_errorState; @@ -322,7 +362,11 @@ muniResult muniScriptIterator_NextRun(muniScriptIterator* iterator, muniScriptRu { return muni_errorInvalid; } - ScriptState state = Load(iterator); + ScriptState state; + if (!Load(iterator, &state)) + { + return muni_errorInvalid; + } muniResult status = Next(&state, runOut); state.waiting = status == muni_needMoreText; Store(iterator, &state); diff --git a/src/script_state.h b/src/script_state.h new file mode 100644 index 0000000..b01160f --- /dev/null +++ b/src/script_state.h @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Sirac Ozmen +// +// The state of a script iterator, which script_runs.c keeps in the public +// iterator's opaque storage. Here so that a fuzz target can change the +// fields that hold no pointer and see every call refuse what is out of +// range. + +#ifndef MAUL_UNICODE_SRC_SCRIPT_STATE_H +#define MAUL_UNICODE_SRC_SCRIPT_STATE_H + +#include "cursor.h" + +#define BRACKET_DEPTH 32 +#define NO_SCRIPT 0xFFu + +typedef struct Scripts +{ + uint64_t bits[4]; +} Scripts; + +typedef struct ScriptState +{ + muniCursor cursor; + Scripts candidates; // the scripts the run can still have + uint32_t brackets[BRACKET_DEPTH]; // closing bracket << 8 | Script index or NO_SCRIPT + uint8_t bracketCount; + uint8_t firstScript; // the Script of the run's first code point that has one + bool determined; // a code point with a real script is in the run + bool started; // a code point has been read + bool waiting; // the last call returned muni_needMoreText + bool finished; // the last run has been reported +} ScriptState; + +#endif // MAUL_UNICODE_SRC_SCRIPT_STATE_H diff --git a/src/segment.c b/src/segment.c index e51d245..423ca1c 100644 --- a/src/segment.c +++ b/src/segment.c @@ -8,16 +8,77 @@ #include "maul-unicode/segment.h" #include "encoding.h" +#include "grapheme.h" #include "segmenter.h" #include #include -static muniSegmenter Load(const muniSegmentIterator* iterator) +// What an init call writes in the cursor's cookie, in bytes that were its +// padding, so that an iterator no init call set up is refused instead of +// read. Its top bit is set, so a small count or an ASCII byte pattern is +// no cookie; the checks after it keep any other bit pattern safe too. +#define COOKIE 0xC5E6u + +// The most overrides a table can hold: one per scalar value, since its +// code points strictly increase. +#define MAX_OVERRIDES 0x10F800u + +// The byte at offset in the caller's iterator; four flags are checked at +// once by taking their bytes together. +static inline uint8_t Byte(const muniSegmentIterator* iterator, size_t offset) { - muniSegmenter segmenter; - memcpy(&segmenter, iterator, sizeof(segmenter)); - return segmenter; + return ((const unsigned char*)iterator)[offset]; +} + +// Copies the iterator's state out of caller memory when it is one an +// init call set up and every flag, count and value in it is in range, so +// that no bit pattern makes a call read outside the state or the text +// and overrides it names; false otherwise. An iterator with the cookie +// whose fields were changed by hand is outside the contract, and how its +// fields agree with each other is not checked again. +static bool Load(const muniSegmentIterator* iterator, muniSegmenter* segmenterOut) +{ + memcpy(segmenterOut, iterator, sizeof(*segmenterOut)); + const muniSegmenter* segmenter = segmenterOut; + if (segmenter->cursor.cookie != COOKIE || segmenter->kind < muni_segmentGrapheme || + segmenter->kind > muni_segmentLine || segmenter->phase > muni_phaseBusy || + !muniCursorValid(&segmenter->cursor, iterator, offsetof(muniSegmenter, cursor), + segmenter->phase == muni_phaseWaiting) || + (Byte(iterator, offsetof(muniSegmenter, started)) | + Byte(iterator, offsetof(muniSegmenter, endReported)) | + Byte(iterator, offsetof(muniSegmenter, holding)) | + Byte(iterator, offsetof(muniSegmenter, mandatory))) > 1 || + (segmenter->complex.text != nullptr && segmenter->complex.next > segmenter->complex.length)) + { + return false; + } + // A boundary lies after a code point, and a hold before the cursor. + size_t offset = segmenter->cursor.offset; + if ((segmenter->started && offset == 0) || + (segmenter->holding && (segmenter->heldOffset == 0 || segmenter->heldOffset >= offset))) + { + return false; + } + switch (segmenter->kind) + { + case muni_segmentGrapheme: + return segmenter->lineSettings == 0 && segmenter->holdKind == 0 && + muniGraphemeRulesValid(&segmenter->rules.grapheme, iterator, + offsetof(muniSegmenter, rules.grapheme)); + case muni_segmentWord: + return segmenter->lineSettings == 0 && + segmenter->rules.word.overrideCount <= MAX_OVERRIDES && + (segmenter->rules.word.overrideCount == 0 || segmenter->overrides != nullptr) && + muniWordRulesValid(segmenter, iterator); + case muni_segmentSentence: + return segmenter->lineSettings == 0 && + segmenter->rules.sentence.overrideCount <= MAX_OVERRIDES && + (segmenter->rules.sentence.overrideCount == 0 || segmenter->overrides != nullptr) && + muniSentenceRulesValid(segmenter, iterator); + default: + return segmenter->lineSettings >> 3 == 0 && muniLineRulesValid(segmenter, iterator); + } } static void Store(muniSegmentIterator* iterator, const muniSegmenter* segmenter) @@ -49,6 +110,7 @@ static muniResult Init(muniSegmentIterator* iterator, uint8_t kind, const char* memset(&segmenter, 0, sizeof(segmenter)); segmenter.kind = kind; muniCursorInit(&segmenter.cursor, text, length, moreFollows); + segmenter.cursor.cookie = COOKIE; memset(iterator, 0, sizeof(*iterator)); Store(iterator, &segmenter); return muni_success; @@ -85,7 +147,11 @@ muniResult muniSegmentIterator_SetComplexBreaker(muniSegmentIterator* iterator, { return muni_errorInvalid; } - muniSegmenter segmenter = Load(iterator); + muniSegmenter segmenter; + if (!Load(iterator, &segmenter)) + { + return muni_errorInvalid; + } bool supported = segmenter.kind == muni_segmentLine || segmenter.kind == muni_segmentWord; if (!supported) { @@ -105,10 +171,6 @@ muniResult muniSegmentIterator_SetComplexBreaker(muniSegmentIterator* iterator, return muni_success; } -// The most overrides a table can hold: one per scalar value, since its -// code points strictly increase. -#define MAX_OVERRIDES 0x10F800u - // Whether a table of overrides is in strictly increasing order of code // point and names only code points and known classes: up to classCount. static bool ValidOverrides(const muniBreakClassOverride* overrides, size_t count, @@ -135,7 +197,11 @@ muniResult muniSegmentIterator_SetBreakClassOverrides(muniSegmentIterator* itera { return muni_errorInvalid; } - muniSegmenter segmenter = Load(iterator); + muniSegmenter segmenter; + if (!Load(iterator, &segmenter)) + { + return muni_errorInvalid; + } uint32_t* countField = nullptr; uint8_t classCount = 0; if (segmenter.kind == muni_segmentWord) @@ -176,7 +242,11 @@ static muniResult SetLineSettings(muniSegmentIterator* iterator, int32_t value, { return muni_errorInvalid; } - muniSegmenter segmenter = Load(iterator); + muniSegmenter segmenter; + if (!Load(iterator, &segmenter)) + { + return muni_errorInvalid; + } if (segmenter.kind != muni_segmentLine) { return muni_errorInvalid; @@ -208,7 +278,11 @@ muniSegmentIterator_GetLineBreakStrictness(const muniSegmentIterator* iterator) { return muni_lineBreakNormal; } - muniSegmenter segmenter = Load(iterator); + muniSegmenter segmenter; + if (!Load(iterator, &segmenter)) + { + return muni_lineBreakNormal; + } return segmenter.kind == muni_segmentLine ? muniLineStrictness(&segmenter) : muni_lineBreakNormal; } @@ -226,7 +300,11 @@ muniWritingSystem muniSegmentIterator_GetWritingSystem(const muniSegmentIterator { return muni_writingOther; } - muniSegmenter segmenter = Load(iterator); + muniSegmenter segmenter; + if (!Load(iterator, &segmenter)) + { + return muni_writingOther; + } return segmenter.kind == muni_segmentLine ? muniLineWritingSystem(&segmenter) : muni_writingOther; } @@ -238,7 +316,11 @@ muniResult muniSegmentIterator_Feed(muniSegmentIterator* iterator, const char* t { return muni_errorInvalid; } - muniSegmenter segmenter = Load(iterator); + muniSegmenter segmenter; + if (!Load(iterator, &segmenter)) + { + return muni_errorInvalid; + } if (segmenter.phase == muni_phaseBusy) { return muni_errorBusy; @@ -283,7 +365,11 @@ muniResult muniSegmentIterator_NextBreak(muniSegmentIterator* iterator, size_t* { return muni_errorInvalid; } - muniSegmenter segmenter = Load(iterator); + muniSegmenter segmenter; + if (!Load(iterator, &segmenter)) + { + return muni_errorInvalid; + } if (segmenter.phase == muni_phaseBusy) { return muni_errorBusy; @@ -302,7 +388,11 @@ muniResult muniSegmentIterator_NextLineBreak(muniSegmentIterator* iterator, size { return muni_errorInvalid; } - muniSegmenter segmenter = Load(iterator); + muniSegmenter segmenter; + if (!Load(iterator, &segmenter)) + { + return muni_errorInvalid; + } if (segmenter.kind != muni_segmentLine) { return muni_errorInvalid; @@ -380,7 +470,11 @@ muniResult muniSegmentIterator_GetWordKind(const muniSegmentIterator* iterator, { return muni_errorInvalid; } - muniSegmenter segmenter = Load(iterator); + muniSegmenter segmenter; + if (!Load(iterator, &segmenter)) + { + return muni_errorInvalid; + } if (segmenter.kind != muni_segmentWord) { return muni_errorInvalid; diff --git a/src/segmenter.h b/src/segmenter.h index 75671fa..ea46e7d 100644 --- a/src/segmenter.h +++ b/src/segmenter.h @@ -149,8 +149,15 @@ typedef struct muniSegmenter const muniBreakClassOverride* overrides; } muniSegmenter; +// The public iterator's size follows the segmenter: it holds it on every +// target, and on 64-bit targets, where the segmenter is largest, it has +// no whole uint64_t to spare. A change that breaks either resizes the +// public iterator, which a minor release may do. static_assert(sizeof(muniSegmenter) <= sizeof(muniSegmentIterator), "the public iterator must hold the segmenter"); +static_assert(sizeof(void*) < 8 || + sizeof(muniSegmentIterator) - sizeof(muniSegmenter) < sizeof(uint64_t), + "the public iterator must be no larger than the segmenter needs"); // The parts of lineSettings: a muni_lineBreak value in the low two bits, // a muni_writing value in the third. @@ -310,4 +317,12 @@ muniResult muniNextLineSegment(muniSegmenter* segmenter, size_t* offsetOut); muniResult muniNextLineTailoredSegment(muniSegmenter* segmenter, size_t* offsetOut); muniResult muniNextLineAnywhereSegment(muniSegmenter* segmenter, size_t* offsetOut); +// Whether the rules of a segmenter copied out of raw, state the caller +// holds, are ones the rule set writes: each class and state in range, +// each flag a 0 or 1 and the hold kind one it gives. Defined beside each +// rule set; the grapheme rules' is in grapheme.h. +bool muniWordRulesValid(const muniSegmenter* segmenter, const void* raw); +bool muniSentenceRulesValid(const muniSegmenter* segmenter, const void* raw); +bool muniLineRulesValid(const muniSegmenter* segmenter, const void* raw); + #endif // MAUL_UNICODE_SRC_SEGMENTER_H diff --git a/src/sentence.c b/src/sentence.c index f204f75..1e76333 100644 --- a/src/sentence.c +++ b/src/sentence.c @@ -160,3 +160,12 @@ muniResult muniNextSentenceSegment(muniSegmenter* segmenter, size_t* offsetOut) { return muniSegmenterNext(segmenter, offsetOut, Lookup, Decide, Resolve, Absorb, nullptr); } + +bool muniSentenceRulesValid(const muniSegmenter* segmenter, const void* raw) +{ + const muniSentenceRules* state = &segmenter->rules.sentence; + return state->actual <= muni_sbClose && state->previous <= muni_sbClose && + state->before <= muni_sbClose && state->ending <= EndingSpace && + segmenter->holdKind == 0 && + muniIsFlagByte(raw, offsetof(muniSegmenter, rules.sentence.aterm)); +} diff --git a/src/word.c b/src/word.c index d20c768..e5aefde 100644 --- a/src/word.c +++ b/src/word.c @@ -245,3 +245,12 @@ muniResult muniNextWordSegment(muniSegmenter* segmenter, size_t* offsetOut) { return muniSegmenterNext(segmenter, offsetOut, Lookup, Decide, Resolve, Absorb, Boundary); } + +bool muniWordRulesValid(const muniSegmenter* segmenter, const void* raw) +{ + const muniWordRules* state = &segmenter->rules.word; + return state->actual <= muni_wbWSegSpace && state->previous <= muni_wbWSegSpace && + state->before <= muni_wbWSegSpace && state->kind <= muni_wordIdeograph && + state->reportedKind <= muni_wordIdeograph && segmenter->holdKind == 0 && + muniIsFlagByte(raw, offsetof(muniSegmenter, rules.word.oddRegional)); +} diff --git a/test/fuzz_iterators.c b/test/fuzz_iterators.c new file mode 100644 index 0000000..09676f9 --- /dev/null +++ b/test/fuzz_iterators.c @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Sirac Ozmen +// +// libFuzzer target for the state the iterators keep in caller memory. The +// first input byte picks an iterator and its settings, the second how +// many calls it answers first, and each pair of bytes after them sets one +// byte of its state to a value. The pointers, and the lengths and counts +// that go with them (the text, the class overrides, the caller's +// breaker), are left alone: an iterator forged there is outside the +// contract. Every call on what is left must succeed or refuse, without +// reading or writing outside the state, the text or the overrides, and +// without tripping an assertion. + +#include "script_state.h" +#include "segmenter.h" + +#include "maul-unicode/script.h" +#include "maul-unicode/segment.h" + +#include +#include +#include + +// Thai for the breaker, a CR LF, an emoji sequence, brackets around +// Greek, Han, digits and spaces. +static const char s_text[] = + "\xE0\xB8\xAA\xE0\xB8\xA7\xE0\xB8\xB1\xE0\xB8\xAA\xE0\xB8\x94\xE0\xB8\xB5" + " ab\r\n\xF0\x9F\x91\xA9\xE2\x80\x8D\xF0\x9F\x92\xBB (\xCE\xB1\xCE\xB2) " + "\xE4\xBA\xBA 1,5% x"; + +static const muniBreakClassOverride s_overrides[] = {{'_', muni_wbExtendNumLet}, + {'-', muni_wbMidLetter}}; + +typedef struct Range +{ + size_t start; + size_t size; +} Range; + +static size_t Breaker(void* context, const char* run, size_t length, size_t from) +{ + (void)context; + (void)run; + return from + 3 < length ? from + 3 : length; +} + +static bool Inside(const Range* ranges, size_t count, size_t offset) +{ + for (size_t i = 0; i < count; i++) + { + if (offset - ranges[i].start < ranges[i].size) + { + return true; + } + } + return false; +} + +// Sets the state's bytes the input names, outside the ranges given. +static void Overwrite(unsigned char* state, size_t size, const Range* ranges, size_t count, + const uint8_t* data, size_t length) +{ + for (size_t i = 0; i + 1 < length; i += 2) + { + size_t offset = data[i] % size; + if (!Inside(ranges, count, offset)) + { + state[offset] = data[i + 1]; + } + } +} + +static void Script(uint8_t steps, const uint8_t* data, size_t length) +{ + muniScriptIterator iterator; + muniScriptRun run; + if (muniInitScriptIterator(&iterator, s_text, sizeof(s_text) - 1, false) != muni_success) + { + return; + } + for (uint8_t i = 0; i < steps % 8; i++) + { + (void)muniScriptIterator_NextRun(&iterator, &run); + } + static const Range kept[] = { + {offsetof(ScriptState, cursor) + offsetof(muniCursor, text), sizeof(const uint8_t*)}, + {offsetof(ScriptState, cursor) + offsetof(muniCursor, length), sizeof(size_t)}, + }; + Overwrite((unsigned char*)&iterator, sizeof(ScriptState), kept, sizeof(kept) / sizeof(kept[0]), + data, length); + for (int i = 0; i < 64; i++) + { + muniResult status = muniScriptIterator_NextRun(&iterator, &run); + if (status == muni_needMoreText) + { + status = muniScriptIterator_Feed(&iterator, "", 0, false); + } + if (status != muni_success) + { + break; + } + } +} + +static void Segment(uint8_t choice, uint8_t steps, const uint8_t* data, size_t length) +{ + muniSegmentIterator iterator; + size_t offset; + bool mandatory; + bool more = (choice & 8) != 0; + muniResult status; + switch (choice % 4) + { + case 0: + status = muniInitGraphemeIterator(&iterator, s_text, sizeof(s_text) - 1, more); + break; + case 1: + status = muniInitWordIterator(&iterator, s_text, sizeof(s_text) - 1, more); + if (status == muni_success && (choice & 16) != 0) + { + status = muniSegmentIterator_SetBreakClassOverrides(&iterator, s_overrides, 2); + } + break; + case 2: + status = muniInitSentenceIterator(&iterator, s_text, sizeof(s_text) - 1, more); + break; + default: + status = muniInitLineIterator(&iterator, s_text, sizeof(s_text) - 1, more); + if (status == muni_success) + { + status = muniSegmentIterator_SetLineBreakStrictness(&iterator, (choice >> 5) & 3); + } + break; + } + if (status != muni_success) + { + return; + } + if ((choice & 32) != 0) + { + (void)muniSegmentIterator_SetComplexBreaker(&iterator, Breaker, nullptr); + } + for (uint8_t i = 0; i < steps % 8; i++) + { + (void)muniSegmentIterator_NextBreak(&iterator, &offset); + } + // The override count shares the rules' storage, so it is kept only + // where the rules are a word or sentence iterator's. + Range kept[] = { + {offsetof(muniSegmenter, cursor) + offsetof(muniCursor, text), sizeof(const uint8_t*)}, + {offsetof(muniSegmenter, cursor) + offsetof(muniCursor, length), sizeof(size_t)}, + {offsetof(muniSegmenter, complex), sizeof(muniComplexRun)}, + {offsetof(muniSegmenter, kind), 1}, + {offsetof(muniSegmenter, overrides), sizeof(const muniBreakClassOverride*)}, + {0, 0}, + }; + if (choice % 4 == 1) + { + kept[5] = (Range){offsetof(muniSegmenter, rules.word.overrideCount), sizeof(uint32_t)}; + } + else if (choice % 4 == 2) + { + kept[5] = (Range){offsetof(muniSegmenter, rules.sentence.overrideCount), sizeof(uint32_t)}; + } + Overwrite((unsigned char*)&iterator, sizeof(muniSegmenter), kept, + sizeof(kept) / sizeof(kept[0]), data, length); + muniWordKind kind; + (void)muniSegmentIterator_GetWordKind(&iterator, &kind); + (void)muniSegmentIterator_GetLineBreakStrictness(&iterator); + (void)muniSegmentIterator_GetWritingSystem(&iterator); + for (int i = 0; i < 64; i++) + { + status = (choice & 64) != 0 + ? muniSegmentIterator_NextLineBreak(&iterator, &offset, &mandatory) + : muniSegmentIterator_NextBreak(&iterator, &offset); + if (status == muni_needMoreText) + { + status = muniSegmentIterator_Feed(&iterator, "", 0, false); + } + if (status != muni_success) + { + break; + } + } +} + +int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size); + +int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) +{ + if (size < 2) + { + return 0; + } + if ((data[0] & 128) != 0) + { + Script(data[1], data + 2, size - 2); + } + else + { + Segment(data[0], data[1], data + 2, size - 2); + } + return 0; +} diff --git a/test/seeds/fuzz_iterators/grapheme-more b/test/seeds/fuzz_iterators/grapheme-more new file mode 100644 index 0000000..88726d6 --- /dev/null +++ b/test/seeds/fuzz_iterators/grapheme-more @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/seeds/fuzz_iterators/line-breaker b/test/seeds/fuzz_iterators/line-breaker new file mode 100644 index 0000000..1310f4a --- /dev/null +++ b/test/seeds/fuzz_iterators/line-breaker @@ -0,0 +1 @@ +c`h \ No newline at end of file diff --git a/test/seeds/fuzz_iterators/script b/test/seeds/fuzz_iterators/script new file mode 100644 index 0000000..24b6421 --- /dev/null +++ b/test/seeds/fuzz_iterators/script @@ -0,0 +1 @@ +€ \ No newline at end of file diff --git a/test/seeds/fuzz_iterators/script-overwritten b/test/seeds/fuzz_iterators/script-overwritten new file mode 100644 index 0000000..ad3d82c --- /dev/null +++ b/test/seeds/fuzz_iterators/script-overwritten @@ -0,0 +1 @@ +€0A1È \ No newline at end of file diff --git a/test/seeds/fuzz_iterators/sentence b/test/seeds/fuzz_iterators/sentence new file mode 100644 index 0000000..ad45e54 --- /dev/null +++ b/test/seeds/fuzz_iterators/sentence @@ -0,0 +1 @@ +p \ No newline at end of file diff --git a/test/seeds/fuzz_iterators/word-overrides b/test/seeds/fuzz_iterators/word-overrides new file mode 100644 index 0000000..8e6e78e --- /dev/null +++ b/test/seeds/fuzz_iterators/word-overrides @@ -0,0 +1 @@ +PAQ \ No newline at end of file diff --git a/test/test_script.c b/test/test_script.c index bac96cd..a70e4bd 100644 --- a/test/test_script.c +++ b/test/test_script.c @@ -229,6 +229,35 @@ static void TestCapacityAndArguments(void) "no count, and no run written"); } +// An iterator no init call set up, of one byte repeated: zeroed as a +// static one is, or of bytes that would make the pending count copy past +// its four bytes (0x41) or a flag no bool (0x02, 0x04). Both calls that +// take it back refuse it as a bad argument and change nothing in it. +static void TestUninitializedIterators(void) +{ + static const unsigned char fills[] = {0x00, 0x41, 0x02, 0x04}; + for (size_t i = 0; i < sizeof(fills); i++) + { + muniScriptIterator iterator; + memset(&iterator, fills[i], sizeof(iterator)); + muniScriptIterator before = iterator; + muniScriptRun run = {7, LATN}; + CHECK(muniScriptIterator_NextRun(&iterator, &run) == muni_errorInvalid && + muniScriptIterator_Feed(&iterator, "a", 1, false) == muni_errorInvalid && + run.end == 7 && run.script == LATN && + memcmp(&iterator, &before, sizeof(iterator)) == 0, + "an iterator no init call set up is refused and left as it was"); + } + // A copy of an iterator an init call set up is one too. + muniScriptIterator iterator; + muniScriptRun run; + CHECK(muniInitScriptIterator(&iterator, "ab", 2, false) == muni_success, "init"); + muniScriptIterator copy = iterator; + CHECK(muniScriptIterator_NextRun(©, &run) == muni_success && run.end == 2 && + run.script == LATN, + "a copy of an iterator steps on"); +} + int main(void) { TestRuns(); @@ -237,5 +266,6 @@ int main(void) TestBracketDepth(); TestScriptExtensionsList(); TestCapacityAndArguments(); + TestUninitializedIterators(); return s_failures == 0 ? 0 : 1; } diff --git a/test/test_segment.c b/test/test_segment.c index eebee69..3793f26 100644 --- a/test/test_segment.c +++ b/test/test_segment.c @@ -1170,6 +1170,53 @@ static void TestFeedingRules(void) "never initialized"); } +// An iterator no init call set up, of one byte repeated: zeroed as a +// static one is, or of bytes that would make a count run past the state +// (0x41) or a flag no bool (0x02, 0x04). Every call that takes it back +// refuses it as a bad argument and changes nothing in it, and the getters +// give their defaults. +static void TestUninitializedIterators(void) +{ + static const unsigned char fills[] = {0x00, 0x41, 0x02, 0x04}; + static const muniBreakClassOverride posix[] = {{'_', muni_wbExtendNumLet}}; + static const size_t stops[] = {0}; + for (size_t i = 0; i < sizeof(fills); i++) + { + muniSegmentIterator iterator; + memset(&iterator, fills[i], sizeof(iterator)); + muniSegmentIterator before = iterator; + size_t offset = 7; + bool mandatory = true; + muniWordKind kind = muni_wordLetter; + bool refused = + muniSegmentIterator_NextBreak(&iterator, &offset) == muni_errorInvalid && + muniSegmentIterator_NextLineBreak(&iterator, &offset, &mandatory) == + muni_errorInvalid && + muniSegmentIterator_Feed(&iterator, "a", 1, false) == muni_errorInvalid && + muniSegmentIterator_SetComplexBreaker(&iterator, ToyBreaker, (void*)stops) == + muni_errorInvalid && + muniSegmentIterator_SetBreakClassOverrides(&iterator, posix, 1) == muni_errorInvalid && + muniSegmentIterator_SetLineBreakStrictness(&iterator, muni_lineBreakLoose) == + muni_errorInvalid && + muniSegmentIterator_SetWritingSystem(&iterator, muni_writingChineseJapanese) == + muni_errorInvalid && + muniSegmentIterator_GetWordKind(&iterator, &kind) == muni_errorInvalid; + bool defaults = + muniSegmentIterator_GetLineBreakStrictness(&iterator) == muni_lineBreakNormal && + muniSegmentIterator_GetWritingSystem(&iterator) == muni_writingOther; + CHECK(refused && defaults && offset == 7 && mandatory && kind == muni_wordLetter && + memcmp(&iterator, &before, sizeof(iterator)) == 0, + "an iterator no init call set up is refused and left as it was"); + } + // A copy of an iterator an init call set up is one too. + muniSegmentIterator iterator; + size_t offset = 0; + CHECK(muniInitWordIterator(&iterator, "a b", 3, false) == muni_success, "init"); + muniSegmentIterator copy = iterator; + CHECK(muniSegmentIterator_NextBreak(©, &offset) == muni_success && offset == 1, + "a copy of an iterator steps on"); +} + // A kind is an int32_t, so a caller can pass a negative one; every call // that takes a kind refuses -1 and INT32_MIN as it refuses a kind past the // last, and leaves its outputs as they were. @@ -1525,6 +1572,7 @@ int main(void) TestAnywhere(); TestInvalidArguments(); TestFeedingRules(); + TestUninitializedIterators(); TestWordKinds(); TestMeasureLimits(); TestMeasureUses(); diff --git a/tools/modules.txt b/tools/modules.txt index b06cc62..6964ffe 100755 --- a/tools/modules.txt +++ b/tools/modules.txt @@ -18,7 +18,7 @@ grapheme: segmenter tables harfbuzz: identifier: encoding tables invariant: -line: line_rules segmenter +line: grapheme line_rules segmenter line_anywhere: grapheme segmenter tables line_rules: complex segmenter tables line_tailored: line_rules segmenter @@ -27,8 +27,9 @@ normalizer: decompose tables writer overlap: nfkc_casefold: case_map decompose encoding invariant normalizer overlap tables properties: tables -script_runs: cursor invariant tables -segment: encoding segmenter +script_runs: invariant script_state tables +script_state: cursor +segment: encoding grapheme segmenter segmenter: cursor invariant security: bits encoding tables sentence: segmenter tables From d121e6956254601f5d54e45a4bb4dfbc99fb3122 Mon Sep 17 00:00:00 2001 From: Sirac Ozmen Date: Wed, 4 Nov 2026 06:39:30 +0300 Subject: [PATCH 2/2] fix(segment): inline the iterator check and read its flags in place Every segment call checked the iterator through a validator per rule set, called out of line, which cost a grapheme break about 2.7 ns (16%) and a line break about 6.9 ns (28%) on the bench. The check now lives in segment.c and is inlined into each call (MUNI_ALWAYS_INLINE), with a switch on the rule set that checks only that set's fields, and line's anywhere setting only the grapheme state it keeps. The flags' bytes are read from the caller's iterator rather than from the copy just made, so no load waits on the copy's stores. Every field is still checked before it is read. The largest values the rule sets write are named in segmenter.h and tied to each rule set's enumeration by a static_assert. The check now costs a grapheme break about 1.9 ns (11%) and a line break about 4.9 ns (20%), and a script run about 1.3 ns (4%), all of it executed instructions: perf shows no added branch or cache miss. --- docs/internals/segmentation.md | 2 +- src/grapheme.c | 9 +-- src/grapheme.h | 4 - src/invariant.h | 6 ++ src/line.c | 24 ------ src/line_rules.h | 3 + src/segment.c | 131 +++++++++++++++++++++++---------- src/segmenter.h | 19 +++-- src/sentence.c | 11 +-- src/word.c | 9 --- tools/modules.txt | 4 +- 11 files changed, 121 insertions(+), 101 deletions(-) diff --git a/docs/internals/segmentation.md b/docs/internals/segmentation.md index 54eb1a8..dd024df 100644 --- a/docs/internals/segmentation.md +++ b/docs/internals/segmentation.md @@ -8,7 +8,7 @@ UAX #29 defines three segmentations of the same shape: walk the text and decide, The public struct is opaque storage. [`src/segment.c`](../../src/segment.c) copies the internal `muniSegmenter` in and out of it with `memcpy`, which keeps every access within the C aliasing rules, and dispatches on the kind once per boundary, not per code point. -The caller holds that storage, so every call checks what it copied out before it reads it, as the family's rule on state a caller holds asks ([maul-conventions.md](maul-conventions.md)): the cookie an init call writes in the cursor, every count against the storage it counts (the cursor's pending bytes and position, the override count), every class, state and hold kind against the values its rule set writes, and every flag's byte as a 0 or 1 before the copy is read as a `bool`. It also checks what the assertions downstream rely on: a waiting cursor holds less than a code point, and a hold lies before the cursor. An iterator no init call set up is refused with `muni_errorInvalid`, so no bit pattern makes a call read outside the state; the pointers it keeps (the text, the overrides, the caller's breaker) cannot be checked, and one forged there is outside the contract. The script iterator ([`src/script_runs.c`](../../src/script_runs.c)) checks its state the same way. The checks run once per call, not per code point. `test/fuzz_iterators.c` sets every byte outside those pointers, and the lengths and counts that go with them, from its input and calls every function again. +The caller holds that storage, so every call checks what it copied out before it reads it, as the family's rule on state a caller holds asks ([maul-conventions.md](maul-conventions.md)): the cookie an init call writes in the cursor, every count against the storage it counts (the cursor's pending bytes and position, the override count), every class, state and hold kind against the values its rule set writes, and every flag's byte as a 0 or 1 before the copy is read as a `bool`. It also checks what the assertions downstream rely on: a waiting cursor holds less than a code point, and a hold lies before the cursor. An iterator no init call set up is refused with `muni_errorInvalid`, so no bit pattern makes a call read outside the state; the pointers it keeps (the text, the overrides, the caller's breaker) cannot be checked, and one forged there is outside the contract. The script iterator ([`src/script_runs.c`](../../src/script_runs.c)) checks its state the same way. The checks run once per call, not per code point. They are inlined into each call (`MUNI_ALWAYS_INLINE`), since the compiler left them outlined at 1 to 2 ns more per call, and they read the flags' bytes from the caller's iterator rather than from the copy just made of it, so that no load waits on the copy's stores. On the bench, where every call reports one boundary, they cost a grapheme break about 2 ns (11%) and a line break about 5 ns (20%), all of it instructions executed: they add no branch or cache miss. `test/fuzz_iterators.c` sets every byte outside those pointers, and the lengths and counts that go with them, from its input and calls every function again. Its size follows the state: a `static_assert` in [`src/segmenter.h`](../../src/segmenter.h) keeps `muniSegmenter` within it, so a change that grows the state fails to build until the public iterator grows with it, and on 64-bit targets a second one keeps the iterator from growing a whole `uint64_t` more than the state needs. On 64-bit targets `muniSegmenter` fills all 128 bytes, with the cookie in the two bytes that were the cursor's padding, a line iterator's strictness and writing system packed in the byte after the hold's kind and the class overrides' pointer in the last 8. The next switch or piece of state therefore grows the public iterator, which changes the ABI; from 1.0.0 that is allowed in a minor release, so its pull request is a `feat` with no `!`. diff --git a/src/grapheme.c b/src/grapheme.c index 7bd98fe..c272e9d 100644 --- a/src/grapheme.c +++ b/src/grapheme.c @@ -24,6 +24,8 @@ enum EmojiPictographic = 1, // Extended_Pictographic Extend* EmojiJoined = 2, // Extended_Pictographic Extend* ZWJ }; +static_assert((int)EmojiJoined == (int)muni_graphemeEmojiLast, + "segment.c checks the emoji state up to it"); static bool IsControl(uint8_t value) { @@ -122,13 +124,6 @@ void muniGraphemeAdvance(muniGraphemeRules* state, uint8_t value, uint32_t codeP state->previous = value; } -bool muniGraphemeRulesValid(const muniGraphemeRules* state, const void* raw, size_t at) -{ - return state->previous <= muni_gcbLvt && state->emoji <= EmojiJoined && - muniIsFlagByte(raw, at + offsetof(muniGraphemeRules, linker)) && - muniIsFlagByte(raw, at + offsetof(muniGraphemeRules, oddRegional)); -} - static void Absorb(muniSegmenter* segmenter, uint8_t value, uint32_t codePoint) { muniGraphemeAdvance(&segmenter->rules.grapheme, value, codePoint); diff --git a/src/grapheme.h b/src/grapheme.h index cd36dac..226fa9c 100644 --- a/src/grapheme.h +++ b/src/grapheme.h @@ -16,8 +16,4 @@ bool muniGraphemeIsBoundary(const muniGraphemeRules* state, uint8_t after, uint3 // Moves the state past a code point whose Grapheme_Cluster_Break is value. void muniGraphemeAdvance(muniGraphemeRules* state, uint8_t value, uint32_t codePoint); -// Whether a state copied out of raw, from offset at, holds what Advance -// writes: a Grapheme_Cluster_Break value, an emoji state, flags of 0 or 1. -bool muniGraphemeRulesValid(const muniGraphemeRules* state, const void* raw, size_t at); - #endif // MAUL_UNICODE_SRC_GRAPHEME_H diff --git a/src/invariant.h b/src/invariant.h index b94d3ae..b049630 100644 --- a/src/invariant.h +++ b/src/invariant.h @@ -18,4 +18,10 @@ #define MUNI_ASSERT(condition) ((condition) ? (void)0 : __builtin_trap()) #endif +// MUNI_ALWAYS_INLINE asks that a small function be inlined into every +// caller, for a check on a call's hot path that the compiler would +// otherwise leave as a call of its own. Every compiler the family builds +// with, clang-cl included, takes the attribute. +#define MUNI_ALWAYS_INLINE inline __attribute__((always_inline)) + #endif // MAUL_UNICODE_SRC_INVARIANT_H diff --git a/src/line.c b/src/line.c index 175541f..4a48b3f 100644 --- a/src/line.c +++ b/src/line.c @@ -7,7 +7,6 @@ // segment.c sends the other settings to line_tailored.c and // line_anywhere.c, once per call. -#include "grapheme.h" #include "line_rules.h" static uint8_t Lookup(const muniSegmenter* segmenter, uint32_t codePoint) @@ -24,26 +23,3 @@ muniResult muniNextLineSegment(muniSegmenter* segmenter, size_t* offsetOut) { return muniSegmenterNext(segmenter, offsetOut, Lookup, Decide, Resolve, Absorb, nullptr); } - -// The rules of every setting, checked here beside the default ones: -// breaking anywhere keeps the grapheme rules where the others keep the -// classes before the position. -bool muniLineRulesValid(const muniSegmenter* segmenter, const void* raw) -{ - const muniLineRules* state = &segmenter->rules.line; - if (state->actual > muni_lbHh) - { - return false; - } - if (muniLineStrictness(segmenter) == muni_lineBreakAnywhere) - { - return segmenter->holdKind == 0 && - muniGraphemeRulesValid(&state->grapheme, raw, - offsetof(muniSegmenter, rules.line.grapheme)); - } - return state->previous <= muni_lbHh && state->before <= muni_lbHh && state->lead <= muni_lbHh && - state->number <= NumberClosed && segmenter->holdKind <= HoldAksara && - muniIsFlagByte(raw, offsetof(muniSegmenter, rules.line.leadInitialQuote)) && - muniIsFlagByte(raw, offsetof(muniSegmenter, rules.line.sawInfix)) && - muniIsFlagByte(raw, offsetof(muniSegmenter, rules.line.oddRegional)); -} diff --git a/src/line_rules.h b/src/line_rules.h index 55a1499..d591e5d 100644 --- a/src/line_rules.h +++ b/src/line_rules.h @@ -56,6 +56,7 @@ enum HoldPrefix = 3, // LB25: (PR | PO) × OP, if (IS)? NU follows HoldAksara = 4, // LB28a: aksara × aksara, if VF follows }; +static_assert((int)HoldAksara == (int)muni_lineHoldLast, "segment.c checks the hold kind up to it"); // Where an LB25 number stands. enum @@ -64,6 +65,8 @@ enum NumberDigits = 1, // NU (SY | IS)* NumberClosed = 2, // NU (SY | IS)* (CL | CP) }; +static_assert((int)NumberClosed == (int)muni_lineNumberLast, + "segment.c checks the number state up to it"); enum : uint64_t { diff --git a/src/segment.c b/src/segment.c index 423ca1c..6b4926a 100644 --- a/src/segment.c +++ b/src/segment.c @@ -8,7 +8,6 @@ #include "maul-unicode/segment.h" #include "encoding.h" -#include "grapheme.h" #include "segmenter.h" #include @@ -24,61 +23,119 @@ // code points strictly increase. #define MAX_OVERRIDES 0x10F800u -// The byte at offset in the caller's iterator; four flags are checked at -// once by taking their bytes together. -static inline uint8_t Byte(const muniSegmentIterator* iterator, size_t offset) +// The byte at offset in the iterator in caller memory. The flags are +// checked as these bytes, before the copy is read as bools. Reading them +// from the iterator rather than the copy just made of it keeps each load +// from waiting on the copy's stores. +static inline unsigned Byte(const muniSegmentIterator* iterator, size_t offset) { return ((const unsigned char*)iterator)[offset]; } +// Whether the rules of a segmenter copied out of iterator hold what their +// rule set writes, each flag's byte a 0 or 1 included. +static MUNI_ALWAYS_INLINE bool RulesValid(const muniSegmenter* segmenter, + const muniSegmentIterator* iterator) +{ + switch (segmenter->kind) + { + case muni_segmentGrapheme: + { + const muniGraphemeRules* rules = &segmenter->rules.grapheme; + return segmenter->lineSettings == 0 && segmenter->holdKind == 0 && + rules->previous <= muni_gcbLvt && rules->emoji <= muni_graphemeEmojiLast && + (Byte(iterator, offsetof(muniSegmenter, rules.grapheme.linker)) | + Byte(iterator, offsetof(muniSegmenter, rules.grapheme.oddRegional))) <= 1; + } + case muni_segmentWord: + { + const muniWordRules* rules = &segmenter->rules.word; + uint32_t count = rules->overrideCount; + return segmenter->lineSettings == 0 && segmenter->holdKind == 0 && + rules->actual <= muni_wbWSegSpace && rules->previous <= muni_wbWSegSpace && + rules->before <= muni_wbWSegSpace && rules->kind <= muni_wordIdeograph && + rules->reportedKind <= muni_wordIdeograph && count <= MAX_OVERRIDES && + (count == 0 || segmenter->overrides != nullptr) && + Byte(iterator, offsetof(muniSegmenter, rules.word.oddRegional)) <= 1; + } + case muni_segmentSentence: + { + const muniSentenceRules* rules = &segmenter->rules.sentence; + uint32_t count = rules->overrideCount; + return segmenter->lineSettings == 0 && segmenter->holdKind == 0 && + rules->actual <= muni_sbClose && rules->previous <= muni_sbClose && + rules->before <= muni_sbClose && rules->ending <= muni_sentenceEndingLast && + count <= MAX_OVERRIDES && (count == 0 || segmenter->overrides != nullptr) && + Byte(iterator, offsetof(muniSegmenter, rules.sentence.aterm)) <= 1; + } + case muni_segmentLine: + { + // Breaking anywhere keeps the grapheme rules where the other + // settings keep the classes before the position. + const muniLineRules* rules = &segmenter->rules.line; + if (segmenter->lineSettings >> 3 != 0 || rules->actual > muni_lbHh) + { + return false; + } + if (muniLineStrictness(segmenter) == muni_lineBreakAnywhere) + { + return segmenter->holdKind == 0 && rules->grapheme.previous <= muni_gcbLvt && + rules->grapheme.emoji <= muni_graphemeEmojiLast && + (Byte(iterator, offsetof(muniSegmenter, rules.line.grapheme.linker)) | + Byte(iterator, offsetof(muniSegmenter, rules.line.grapheme.oddRegional))) <= 1; + } + return segmenter->holdKind <= muni_lineHoldLast && rules->previous <= muni_lbHh && + rules->before <= muni_lbHh && rules->lead <= muni_lbHh && + rules->number <= muni_lineNumberLast && + (Byte(iterator, offsetof(muniSegmenter, rules.line.leadInitialQuote)) | + Byte(iterator, offsetof(muniSegmenter, rules.line.sawInfix)) | + Byte(iterator, offsetof(muniSegmenter, rules.line.oddRegional))) <= 1; + } + default: + return false; + } +} + // Copies the iterator's state out of caller memory when it is one an // init call set up and every flag, count and value in it is in range, so // that no bit pattern makes a call read outside the state or the text // and overrides it names; false otherwise. An iterator with the cookie // whose fields were changed by hand is outside the contract, and how its -// fields agree with each other is not checked again. -static bool Load(const muniSegmentIterator* iterator, muniSegmenter* segmenterOut) +// fields agree with each other is not checked again. Every call reads +// most of the state, so all of it is checked, inline in each call so that +// the copy stays inline too. +static MUNI_ALWAYS_INLINE bool Load(const muniSegmentIterator* iterator, + muniSegmenter* segmenterOut) { memcpy(segmenterOut, iterator, sizeof(*segmenterOut)); const muniSegmenter* segmenter = segmenterOut; - if (segmenter->cursor.cookie != COOKIE || segmenter->kind < muni_segmentGrapheme || - segmenter->kind > muni_segmentLine || segmenter->phase > muni_phaseBusy || - !muniCursorValid(&segmenter->cursor, iterator, offsetof(muniSegmenter, cursor), - segmenter->phase == muni_phaseWaiting) || - (Byte(iterator, offsetof(muniSegmenter, started)) | - Byte(iterator, offsetof(muniSegmenter, endReported)) | - Byte(iterator, offsetof(muniSegmenter, holding)) | + const muniCursor* cursor = &segmenter->cursor; + size_t at = offsetof(muniSegmenter, cursor); + unsigned more = Byte(iterator, at + offsetof(muniCursor, moreFollows)); + unsigned started = Byte(iterator, offsetof(muniSegmenter, started)); + unsigned holding = Byte(iterator, offsetof(muniSegmenter, holding)); + if (cursor->cookie != COOKIE || segmenter->phase > muni_phaseBusy || + (more | started | holding | Byte(iterator, offsetof(muniSegmenter, endReported)) | Byte(iterator, offsetof(muniSegmenter, mandatory))) > 1 || - (segmenter->complex.text != nullptr && segmenter->complex.next > segmenter->complex.length)) + cursor->pendingCount > sizeof(cursor->pending) || cursor->position > cursor->length || + (cursor->text == nullptr && cursor->length != 0)) { return false; } - // A boundary lies after a code point, and a hold before the cursor. - size_t offset = segmenter->cursor.offset; - if ((segmenter->started && offset == 0) || - (segmenter->holding && (segmenter->heldOffset == 0 || segmenter->heldOffset >= offset))) + // A waiting cursor holds less than a code point, as muniCursorFeed + // relies on; a boundary lies after a code point and a hold before the + // cursor, as the engine's assertions do. + size_t offset = cursor->offset; + if ((segmenter->phase == muni_phaseWaiting && + (more == 0 || + cursor->pendingCount + (cursor->length - cursor->position) >= sizeof(cursor->pending))) || + (started != 0 && offset == 0) || + (holding != 0 && (segmenter->heldOffset == 0 || segmenter->heldOffset >= offset)) || + (segmenter->complex.text != nullptr && segmenter->complex.next > segmenter->complex.length)) { return false; } - switch (segmenter->kind) - { - case muni_segmentGrapheme: - return segmenter->lineSettings == 0 && segmenter->holdKind == 0 && - muniGraphemeRulesValid(&segmenter->rules.grapheme, iterator, - offsetof(muniSegmenter, rules.grapheme)); - case muni_segmentWord: - return segmenter->lineSettings == 0 && - segmenter->rules.word.overrideCount <= MAX_OVERRIDES && - (segmenter->rules.word.overrideCount == 0 || segmenter->overrides != nullptr) && - muniWordRulesValid(segmenter, iterator); - case muni_segmentSentence: - return segmenter->lineSettings == 0 && - segmenter->rules.sentence.overrideCount <= MAX_OVERRIDES && - (segmenter->rules.sentence.overrideCount == 0 || segmenter->overrides != nullptr) && - muniSentenceRulesValid(segmenter, iterator); - default: - return segmenter->lineSettings >> 3 == 0 && muniLineRulesValid(segmenter, iterator); - } + return RulesValid(segmenter, iterator); } static void Store(muniSegmentIterator* iterator, const muniSegmenter* segmenter) diff --git a/src/segmenter.h b/src/segmenter.h index ea46e7d..0ad34b3 100644 --- a/src/segmenter.h +++ b/src/segmenter.h @@ -110,6 +110,17 @@ typedef struct muniLineRules bool oddRegional; // an odd number of regional indicators precedes } muniLineRules; +// The last value each rule set writes in its states, for the check of an +// iterator the caller holds; each rule set ties its own enumeration to +// these with a static_assert. +enum +{ + muni_graphemeEmojiLast = 2, // grapheme.c's EmojiJoined + muni_sentenceEndingLast = 2, // sentence.c's EndingSpace + muni_lineNumberLast = 2, // line_rules.h's NumberClosed + muni_lineHoldLast = 4, // line_rules.h's HoldAksara +}; + // A run of Thai, Lao, Khmer or Burmese (Line_Break=SA) that a caller's // breaker segments. The run lies in the current piece. typedef struct muniComplexRun @@ -317,12 +328,4 @@ muniResult muniNextLineSegment(muniSegmenter* segmenter, size_t* offsetOut); muniResult muniNextLineTailoredSegment(muniSegmenter* segmenter, size_t* offsetOut); muniResult muniNextLineAnywhereSegment(muniSegmenter* segmenter, size_t* offsetOut); -// Whether the rules of a segmenter copied out of raw, state the caller -// holds, are ones the rule set writes: each class and state in range, -// each flag a 0 or 1 and the hold kind one it gives. Defined beside each -// rule set; the grapheme rules' is in grapheme.h. -bool muniWordRulesValid(const muniSegmenter* segmenter, const void* raw); -bool muniSentenceRulesValid(const muniSegmenter* segmenter, const void* raw); -bool muniLineRulesValid(const muniSegmenter* segmenter, const void* raw); - #endif // MAUL_UNICODE_SRC_SEGMENTER_H diff --git a/src/sentence.c b/src/sentence.c index 1e76333..bf3ab1a 100644 --- a/src/sentence.c +++ b/src/sentence.c @@ -22,6 +22,8 @@ enum EndingTerm = 1, // SATerm Close* EndingSpace = 2, // SATerm Close* Sp+ }; +static_assert((int)EndingSpace == (int)muni_sentenceEndingLast, + "segment.c checks the ending up to it"); enum { @@ -160,12 +162,3 @@ muniResult muniNextSentenceSegment(muniSegmenter* segmenter, size_t* offsetOut) { return muniSegmenterNext(segmenter, offsetOut, Lookup, Decide, Resolve, Absorb, nullptr); } - -bool muniSentenceRulesValid(const muniSegmenter* segmenter, const void* raw) -{ - const muniSentenceRules* state = &segmenter->rules.sentence; - return state->actual <= muni_sbClose && state->previous <= muni_sbClose && - state->before <= muni_sbClose && state->ending <= EndingSpace && - segmenter->holdKind == 0 && - muniIsFlagByte(raw, offsetof(muniSegmenter, rules.sentence.aterm)); -} diff --git a/src/word.c b/src/word.c index e5aefde..d20c768 100644 --- a/src/word.c +++ b/src/word.c @@ -245,12 +245,3 @@ muniResult muniNextWordSegment(muniSegmenter* segmenter, size_t* offsetOut) { return muniSegmenterNext(segmenter, offsetOut, Lookup, Decide, Resolve, Absorb, Boundary); } - -bool muniWordRulesValid(const muniSegmenter* segmenter, const void* raw) -{ - const muniWordRules* state = &segmenter->rules.word; - return state->actual <= muni_wbWSegSpace && state->previous <= muni_wbWSegSpace && - state->before <= muni_wbWSegSpace && state->kind <= muni_wordIdeograph && - state->reportedKind <= muni_wordIdeograph && segmenter->holdKind == 0 && - muniIsFlagByte(raw, offsetof(muniSegmenter, rules.word.oddRegional)); -} diff --git a/tools/modules.txt b/tools/modules.txt index 6964ffe..0fae5cf 100755 --- a/tools/modules.txt +++ b/tools/modules.txt @@ -18,7 +18,7 @@ grapheme: segmenter tables harfbuzz: identifier: encoding tables invariant: -line: grapheme line_rules segmenter +line: line_rules segmenter line_anywhere: grapheme segmenter tables line_rules: complex segmenter tables line_tailored: line_rules segmenter @@ -29,7 +29,7 @@ nfkc_casefold: case_map decompose encoding invariant normalizer overlap tables properties: tables script_runs: invariant script_state tables script_state: cursor -segment: encoding grapheme segmenter +segment: encoding segmenter segmenter: cursor invariant security: bits encoding tables sentence: segmenter tables