diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9955fae..e6f17c8 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -248,7 +248,7 @@ jobs:
#
# A dispatch's fuzz-seconds must leave time for the minimize and
# save, or the timeout kills the job and the corpus is lost: the
- # ten targets run in three waves of four, and the night's 50
+ # eleven targets run in three waves of four, and the night's 50
# minutes less about 10 for the build, the merge and the save leave
# 40, so at most 800 seconds a wave. libFuzzer reads a negative
# time as no limit.
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 181ba17..83ae4bf 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -224,7 +224,7 @@ if(MAUL_UNICODE_FUZZ)
target_compile_options(maul-unicode PRIVATE -fsanitize=fuzzer-no-link,address,undefined
-fno-sanitize-recover=all)
target_link_options(maul-unicode PUBLIC -fsanitize=address,undefined -fno-sanitize-recover=all)
- set(MUNI_FUZZ_AREAS encoding segment bidi bidi_long script)
+ set(MUNI_FUZZ_AREAS encoding segment bidi bidi_long script iterators)
if(MAUL_UNICODE_NORMALIZATION)
list(APPEND MUNI_FUZZ_AREAS normalize)
endif()
@@ -249,6 +249,9 @@ if(MAUL_UNICODE_FUZZ)
target_link_options(fuzz_${area} PRIVATE -fsanitize=fuzzer,address,undefined
-fno-sanitize-recover=all)
endforeach()
+ # fuzz_iterators sets the fields of the iterators' state, which only
+ # src/ declares.
+ target_include_directories(fuzz_iterators PRIVATE src)
# ICU is an oracle for the tests alone, never linked into the library.
if(TARGET fuzz_icu)
find_package(PkgConfig REQUIRED)
diff --git a/docs/internals/overview.md b/docs/internals/overview.md
index 5556a68..6362dee 100644
--- a/docs/internals/overview.md
+++ b/docs/internals/overview.md
@@ -77,7 +77,7 @@ Outside the promise:
- `MUNI_MAX_SCRIPT_EXTENSIONS`, `MUNI_MAX_RESOLVED_SCRIPTS` and `MUNI_MAX_DECOMPOSITION` hold for one Unicode version and may rise in a minor release; the calls always report the count they need.
- The size of the iterator structs across minor releases.
- Everything outside `include/`: `src/`, the generated tables and their sizes, speed, the tools, tests, samples and benchmarks, and the HarfBuzz versions the adapter builds with.
-- What a call does when its caller breaks a header's contract, such as a workspace that overlaps its text or an iterator that was never initialized.
+- What a call does when its caller breaks a header's contract, such as a workspace that overlaps its text, or an iterator whose bytes were changed by hand while keeping the cookie its init call wrote. An iterator no init call set up is refused with `muni_errorInvalid`, which is promised.
The library reads and writes no asset or state format, so it promises none, and its headers name no other Maul library that would have to reach 1.0.0 first.
diff --git a/docs/internals/segmentation.md b/docs/internals/segmentation.md
index 9c232b4..dd024df 100644
--- a/docs/internals/segmentation.md
+++ b/docs/internals/segmentation.md
@@ -8,7 +8,9 @@ UAX #29 defines three segmentations of the same shape: walk the text and decide,
The public struct is opaque storage. [`src/segment.c`](../../src/segment.c) copies the internal `muniSegmenter` in and out of it with `memcpy`, which keeps every access within the C aliasing rules, and dispatches on the kind once per boundary, not per code point.
-Its size follows the state: a `static_assert` in [`src/segmenter.h`](../../src/segmenter.h) keeps `muniSegmenter` within it, so a change that grows the state fails to build until the public iterator grows with it. On 64-bit targets `muniSegmenter` fills all 128 bytes, with a line iterator's strictness and writing system packed in the byte after the hold's kind and the class overrides' pointer in the last 8. The next switch or piece of state therefore grows the public iterator, which changes the ABI, so its pull request's title carries `!`.
+The caller holds that storage, so every call checks what it copied out before it reads it, as the family's rule on state a caller holds asks ([maul-conventions.md](maul-conventions.md)): the cookie an init call writes in the cursor, every count against the storage it counts (the cursor's pending bytes and position, the override count), every class, state and hold kind against the values its rule set writes, and every flag's byte as a 0 or 1 before the copy is read as a `bool`. It also checks what the assertions downstream rely on: a waiting cursor holds less than a code point, and a hold lies before the cursor. An iterator no init call set up is refused with `muni_errorInvalid`, so no bit pattern makes a call read outside the state; the pointers it keeps (the text, the overrides, the caller's breaker) cannot be checked, and one forged there is outside the contract. The script iterator ([`src/script_runs.c`](../../src/script_runs.c)) checks its state the same way. The checks run once per call, not per code point. They are inlined into each call (`MUNI_ALWAYS_INLINE`), since the compiler left them outlined at 1 to 2 ns more per call, and they read the flags' bytes from the caller's iterator rather than from the copy just made of it, so that no load waits on the copy's stores. On the bench, where every call reports one boundary, they cost a grapheme break about 2 ns (11%) and a line break about 5 ns (20%), all of it instructions executed: they add no branch or cache miss. `test/fuzz_iterators.c` sets every byte outside those pointers, and the lengths and counts that go with them, from its input and calls every function again.
+
+Its size follows the state: a `static_assert` in [`src/segmenter.h`](../../src/segmenter.h) keeps `muniSegmenter` within it, so a change that grows the state fails to build until the public iterator grows with it, and on 64-bit targets a second one keeps the iterator from growing a whole `uint64_t` more than the state needs. On 64-bit targets `muniSegmenter` fills all 128 bytes, with the cookie in the two bytes that were the cursor's padding, a line iterator's strictness and writing system packed in the byte after the hold's kind and the class overrides' pointer in the last 8. The next switch or piece of state therefore grows the public iterator, which changes the ABI; from 1.0.0 that is allowed in a minor release, so its pull request is a `feat` with no `!`.
## The engine and its holds
diff --git a/docs/operations/checks.md b/docs/operations/checks.md
index ca205c0..63fe0f5 100644
--- a/docs/operations/checks.md
+++ b/docs/operations/checks.md
@@ -45,7 +45,7 @@ A test that fails without a fault in the change under test is quarantined, as [m
Every fuzz target (`test/fuzz_.c`) checks properties that must hold for any input rather than known answers: no crash, no broken assertion, piecewise iteration giving the whole-text answer, round trips, idempotence where the standard promises it, agreement between parts (validation and conversion, normalization and its quick check, NFKC_Casefold and the normal forms), and output through a short buffer matching output through a large one. `fuzz_icu` compares with ICU, a test-only oracle, where both Unicode versions agree; its header lists what it leaves out and why. The targets take at most 256 bytes, but `fuzz_bidi_long` repeats its input into a text of 2 KiB to 128 KiB, so work that grows faster than the text runs past libFuzzer's 30-second timeout and is a finding.
-Each target's corpus carries over from run to run in one Actions cache entry, `fuzz-corpus-`, holding a directory per target. Every run restores the latest entry, a pull request's included, and adds the committed seeds, which are the floor. After fuzzing, runs on `main` and at night minimize each target's corpus through `tools/fuzz_target.sh --merge` into a fresh directory, swap it in only when the merge finished, and save a new entry. A clean merge prints one line, the target and the size of its merged corpus; an input that crashes the target during the merge fails the step and prints the same report as a fuzzing finding, the target, the kind of finding and the sanitizers' report, never the input. A run whose fuzzing or merge found something saves nothing. A pull request's run reads the corpus too, so a change that breaks a harness's promise fails on its own pull request and not later on `main`. A pull request, or a dispatch on any other branch, never minimizes or saves it, so only `main` writes the corpus. A target whose minimized corpus passes 25 MB fails the step, and the corpora are not saved, so one target cannot crowd the others out of the cache. The `prune-caches` job then deletes the entry the new one replaced. The organization's other libraries run one fuzz job per target, each with its own cache. This repository has ten targets in one job, four at a time, because ten more containers per run would cost more runner time than the targets' corpora need, and the corpora are still kept and minimized separately.
+Each target's corpus carries over from run to run in one Actions cache entry, `fuzz-corpus-`, holding a directory per target. Every run restores the latest entry, a pull request's included, and adds the committed seeds, which are the floor. After fuzzing, runs on `main` and at night minimize each target's corpus through `tools/fuzz_target.sh --merge` into a fresh directory, swap it in only when the merge finished, and save a new entry. A clean merge prints one line, the target and the size of its merged corpus; an input that crashes the target during the merge fails the step and prints the same report as a fuzzing finding, the target, the kind of finding and the sanitizers' report, never the input. A run whose fuzzing or merge found something saves nothing. A pull request's run reads the corpus too, so a change that breaks a harness's promise fails on its own pull request and not later on `main`. A pull request, or a dispatch on any other branch, never minimizes or saves it, so only `main` writes the corpus. A target whose minimized corpus passes 25 MB fails the step, and the corpora are not saved, so one target cannot crowd the others out of the cache. The `prune-caches` job then deletes the entry the new one replaced. The organization's other libraries run one fuzz job per target, each with its own cache. This repository has eleven targets in one job, four at a time, because eleven more containers per run would cost more runner time than the targets' corpora need, and the corpora are still kept and minimized separately.
A crash a fuzzer finds becomes a test in the suite of its module (see [development.md](development.md#reproducing-a-fuzz-finding)).
diff --git a/docs/user/segmentation.md b/docs/user/segmentation.md
index 62ab7ee..0042cec 100644
--- a/docs/user/segmentation.md
+++ b/docs/user/segmentation.md
@@ -1,6 +1,6 @@
# Characters, words, sentences and lines
-`segment.h` finds the boundaries of grapheme clusters, words and sentences (UAX #29) and the line break opportunities (UAX #14). All four use one iterator type, `muniSegmentIterator`, a plain struct the caller keeps on the stack that needs no cleanup.
+`segment.h` finds the boundaries of grapheme clusters, words and sentences (UAX #29) and the line break opportunities (UAX #14). All four use one iterator type, `muniSegmentIterator`, a plain struct the caller keeps on the stack that needs no cleanup. A copy of an initialized iterator is one too, and every call refuses an iterator no init call set up, such as a zeroed one, with `muni_errorInvalid`.
## Grapheme clusters, words and sentences
diff --git a/docs/user/shaping.md b/docs/user/shaping.md
index a407112..27b142c 100644
--- a/docs/user/shaping.md
+++ b/docs/user/shaping.md
@@ -18,7 +18,7 @@ if (muniInitScriptIterator(&scripts, text, length, false) == muni_success)
}
```
-Like the segment iterators, it takes text in pieces through `muniScriptIterator_Feed` after `muni_needMoreText`. `muniFindScriptRuns` writes the runs of a whole text into an array.
+Like the segment iterators, it takes text in pieces through `muniScriptIterator_Feed` after `muni_needMoreText`, and its calls refuse an iterator no init call set up with `muni_errorInvalid`. `muniFindScriptRuns` writes the runs of a whole text into an array.
A text layout pipeline itemizes by paragraph ([bidi](bidi.md)), then by script and bidi run, then by font, and shapes each item.
diff --git a/include/maul-unicode/script.h b/include/maul-unicode/script.h
index 531fbe4..486e550 100644
--- a/include/maul-unicode/script.h
+++ b/include/maul-unicode/script.h
@@ -28,7 +28,10 @@ extern "C"
#endif
// An iterator over script runs. Its contents are private; it is plain
- // data the caller keeps anywhere and needs no cleanup.
+ // data the caller keeps anywhere and needs no cleanup. A copy of an
+ // iterator an init call set up is one too; every call refuses one no
+ // init call set up, such as a zeroed one or one copied from other
+ // bytes, as an invalid argument.
typedef struct muniScriptIterator
{
uint64_t opaque[26];
@@ -65,8 +68,9 @@ extern "C"
/// @param moreFollows true when still more pieces will be fed.
/// @return `muni_success`; `muni_errorState` for an iterator that did
/// not ask for more text, such as one initialized without more
- /// text to follow; `muni_errorInvalid` for a NULL argument.
- /// `muni_errorInvalid` is reported before `muni_errorState`.
+ /// text to follow; `muni_errorInvalid` for a NULL argument or
+ /// an iterator no init call set up. `muni_errorInvalid` is
+ /// reported before `muni_errorState`.
/// @par Thread safety
/// Safe from any thread; an iterator is used by one thread at a time.
MUNI_NODISCARD MUNI_API muniResult muniScriptIterator_Feed(muniScriptIterator* iterator,
@@ -83,7 +87,8 @@ extern "C"
/// returns `muni_success`.
/// @return `muni_success` with a run; `muni_done` after the last one;
/// `muni_needMoreText` when the run's end or script depends on
- /// text not yet fed; `muni_errorInvalid` for a NULL argument.
+ /// text not yet fed; `muni_errorInvalid` for a NULL argument or
+ /// an iterator no init call set up.
/// @par Thread safety
/// Safe from any thread; an iterator is used by one thread at a time.
MUNI_NODISCARD MUNI_API muniResult muniScriptIterator_NextRun(muniScriptIterator* iterator,
diff --git a/include/maul-unicode/segment.h b/include/maul-unicode/segment.h
index 8192b7a..3a27577 100644
--- a/include/maul-unicode/segment.h
+++ b/include/maul-unicode/segment.h
@@ -36,7 +36,9 @@ extern "C"
// An iterator over grapheme cluster, word or sentence boundaries or
// line break opportunities. Its contents are private; it is plain data
// the caller keeps anywhere, usually on the stack, and needs no
- // cleanup.
+ // cleanup. A copy of an iterator an init call set up is one too; every
+ // call refuses one no init call set up, such as a zeroed one or one
+ // copied from other bytes, as an invalid argument.
typedef struct muniSegmentIterator
{
uint64_t opaque[16];
@@ -192,7 +194,8 @@ extern "C"
/// @return `muni_success`; `muni_errorState` for an iterator already
/// asked for a break; `muni_errorBusy` when called from inside
/// the iterator's word segmenter; `muni_errorInvalid` for a NULL
- /// iterator or another kind of iterator. `muni_errorInvalid` is
+ /// iterator, one no init call set up or another kind of
+ /// iterator. `muni_errorInvalid` is
/// reported before `muni_errorBusy`, and that before
/// `muni_errorState`.
/// @par Thread safety
@@ -236,7 +239,8 @@ extern "C"
/// assigns.
/// @return `muni_success`; `muni_errorState` for an iterator already
/// asked for a boundary; `muni_errorInvalid` for a NULL
- /// iterator, NULL overrides with a nonzero count, another kind
+ /// iterator or one no init call set up, NULL overrides with a
+ /// nonzero count, another kind
/// of iterator, a count over 0x10F800, a code point past
/// U+10FFFF or a surrogate, code points out of order or
/// repeated, or a value that is not a class of the iterator's
@@ -257,7 +261,8 @@ extern "C"
/// @return `muni_success`; `muni_errorState` for an iterator already
/// asked for a break; `muni_errorBusy` when called from inside
/// the iterator's word segmenter; `muni_errorInvalid` for a NULL
- /// iterator, another kind of iterator or an unknown strictness.
+ /// iterator, one no init call set up, another kind of iterator
+ /// or an unknown strictness.
/// `muni_errorInvalid` is reported before `muni_errorBusy`, and
/// that before `muni_errorState`.
/// @par Thread safety
@@ -270,7 +275,8 @@ extern "C"
/// @param iterator The iterator.
/// @return The strictness muniSegmentIterator_SetLineBreakStrictness
/// set, muni_lineBreakNormal when none was set, and for a NULL
- /// iterator or another kind of iterator.
+ /// iterator, one no init call set up or another kind of
+ /// iterator.
/// @par Thread safety
/// Safe from any thread; an iterator is used by one thread at a time.
MUNI_API muniLineBreakStrictness
@@ -284,7 +290,8 @@ extern "C"
/// @return `muni_success`; `muni_errorState` for an iterator already
/// asked for a break; `muni_errorBusy` when called from inside
/// the iterator's word segmenter; `muni_errorInvalid` for a NULL
- /// iterator, another kind of iterator or an unknown writing
+ /// iterator, one no init call set up, another kind of iterator
+ /// or an unknown writing
/// system. `muni_errorInvalid` is reported before
/// `muni_errorBusy`, and that before `muni_errorState`.
/// @par Thread safety
@@ -296,8 +303,8 @@ extern "C"
///
/// @param iterator The iterator.
/// @return The writing system muniSegmentIterator_SetWritingSystem set,
- /// muni_writingOther when none was set, and for a NULL iterator
- /// or another kind of iterator.
+ /// muni_writingOther when none was set, and for a NULL iterator,
+ /// one no init call set up or another kind of iterator.
/// @par Thread safety
/// Safe from any thread; an iterator is used by one thread at a time.
MUNI_API muniWritingSystem
@@ -314,9 +321,9 @@ extern "C"
/// not ask for more text, such as one initialized without more
/// text to follow or one fed before it asked; `muni_errorBusy`
/// when called from inside the iterator's word segmenter;
- /// `muni_errorInvalid` for a NULL argument. `muni_errorInvalid`
- /// is reported before `muni_errorBusy`, and that before
- /// `muni_errorState`.
+ /// `muni_errorInvalid` for a NULL argument or an iterator no
+ /// init call set up. `muni_errorInvalid` is reported before
+ /// `muni_errorBusy`, and that before `muni_errorState`.
/// @par Thread safety
/// Safe from any thread; an iterator is used by one thread at a time.
MUNI_NODISCARD MUNI_API muniResult muniSegmentIterator_Feed(muniSegmentIterator* iterator,
@@ -333,7 +340,8 @@ extern "C"
/// one; `muni_needMoreText` when the next boundary depends on
/// text not yet fed; `muni_errorBusy` when called from inside
/// the iterator's word segmenter, and then the iterator is left
- /// as it was; `muni_errorInvalid` for a NULL argument.
+ /// as it was; `muni_errorInvalid` for a NULL argument or an
+ /// iterator no init call set up.
/// @par Thread safety
/// Safe from any thread; an iterator is used by one thread at a time.
MUNI_NODISCARD MUNI_API muniResult muniSegmentIterator_NextBreak(muniSegmentIterator* iterator,
@@ -390,7 +398,8 @@ extern "C"
/// `muni_success`.
/// @return `muni_success`; `muni_errorBusy` when called from inside the
/// iterator's word segmenter; `muni_errorInvalid` for a NULL
- /// argument or an iterator of another kind.
+ /// argument, an iterator no init call set up or one of another
+ /// kind.
/// @par Thread safety
/// Safe from any thread; an iterator is used by one thread at a time.
MUNI_NODISCARD MUNI_API muniResult
diff --git a/src/cursor.h b/src/cursor.h
index 34a9d5f..98d9e0a 100644
--- a/src/cursor.h
+++ b/src/cursor.h
@@ -23,10 +23,37 @@ typedef struct muniCursor
uint8_t pending[4]; // the start of a sequence the previous piece cut off
uint8_t pendingCount; // how many bytes of it there are
bool moreFollows; // whether more pieces will come
+ uint16_t cookie; // the iterator's, written by its init call and checked by every call
} muniCursor;
void muniCursorInit(muniCursor* cursor, const char* text, size_t length, bool moreFollows);
+// Whether the byte at offset in raw, state the caller holds, is a flag's
+// 0 or 1. The iterators check every flag so before the copy of the state
+// that holds it is read: any other byte would be no bool at all.
+static inline bool muniIsFlagByte(const void* raw, size_t offset)
+{
+ return ((const unsigned char*)raw)[offset] <= 1;
+}
+
+// Whether a cursor copied out of raw, from offset at, keeps its counts
+// within its piece and its pending bytes, and its flag a 0 or 1; when it
+// is waiting for a piece, whether more were announced and less than a
+// code point is unread, as muniCursorFeed relies on.
+static inline bool muniCursorValid(const muniCursor* cursor, const void* raw, size_t at,
+ bool waiting)
+{
+ if (!muniIsFlagByte(raw, at + offsetof(muniCursor, moreFollows)) ||
+ cursor->pendingCount > sizeof(cursor->pending) || cursor->position > cursor->length ||
+ (cursor->text == nullptr && cursor->length != 0))
+ {
+ return false;
+ }
+ return !waiting ||
+ (cursor->moreFollows &&
+ cursor->pendingCount + (cursor->length - cursor->position) < sizeof(cursor->pending));
+}
+
// Hands the cursor its next piece. Bytes still unread in the current
// piece are kept as pending. Only a cursor that answered
// muni_needMoreText may be fed: more pieces were announced, and at most
diff --git a/src/grapheme.c b/src/grapheme.c
index 5f4dd9a..c272e9d 100644
--- a/src/grapheme.c
+++ b/src/grapheme.c
@@ -24,6 +24,8 @@ enum
EmojiPictographic = 1, // Extended_Pictographic Extend*
EmojiJoined = 2, // Extended_Pictographic Extend* ZWJ
};
+static_assert((int)EmojiJoined == (int)muni_graphemeEmojiLast,
+ "segment.c checks the emoji state up to it");
static bool IsControl(uint8_t value)
{
diff --git a/src/invariant.h b/src/invariant.h
index b94d3ae..b049630 100644
--- a/src/invariant.h
+++ b/src/invariant.h
@@ -18,4 +18,10 @@
#define MUNI_ASSERT(condition) ((condition) ? (void)0 : __builtin_trap())
#endif
+// MUNI_ALWAYS_INLINE asks that a small function be inlined into every
+// caller, for a check on a call's hot path that the compiler would
+// otherwise leave as a call of its own. Every compiler the family builds
+// with, clang-cl included, takes the attribute.
+#define MUNI_ALWAYS_INLINE inline __attribute__((always_inline))
+
#endif // MAUL_UNICODE_SRC_INVARIANT_H
diff --git a/src/line_rules.h b/src/line_rules.h
index 55a1499..d591e5d 100644
--- a/src/line_rules.h
+++ b/src/line_rules.h
@@ -56,6 +56,7 @@ enum
HoldPrefix = 3, // LB25: (PR | PO) × OP, if (IS)? NU follows
HoldAksara = 4, // LB28a: aksara × aksara, if VF follows
};
+static_assert((int)HoldAksara == (int)muni_lineHoldLast, "segment.c checks the hold kind up to it");
// Where an LB25 number stands.
enum
@@ -64,6 +65,8 @@ enum
NumberDigits = 1, // NU (SY | IS)*
NumberClosed = 2, // NU (SY | IS)* (CL | CP)
};
+static_assert((int)NumberClosed == (int)muni_lineNumberLast,
+ "segment.c checks the number state up to it");
enum : uint64_t
{
diff --git a/src/script_runs.c b/src/script_runs.c
index 496aa16..1ed1fcc 100644
--- a/src/script_runs.c
+++ b/src/script_runs.c
@@ -10,35 +10,14 @@
// a closing bracket that matches one counts as a code point of that
// script, so "(" and ")" around foreign text land in the same script.
-#include "cursor.h"
#include "invariant.h"
+#include "script_state.h"
#include "tables.h"
#include "maul-unicode/script.h"
#include
-#define BRACKET_DEPTH 32
-#define NO_SCRIPT 0xFFu
-
-typedef struct Scripts
-{
- uint64_t bits[4];
-} Scripts;
-
-typedef struct ScriptState
-{
- muniCursor cursor;
- Scripts candidates; // the scripts the run can still have
- uint32_t brackets[BRACKET_DEPTH]; // closing bracket << 8 | Script index or NO_SCRIPT
- uint8_t bracketCount;
- uint8_t firstScript; // the Script of the run's first code point that has one
- bool determined; // a code point with a real script is in the run
- bool started; // a code point has been read
- bool waiting; // the last call returned muni_needMoreText
- bool finished; // the last run has been reported
-} ScriptState;
-
// The public iterator's size follows the state: it holds the state on
// every target, and on 64-bit targets, where the state is largest, it
// has no whole uint64_t to spare. A change to the state that breaks
@@ -265,11 +244,67 @@ static muniResult Next(ScriptState* state, muniScriptRun* runOut)
}
}
-static ScriptState Load(const muniScriptIterator* iterator)
+// What an init call writes in the cursor's cookie, in bytes that were its
+// padding, so that an iterator no init call set up is refused instead of
+// read. Its top bit is set, so a small count or an ASCII byte pattern is
+// no cookie; the checks after it keep any other bit pattern safe too.
+#define COOKIE 0xB3D0u
+
+// Whether a script index names one of muniScriptTags, or is NO_SCRIPT
+// where that is allowed.
+static bool IsScript(uint8_t script, bool none)
{
- ScriptState state;
- memcpy(&state, iterator, sizeof(state));
- return state;
+ return script < muniScriptTagCount || (none && script == NO_SCRIPT);
+}
+
+// Copies the iterator's state out of caller memory when it is one an
+// init call set up and every flag, count and script in it is in range,
+// so that no bit pattern makes a call read outside the state, the text
+// it names or the script tables; false otherwise. An iterator with the
+// cookie whose fields were changed by hand is outside the contract, and
+// how its fields agree with each other is not checked again.
+static bool Load(const muniScriptIterator* iterator, ScriptState* stateOut)
+{
+ memcpy(stateOut, iterator, sizeof(*stateOut));
+ const ScriptState* state = stateOut;
+ if (state->cursor.cookie != COOKIE ||
+ !muniIsFlagByte(iterator, offsetof(ScriptState, determined)) ||
+ !muniIsFlagByte(iterator, offsetof(ScriptState, started)) ||
+ !muniIsFlagByte(iterator, offsetof(ScriptState, waiting)) ||
+ !muniIsFlagByte(iterator, offsetof(ScriptState, finished)) ||
+ !muniCursorValid(&state->cursor, iterator, offsetof(ScriptState, cursor), state->waiting) ||
+ state->bracketCount > BRACKET_DEPTH || !IsScript(state->firstScript, true))
+ {
+ return false;
+ }
+ for (uint8_t k = 0; k < state->bracketCount; k++)
+ {
+ if (!IsScript((uint8_t)(state->brackets[k] & 0xFF), true))
+ {
+ return false;
+ }
+ }
+ // No candidate past the last script, and at least one in a determined
+ // run, which Best relies on.
+ bool any = false;
+ for (unsigned word = 0; word < 4; word++)
+ {
+ uint64_t bits = state->candidates.bits[word];
+ unsigned first = word * 64;
+ if (first >= muniScriptTagCount)
+ {
+ if (bits != 0)
+ {
+ return false;
+ }
+ }
+ else if (muniScriptTagCount - first < 64 && bits >> (muniScriptTagCount - first) != 0)
+ {
+ return false;
+ }
+ any = any || bits != 0;
+ }
+ return any || !state->determined;
}
static void Store(muniScriptIterator* iterator, const ScriptState* state)
@@ -282,6 +317,7 @@ static void Start(ScriptState* state, const char* text, size_t length, bool more
memset(state, 0, sizeof(*state));
state->firstScript = NO_SCRIPT;
muniCursorInit(&state->cursor, text, length, moreFollows);
+ state->cursor.cookie = COOKIE;
}
muniResult muniInitScriptIterator(muniScriptIterator* iterator, const char* text, size_t length,
@@ -305,7 +341,11 @@ muniResult muniScriptIterator_Feed(muniScriptIterator* iterator, const char* tex
{
return muni_errorInvalid;
}
- ScriptState state = Load(iterator);
+ ScriptState state;
+ if (!Load(iterator, &state))
+ {
+ return muni_errorInvalid;
+ }
if (!state.waiting)
{
return muni_errorState;
@@ -322,7 +362,11 @@ muniResult muniScriptIterator_NextRun(muniScriptIterator* iterator, muniScriptRu
{
return muni_errorInvalid;
}
- ScriptState state = Load(iterator);
+ ScriptState state;
+ if (!Load(iterator, &state))
+ {
+ return muni_errorInvalid;
+ }
muniResult status = Next(&state, runOut);
state.waiting = status == muni_needMoreText;
Store(iterator, &state);
diff --git a/src/script_state.h b/src/script_state.h
new file mode 100644
index 0000000..b01160f
--- /dev/null
+++ b/src/script_state.h
@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2026 Sirac Ozmen
+//
+// The state of a script iterator, which script_runs.c keeps in the public
+// iterator's opaque storage. Here so that a fuzz target can change the
+// fields that hold no pointer and see every call refuse what is out of
+// range.
+
+#ifndef MAUL_UNICODE_SRC_SCRIPT_STATE_H
+#define MAUL_UNICODE_SRC_SCRIPT_STATE_H
+
+#include "cursor.h"
+
+#define BRACKET_DEPTH 32
+#define NO_SCRIPT 0xFFu
+
+typedef struct Scripts
+{
+ uint64_t bits[4];
+} Scripts;
+
+typedef struct ScriptState
+{
+ muniCursor cursor;
+ Scripts candidates; // the scripts the run can still have
+ uint32_t brackets[BRACKET_DEPTH]; // closing bracket << 8 | Script index or NO_SCRIPT
+ uint8_t bracketCount;
+ uint8_t firstScript; // the Script of the run's first code point that has one
+ bool determined; // a code point with a real script is in the run
+ bool started; // a code point has been read
+ bool waiting; // the last call returned muni_needMoreText
+ bool finished; // the last run has been reported
+} ScriptState;
+
+#endif // MAUL_UNICODE_SRC_SCRIPT_STATE_H
diff --git a/src/segment.c b/src/segment.c
index e51d245..6b4926a 100644
--- a/src/segment.c
+++ b/src/segment.c
@@ -13,11 +13,129 @@
#include
#include
-static muniSegmenter Load(const muniSegmentIterator* iterator)
+// What an init call writes in the cursor's cookie, in bytes that were its
+// padding, so that an iterator no init call set up is refused instead of
+// read. Its top bit is set, so a small count or an ASCII byte pattern is
+// no cookie; the checks after it keep any other bit pattern safe too.
+#define COOKIE 0xC5E6u
+
+// The most overrides a table can hold: one per scalar value, since its
+// code points strictly increase.
+#define MAX_OVERRIDES 0x10F800u
+
+// The byte at offset in the iterator in caller memory. The flags are
+// checked as these bytes, before the copy is read as bools. Reading them
+// from the iterator rather than the copy just made of it keeps each load
+// from waiting on the copy's stores.
+static inline unsigned Byte(const muniSegmentIterator* iterator, size_t offset)
{
- muniSegmenter segmenter;
- memcpy(&segmenter, iterator, sizeof(segmenter));
- return segmenter;
+ return ((const unsigned char*)iterator)[offset];
+}
+
+// Whether the rules of a segmenter copied out of iterator hold what their
+// rule set writes, each flag's byte a 0 or 1 included.
+static MUNI_ALWAYS_INLINE bool RulesValid(const muniSegmenter* segmenter,
+ const muniSegmentIterator* iterator)
+{
+ switch (segmenter->kind)
+ {
+ case muni_segmentGrapheme:
+ {
+ const muniGraphemeRules* rules = &segmenter->rules.grapheme;
+ return segmenter->lineSettings == 0 && segmenter->holdKind == 0 &&
+ rules->previous <= muni_gcbLvt && rules->emoji <= muni_graphemeEmojiLast &&
+ (Byte(iterator, offsetof(muniSegmenter, rules.grapheme.linker)) |
+ Byte(iterator, offsetof(muniSegmenter, rules.grapheme.oddRegional))) <= 1;
+ }
+ case muni_segmentWord:
+ {
+ const muniWordRules* rules = &segmenter->rules.word;
+ uint32_t count = rules->overrideCount;
+ return segmenter->lineSettings == 0 && segmenter->holdKind == 0 &&
+ rules->actual <= muni_wbWSegSpace && rules->previous <= muni_wbWSegSpace &&
+ rules->before <= muni_wbWSegSpace && rules->kind <= muni_wordIdeograph &&
+ rules->reportedKind <= muni_wordIdeograph && count <= MAX_OVERRIDES &&
+ (count == 0 || segmenter->overrides != nullptr) &&
+ Byte(iterator, offsetof(muniSegmenter, rules.word.oddRegional)) <= 1;
+ }
+ case muni_segmentSentence:
+ {
+ const muniSentenceRules* rules = &segmenter->rules.sentence;
+ uint32_t count = rules->overrideCount;
+ return segmenter->lineSettings == 0 && segmenter->holdKind == 0 &&
+ rules->actual <= muni_sbClose && rules->previous <= muni_sbClose &&
+ rules->before <= muni_sbClose && rules->ending <= muni_sentenceEndingLast &&
+ count <= MAX_OVERRIDES && (count == 0 || segmenter->overrides != nullptr) &&
+ Byte(iterator, offsetof(muniSegmenter, rules.sentence.aterm)) <= 1;
+ }
+ case muni_segmentLine:
+ {
+ // Breaking anywhere keeps the grapheme rules where the other
+ // settings keep the classes before the position.
+ const muniLineRules* rules = &segmenter->rules.line;
+ if (segmenter->lineSettings >> 3 != 0 || rules->actual > muni_lbHh)
+ {
+ return false;
+ }
+ if (muniLineStrictness(segmenter) == muni_lineBreakAnywhere)
+ {
+ return segmenter->holdKind == 0 && rules->grapheme.previous <= muni_gcbLvt &&
+ rules->grapheme.emoji <= muni_graphemeEmojiLast &&
+ (Byte(iterator, offsetof(muniSegmenter, rules.line.grapheme.linker)) |
+ Byte(iterator, offsetof(muniSegmenter, rules.line.grapheme.oddRegional))) <= 1;
+ }
+ return segmenter->holdKind <= muni_lineHoldLast && rules->previous <= muni_lbHh &&
+ rules->before <= muni_lbHh && rules->lead <= muni_lbHh &&
+ rules->number <= muni_lineNumberLast &&
+ (Byte(iterator, offsetof(muniSegmenter, rules.line.leadInitialQuote)) |
+ Byte(iterator, offsetof(muniSegmenter, rules.line.sawInfix)) |
+ Byte(iterator, offsetof(muniSegmenter, rules.line.oddRegional))) <= 1;
+ }
+ default:
+ return false;
+ }
+}
+
+// Copies the iterator's state out of caller memory when it is one an
+// init call set up and every flag, count and value in it is in range, so
+// that no bit pattern makes a call read outside the state or the text
+// and overrides it names; false otherwise. An iterator with the cookie
+// whose fields were changed by hand is outside the contract, and how its
+// fields agree with each other is not checked again. Every call reads
+// most of the state, so all of it is checked, inline in each call so that
+// the copy stays inline too.
+static MUNI_ALWAYS_INLINE bool Load(const muniSegmentIterator* iterator,
+ muniSegmenter* segmenterOut)
+{
+ memcpy(segmenterOut, iterator, sizeof(*segmenterOut));
+ const muniSegmenter* segmenter = segmenterOut;
+ const muniCursor* cursor = &segmenter->cursor;
+ size_t at = offsetof(muniSegmenter, cursor);
+ unsigned more = Byte(iterator, at + offsetof(muniCursor, moreFollows));
+ unsigned started = Byte(iterator, offsetof(muniSegmenter, started));
+ unsigned holding = Byte(iterator, offsetof(muniSegmenter, holding));
+ if (cursor->cookie != COOKIE || segmenter->phase > muni_phaseBusy ||
+ (more | started | holding | Byte(iterator, offsetof(muniSegmenter, endReported)) |
+ Byte(iterator, offsetof(muniSegmenter, mandatory))) > 1 ||
+ cursor->pendingCount > sizeof(cursor->pending) || cursor->position > cursor->length ||
+ (cursor->text == nullptr && cursor->length != 0))
+ {
+ return false;
+ }
+ // A waiting cursor holds less than a code point, as muniCursorFeed
+ // relies on; a boundary lies after a code point and a hold before the
+ // cursor, as the engine's assertions do.
+ size_t offset = cursor->offset;
+ if ((segmenter->phase == muni_phaseWaiting &&
+ (more == 0 ||
+ cursor->pendingCount + (cursor->length - cursor->position) >= sizeof(cursor->pending))) ||
+ (started != 0 && offset == 0) ||
+ (holding != 0 && (segmenter->heldOffset == 0 || segmenter->heldOffset >= offset)) ||
+ (segmenter->complex.text != nullptr && segmenter->complex.next > segmenter->complex.length))
+ {
+ return false;
+ }
+ return RulesValid(segmenter, iterator);
}
static void Store(muniSegmentIterator* iterator, const muniSegmenter* segmenter)
@@ -49,6 +167,7 @@ static muniResult Init(muniSegmentIterator* iterator, uint8_t kind, const char*
memset(&segmenter, 0, sizeof(segmenter));
segmenter.kind = kind;
muniCursorInit(&segmenter.cursor, text, length, moreFollows);
+ segmenter.cursor.cookie = COOKIE;
memset(iterator, 0, sizeof(*iterator));
Store(iterator, &segmenter);
return muni_success;
@@ -85,7 +204,11 @@ muniResult muniSegmentIterator_SetComplexBreaker(muniSegmentIterator* iterator,
{
return muni_errorInvalid;
}
- muniSegmenter segmenter = Load(iterator);
+ muniSegmenter segmenter;
+ if (!Load(iterator, &segmenter))
+ {
+ return muni_errorInvalid;
+ }
bool supported = segmenter.kind == muni_segmentLine || segmenter.kind == muni_segmentWord;
if (!supported)
{
@@ -105,10 +228,6 @@ muniResult muniSegmentIterator_SetComplexBreaker(muniSegmentIterator* iterator,
return muni_success;
}
-// The most overrides a table can hold: one per scalar value, since its
-// code points strictly increase.
-#define MAX_OVERRIDES 0x10F800u
-
// Whether a table of overrides is in strictly increasing order of code
// point and names only code points and known classes: up to classCount.
static bool ValidOverrides(const muniBreakClassOverride* overrides, size_t count,
@@ -135,7 +254,11 @@ muniResult muniSegmentIterator_SetBreakClassOverrides(muniSegmentIterator* itera
{
return muni_errorInvalid;
}
- muniSegmenter segmenter = Load(iterator);
+ muniSegmenter segmenter;
+ if (!Load(iterator, &segmenter))
+ {
+ return muni_errorInvalid;
+ }
uint32_t* countField = nullptr;
uint8_t classCount = 0;
if (segmenter.kind == muni_segmentWord)
@@ -176,7 +299,11 @@ static muniResult SetLineSettings(muniSegmentIterator* iterator, int32_t value,
{
return muni_errorInvalid;
}
- muniSegmenter segmenter = Load(iterator);
+ muniSegmenter segmenter;
+ if (!Load(iterator, &segmenter))
+ {
+ return muni_errorInvalid;
+ }
if (segmenter.kind != muni_segmentLine)
{
return muni_errorInvalid;
@@ -208,7 +335,11 @@ muniSegmentIterator_GetLineBreakStrictness(const muniSegmentIterator* iterator)
{
return muni_lineBreakNormal;
}
- muniSegmenter segmenter = Load(iterator);
+ muniSegmenter segmenter;
+ if (!Load(iterator, &segmenter))
+ {
+ return muni_lineBreakNormal;
+ }
return segmenter.kind == muni_segmentLine ? muniLineStrictness(&segmenter)
: muni_lineBreakNormal;
}
@@ -226,7 +357,11 @@ muniWritingSystem muniSegmentIterator_GetWritingSystem(const muniSegmentIterator
{
return muni_writingOther;
}
- muniSegmenter segmenter = Load(iterator);
+ muniSegmenter segmenter;
+ if (!Load(iterator, &segmenter))
+ {
+ return muni_writingOther;
+ }
return segmenter.kind == muni_segmentLine ? muniLineWritingSystem(&segmenter)
: muni_writingOther;
}
@@ -238,7 +373,11 @@ muniResult muniSegmentIterator_Feed(muniSegmentIterator* iterator, const char* t
{
return muni_errorInvalid;
}
- muniSegmenter segmenter = Load(iterator);
+ muniSegmenter segmenter;
+ if (!Load(iterator, &segmenter))
+ {
+ return muni_errorInvalid;
+ }
if (segmenter.phase == muni_phaseBusy)
{
return muni_errorBusy;
@@ -283,7 +422,11 @@ muniResult muniSegmentIterator_NextBreak(muniSegmentIterator* iterator, size_t*
{
return muni_errorInvalid;
}
- muniSegmenter segmenter = Load(iterator);
+ muniSegmenter segmenter;
+ if (!Load(iterator, &segmenter))
+ {
+ return muni_errorInvalid;
+ }
if (segmenter.phase == muni_phaseBusy)
{
return muni_errorBusy;
@@ -302,7 +445,11 @@ muniResult muniSegmentIterator_NextLineBreak(muniSegmentIterator* iterator, size
{
return muni_errorInvalid;
}
- muniSegmenter segmenter = Load(iterator);
+ muniSegmenter segmenter;
+ if (!Load(iterator, &segmenter))
+ {
+ return muni_errorInvalid;
+ }
if (segmenter.kind != muni_segmentLine)
{
return muni_errorInvalid;
@@ -380,7 +527,11 @@ muniResult muniSegmentIterator_GetWordKind(const muniSegmentIterator* iterator,
{
return muni_errorInvalid;
}
- muniSegmenter segmenter = Load(iterator);
+ muniSegmenter segmenter;
+ if (!Load(iterator, &segmenter))
+ {
+ return muni_errorInvalid;
+ }
if (segmenter.kind != muni_segmentWord)
{
return muni_errorInvalid;
diff --git a/src/segmenter.h b/src/segmenter.h
index 75671fa..0ad34b3 100644
--- a/src/segmenter.h
+++ b/src/segmenter.h
@@ -110,6 +110,17 @@ typedef struct muniLineRules
bool oddRegional; // an odd number of regional indicators precedes
} muniLineRules;
+// The last value each rule set writes in its states, for the check of an
+// iterator the caller holds; each rule set ties its own enumeration to
+// these with a static_assert.
+enum
+{
+ muni_graphemeEmojiLast = 2, // grapheme.c's EmojiJoined
+ muni_sentenceEndingLast = 2, // sentence.c's EndingSpace
+ muni_lineNumberLast = 2, // line_rules.h's NumberClosed
+ muni_lineHoldLast = 4, // line_rules.h's HoldAksara
+};
+
// A run of Thai, Lao, Khmer or Burmese (Line_Break=SA) that a caller's
// breaker segments. The run lies in the current piece.
typedef struct muniComplexRun
@@ -149,8 +160,15 @@ typedef struct muniSegmenter
const muniBreakClassOverride* overrides;
} muniSegmenter;
+// The public iterator's size follows the segmenter: it holds it on every
+// target, and on 64-bit targets, where the segmenter is largest, it has
+// no whole uint64_t to spare. A change that breaks either resizes the
+// public iterator, which a minor release may do.
static_assert(sizeof(muniSegmenter) <= sizeof(muniSegmentIterator),
"the public iterator must hold the segmenter");
+static_assert(sizeof(void*) < 8 ||
+ sizeof(muniSegmentIterator) - sizeof(muniSegmenter) < sizeof(uint64_t),
+ "the public iterator must be no larger than the segmenter needs");
// The parts of lineSettings: a muni_lineBreak value in the low two bits,
// a muni_writing value in the third.
diff --git a/src/sentence.c b/src/sentence.c
index f204f75..bf3ab1a 100644
--- a/src/sentence.c
+++ b/src/sentence.c
@@ -22,6 +22,8 @@ enum
EndingTerm = 1, // SATerm Close*
EndingSpace = 2, // SATerm Close* Sp+
};
+static_assert((int)EndingSpace == (int)muni_sentenceEndingLast,
+ "segment.c checks the ending up to it");
enum
{
diff --git a/test/fuzz_iterators.c b/test/fuzz_iterators.c
new file mode 100644
index 0000000..09676f9
--- /dev/null
+++ b/test/fuzz_iterators.c
@@ -0,0 +1,204 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2026 Sirac Ozmen
+//
+// libFuzzer target for the state the iterators keep in caller memory. The
+// first input byte picks an iterator and its settings, the second how
+// many calls it answers first, and each pair of bytes after them sets one
+// byte of its state to a value. The pointers, and the lengths and counts
+// that go with them (the text, the class overrides, the caller's
+// breaker), are left alone: an iterator forged there is outside the
+// contract. Every call on what is left must succeed or refuse, without
+// reading or writing outside the state, the text or the overrides, and
+// without tripping an assertion.
+
+#include "script_state.h"
+#include "segmenter.h"
+
+#include "maul-unicode/script.h"
+#include "maul-unicode/segment.h"
+
+#include
+#include
+#include
+
+// Thai for the breaker, a CR LF, an emoji sequence, brackets around
+// Greek, Han, digits and spaces.
+static const char s_text[] =
+ "\xE0\xB8\xAA\xE0\xB8\xA7\xE0\xB8\xB1\xE0\xB8\xAA\xE0\xB8\x94\xE0\xB8\xB5"
+ " ab\r\n\xF0\x9F\x91\xA9\xE2\x80\x8D\xF0\x9F\x92\xBB (\xCE\xB1\xCE\xB2) "
+ "\xE4\xBA\xBA 1,5% x";
+
+static const muniBreakClassOverride s_overrides[] = {{'_', muni_wbExtendNumLet},
+ {'-', muni_wbMidLetter}};
+
+typedef struct Range
+{
+ size_t start;
+ size_t size;
+} Range;
+
+static size_t Breaker(void* context, const char* run, size_t length, size_t from)
+{
+ (void)context;
+ (void)run;
+ return from + 3 < length ? from + 3 : length;
+}
+
+static bool Inside(const Range* ranges, size_t count, size_t offset)
+{
+ for (size_t i = 0; i < count; i++)
+ {
+ if (offset - ranges[i].start < ranges[i].size)
+ {
+ return true;
+ }
+ }
+ return false;
+}
+
+// Sets the state's bytes the input names, outside the ranges given.
+static void Overwrite(unsigned char* state, size_t size, const Range* ranges, size_t count,
+ const uint8_t* data, size_t length)
+{
+ for (size_t i = 0; i + 1 < length; i += 2)
+ {
+ size_t offset = data[i] % size;
+ if (!Inside(ranges, count, offset))
+ {
+ state[offset] = data[i + 1];
+ }
+ }
+}
+
+static void Script(uint8_t steps, const uint8_t* data, size_t length)
+{
+ muniScriptIterator iterator;
+ muniScriptRun run;
+ if (muniInitScriptIterator(&iterator, s_text, sizeof(s_text) - 1, false) != muni_success)
+ {
+ return;
+ }
+ for (uint8_t i = 0; i < steps % 8; i++)
+ {
+ (void)muniScriptIterator_NextRun(&iterator, &run);
+ }
+ static const Range kept[] = {
+ {offsetof(ScriptState, cursor) + offsetof(muniCursor, text), sizeof(const uint8_t*)},
+ {offsetof(ScriptState, cursor) + offsetof(muniCursor, length), sizeof(size_t)},
+ };
+ Overwrite((unsigned char*)&iterator, sizeof(ScriptState), kept, sizeof(kept) / sizeof(kept[0]),
+ data, length);
+ for (int i = 0; i < 64; i++)
+ {
+ muniResult status = muniScriptIterator_NextRun(&iterator, &run);
+ if (status == muni_needMoreText)
+ {
+ status = muniScriptIterator_Feed(&iterator, "", 0, false);
+ }
+ if (status != muni_success)
+ {
+ break;
+ }
+ }
+}
+
+static void Segment(uint8_t choice, uint8_t steps, const uint8_t* data, size_t length)
+{
+ muniSegmentIterator iterator;
+ size_t offset;
+ bool mandatory;
+ bool more = (choice & 8) != 0;
+ muniResult status;
+ switch (choice % 4)
+ {
+ case 0:
+ status = muniInitGraphemeIterator(&iterator, s_text, sizeof(s_text) - 1, more);
+ break;
+ case 1:
+ status = muniInitWordIterator(&iterator, s_text, sizeof(s_text) - 1, more);
+ if (status == muni_success && (choice & 16) != 0)
+ {
+ status = muniSegmentIterator_SetBreakClassOverrides(&iterator, s_overrides, 2);
+ }
+ break;
+ case 2:
+ status = muniInitSentenceIterator(&iterator, s_text, sizeof(s_text) - 1, more);
+ break;
+ default:
+ status = muniInitLineIterator(&iterator, s_text, sizeof(s_text) - 1, more);
+ if (status == muni_success)
+ {
+ status = muniSegmentIterator_SetLineBreakStrictness(&iterator, (choice >> 5) & 3);
+ }
+ break;
+ }
+ if (status != muni_success)
+ {
+ return;
+ }
+ if ((choice & 32) != 0)
+ {
+ (void)muniSegmentIterator_SetComplexBreaker(&iterator, Breaker, nullptr);
+ }
+ for (uint8_t i = 0; i < steps % 8; i++)
+ {
+ (void)muniSegmentIterator_NextBreak(&iterator, &offset);
+ }
+ // The override count shares the rules' storage, so it is kept only
+ // where the rules are a word or sentence iterator's.
+ Range kept[] = {
+ {offsetof(muniSegmenter, cursor) + offsetof(muniCursor, text), sizeof(const uint8_t*)},
+ {offsetof(muniSegmenter, cursor) + offsetof(muniCursor, length), sizeof(size_t)},
+ {offsetof(muniSegmenter, complex), sizeof(muniComplexRun)},
+ {offsetof(muniSegmenter, kind), 1},
+ {offsetof(muniSegmenter, overrides), sizeof(const muniBreakClassOverride*)},
+ {0, 0},
+ };
+ if (choice % 4 == 1)
+ {
+ kept[5] = (Range){offsetof(muniSegmenter, rules.word.overrideCount), sizeof(uint32_t)};
+ }
+ else if (choice % 4 == 2)
+ {
+ kept[5] = (Range){offsetof(muniSegmenter, rules.sentence.overrideCount), sizeof(uint32_t)};
+ }
+ Overwrite((unsigned char*)&iterator, sizeof(muniSegmenter), kept,
+ sizeof(kept) / sizeof(kept[0]), data, length);
+ muniWordKind kind;
+ (void)muniSegmentIterator_GetWordKind(&iterator, &kind);
+ (void)muniSegmentIterator_GetLineBreakStrictness(&iterator);
+ (void)muniSegmentIterator_GetWritingSystem(&iterator);
+ for (int i = 0; i < 64; i++)
+ {
+ status = (choice & 64) != 0
+ ? muniSegmentIterator_NextLineBreak(&iterator, &offset, &mandatory)
+ : muniSegmentIterator_NextBreak(&iterator, &offset);
+ if (status == muni_needMoreText)
+ {
+ status = muniSegmentIterator_Feed(&iterator, "", 0, false);
+ }
+ if (status != muni_success)
+ {
+ break;
+ }
+ }
+}
+
+int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size);
+
+int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
+{
+ if (size < 2)
+ {
+ return 0;
+ }
+ if ((data[0] & 128) != 0)
+ {
+ Script(data[1], data + 2, size - 2);
+ }
+ else
+ {
+ Segment(data[0], data[1], data + 2, size - 2);
+ }
+ return 0;
+}
diff --git a/test/seeds/fuzz_iterators/grapheme-more b/test/seeds/fuzz_iterators/grapheme-more
new file mode 100644
index 0000000..88726d6
--- /dev/null
+++ b/test/seeds/fuzz_iterators/grapheme-more
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/test/seeds/fuzz_iterators/line-breaker b/test/seeds/fuzz_iterators/line-breaker
new file mode 100644
index 0000000..1310f4a
--- /dev/null
+++ b/test/seeds/fuzz_iterators/line-breaker
@@ -0,0 +1 @@
+c`h
\ No newline at end of file
diff --git a/test/seeds/fuzz_iterators/script b/test/seeds/fuzz_iterators/script
new file mode 100644
index 0000000..24b6421
--- /dev/null
+++ b/test/seeds/fuzz_iterators/script
@@ -0,0 +1 @@
+€
\ No newline at end of file
diff --git a/test/seeds/fuzz_iterators/script-overwritten b/test/seeds/fuzz_iterators/script-overwritten
new file mode 100644
index 0000000..ad3d82c
--- /dev/null
+++ b/test/seeds/fuzz_iterators/script-overwritten
@@ -0,0 +1 @@
+€0A1È
\ No newline at end of file
diff --git a/test/seeds/fuzz_iterators/sentence b/test/seeds/fuzz_iterators/sentence
new file mode 100644
index 0000000..ad45e54
--- /dev/null
+++ b/test/seeds/fuzz_iterators/sentence
@@ -0,0 +1 @@
+p
\ No newline at end of file
diff --git a/test/seeds/fuzz_iterators/word-overrides b/test/seeds/fuzz_iterators/word-overrides
new file mode 100644
index 0000000..8e6e78e
--- /dev/null
+++ b/test/seeds/fuzz_iterators/word-overrides
@@ -0,0 +1 @@
+PAQ
\ No newline at end of file
diff --git a/test/test_script.c b/test/test_script.c
index bac96cd..a70e4bd 100644
--- a/test/test_script.c
+++ b/test/test_script.c
@@ -229,6 +229,35 @@ static void TestCapacityAndArguments(void)
"no count, and no run written");
}
+// An iterator no init call set up, of one byte repeated: zeroed as a
+// static one is, or of bytes that would make the pending count copy past
+// its four bytes (0x41) or a flag no bool (0x02, 0x04). Both calls that
+// take it back refuse it as a bad argument and change nothing in it.
+static void TestUninitializedIterators(void)
+{
+ static const unsigned char fills[] = {0x00, 0x41, 0x02, 0x04};
+ for (size_t i = 0; i < sizeof(fills); i++)
+ {
+ muniScriptIterator iterator;
+ memset(&iterator, fills[i], sizeof(iterator));
+ muniScriptIterator before = iterator;
+ muniScriptRun run = {7, LATN};
+ CHECK(muniScriptIterator_NextRun(&iterator, &run) == muni_errorInvalid &&
+ muniScriptIterator_Feed(&iterator, "a", 1, false) == muni_errorInvalid &&
+ run.end == 7 && run.script == LATN &&
+ memcmp(&iterator, &before, sizeof(iterator)) == 0,
+ "an iterator no init call set up is refused and left as it was");
+ }
+ // A copy of an iterator an init call set up is one too.
+ muniScriptIterator iterator;
+ muniScriptRun run;
+ CHECK(muniInitScriptIterator(&iterator, "ab", 2, false) == muni_success, "init");
+ muniScriptIterator copy = iterator;
+ CHECK(muniScriptIterator_NextRun(©, &run) == muni_success && run.end == 2 &&
+ run.script == LATN,
+ "a copy of an iterator steps on");
+}
+
int main(void)
{
TestRuns();
@@ -237,5 +266,6 @@ int main(void)
TestBracketDepth();
TestScriptExtensionsList();
TestCapacityAndArguments();
+ TestUninitializedIterators();
return s_failures == 0 ? 0 : 1;
}
diff --git a/test/test_segment.c b/test/test_segment.c
index eebee69..3793f26 100644
--- a/test/test_segment.c
+++ b/test/test_segment.c
@@ -1170,6 +1170,53 @@ static void TestFeedingRules(void)
"never initialized");
}
+// An iterator no init call set up, of one byte repeated: zeroed as a
+// static one is, or of bytes that would make a count run past the state
+// (0x41) or a flag no bool (0x02, 0x04). Every call that takes it back
+// refuses it as a bad argument and changes nothing in it, and the getters
+// give their defaults.
+static void TestUninitializedIterators(void)
+{
+ static const unsigned char fills[] = {0x00, 0x41, 0x02, 0x04};
+ static const muniBreakClassOverride posix[] = {{'_', muni_wbExtendNumLet}};
+ static const size_t stops[] = {0};
+ for (size_t i = 0; i < sizeof(fills); i++)
+ {
+ muniSegmentIterator iterator;
+ memset(&iterator, fills[i], sizeof(iterator));
+ muniSegmentIterator before = iterator;
+ size_t offset = 7;
+ bool mandatory = true;
+ muniWordKind kind = muni_wordLetter;
+ bool refused =
+ muniSegmentIterator_NextBreak(&iterator, &offset) == muni_errorInvalid &&
+ muniSegmentIterator_NextLineBreak(&iterator, &offset, &mandatory) ==
+ muni_errorInvalid &&
+ muniSegmentIterator_Feed(&iterator, "a", 1, false) == muni_errorInvalid &&
+ muniSegmentIterator_SetComplexBreaker(&iterator, ToyBreaker, (void*)stops) ==
+ muni_errorInvalid &&
+ muniSegmentIterator_SetBreakClassOverrides(&iterator, posix, 1) == muni_errorInvalid &&
+ muniSegmentIterator_SetLineBreakStrictness(&iterator, muni_lineBreakLoose) ==
+ muni_errorInvalid &&
+ muniSegmentIterator_SetWritingSystem(&iterator, muni_writingChineseJapanese) ==
+ muni_errorInvalid &&
+ muniSegmentIterator_GetWordKind(&iterator, &kind) == muni_errorInvalid;
+ bool defaults =
+ muniSegmentIterator_GetLineBreakStrictness(&iterator) == muni_lineBreakNormal &&
+ muniSegmentIterator_GetWritingSystem(&iterator) == muni_writingOther;
+ CHECK(refused && defaults && offset == 7 && mandatory && kind == muni_wordLetter &&
+ memcmp(&iterator, &before, sizeof(iterator)) == 0,
+ "an iterator no init call set up is refused and left as it was");
+ }
+ // A copy of an iterator an init call set up is one too.
+ muniSegmentIterator iterator;
+ size_t offset = 0;
+ CHECK(muniInitWordIterator(&iterator, "a b", 3, false) == muni_success, "init");
+ muniSegmentIterator copy = iterator;
+ CHECK(muniSegmentIterator_NextBreak(©, &offset) == muni_success && offset == 1,
+ "a copy of an iterator steps on");
+}
+
// A kind is an int32_t, so a caller can pass a negative one; every call
// that takes a kind refuses -1 and INT32_MIN as it refuses a kind past the
// last, and leaves its outputs as they were.
@@ -1525,6 +1572,7 @@ int main(void)
TestAnywhere();
TestInvalidArguments();
TestFeedingRules();
+ TestUninitializedIterators();
TestWordKinds();
TestMeasureLimits();
TestMeasureUses();
diff --git a/tools/modules.txt b/tools/modules.txt
index b06cc62..0fae5cf 100755
--- a/tools/modules.txt
+++ b/tools/modules.txt
@@ -27,7 +27,8 @@ normalizer: decompose tables writer
overlap:
nfkc_casefold: case_map decompose encoding invariant normalizer overlap tables
properties: tables
-script_runs: cursor invariant tables
+script_runs: invariant script_state tables
+script_state: cursor
segment: encoding segmenter
segmenter: cursor invariant
security: bits encoding tables