From 7e7bd04b20cb61fa050178cd08b7d25542dec1d6 Mon Sep 17 00:00:00 2001 From: olabusayoT <50379531+olabusayoT@users.noreply.github.com> Date: Fri, 3 Apr 2026 14:40:43 -0400 Subject: [PATCH 1/9] Add support for `dfdl:lengthKind="endOfParent"` - Implemented logic to handle elements with `dfdl:lengthKind="endOfParent"`. - Added validations to enforce schema/spec constraints and error conditions specific to `endOfParent` lengthKind. - Introduced determination of effective length units for parent elements and related error checks. - removed NYI Errors - add tests for endOfParent elements with different LengthKinds incl nested EndOfParent DAFFODIL-238 --- .../core/dsom/LocalElementMixin.scala | 6 +- .../org/apache/daffodil/core/dsom/Term.scala | 20 + .../daffodil/core/grammar/AlignedMixin.scala | 6 +- .../grammar/ElementBaseGrammarMixin.scala | 157 +- .../primitives/PrimitivesLengthKind.scala | 10 + .../grammar/primitives/SpecifiedLength.scala | 24 + .../parsers/HexBinaryLengthParsers.scala | 7 +- ...yNumberTraits.scala => ParserTraits.scala} | 0 .../parsers/SpecifiedLengthParsers.scala | 11 + .../runtime1/SpecifiedLengthUnparsers.scala | 14 + .../section12/aligned_data/Aligned_Data.tdml | 79 + .../lengthKind/EndOfParentTests.tdml | 1965 ++++++++++++++++- .../aligned_data/TestAlignedData.scala | 4 + .../TestLengthKindEndOfParent.scala | 74 + .../TestLengthKindEndOfParent2.scala | 34 - 15 files changed, 2319 insertions(+), 92 deletions(-) rename daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/{BinaryNumberTraits.scala => ParserTraits.scala} (100%) create mode 100644 daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindEndOfParent.scala delete mode 100644 daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindEndOfParent2.scala diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/LocalElementMixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/LocalElementMixin.scala index 0e9aaf57e9..4756afcc0f 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/LocalElementMixin.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/LocalElementMixin.scala @@ -61,10 +61,8 @@ trait LocalElementMixin extends ParticleMixin with LocalElementGrammarMixin { else if (representation =:= Representation.Binary) true else false } - case LengthKind.EndOfParent if isComplexType => - notYetImplemented("lengthKind='endOfParent' for complex type") - case LengthKind.EndOfParent => - notYetImplemented("lengthKind='endOfParent' for simple type") + // per DFDL Spec 9.3.2, endOfParent is already positioned at parent's end so length is zero + case LengthKind.EndOfParent => false } res }.value diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/Term.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/Term.scala index 7558483d5b..4e6495b3d1 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/Term.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/Term.scala @@ -269,6 +269,26 @@ trait Term .getOrElse(false) } + final lazy val immediatelyEnclosingElementParent: Option[ElementBase] = { + val p = optLexicalParent.flatMap { + case e: ElementBase => Some(e) + case ge: GlobalElementDecl => Some(ge.asRoot) + case s: SequenceTermBase => s.immediatelyEnclosingElementParent + case c: ChoiceTermBase => c.immediatelyEnclosingElementParent + case ct: ComplexTypeBase => { + ct.optLexicalParent.flatMap { + case e: ElementBase => Some(e) + case ge: GlobalElementDecl => Some(ge.asRoot) + case _ => { + None + } + } + } + case _ => None + } + p + } + final lazy val immediatelyEnclosingGroupDef: Option[GroupDefLike] = { optLexicalParent.flatMap { lexicalParent => val res: Option[GroupDefLike] = lexicalParent match { diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/AlignedMixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/AlignedMixin.scala index 00ebdee130..63c7936eb3 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/AlignedMixin.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/AlignedMixin.scala @@ -456,7 +456,11 @@ trait AlignedMixin extends GrammarMixin { self: Term => } case LengthKind.Delimited => encodingLengthApprox case LengthKind.Pattern => encodingLengthApprox - case LengthKind.EndOfParent => LengthMultipleOf(1) // NYI + case LengthKind.EndOfParent => + eb.immediatelyEnclosingElementParent match { + case Some(parent) => parent.elementSpecifiedLengthApprox + case _ => LengthMultipleOf(1) + } // If an element is lengthKind="prefixed", the element's length is the length // of the value of the prefix element, which can't be known till runtime case LengthKind.Prefixed => LengthMultipleOf(1) // NYI (see DAFFODIL-3066) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala index 3a33fdfb07..011f323b23 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala @@ -19,10 +19,14 @@ package org.apache.daffodil.core.grammar import java.lang.Long as JLong +import org.apache.daffodil.core.dsom.ChoiceTermBase import org.apache.daffodil.core.dsom.ElementBase import org.apache.daffodil.core.dsom.ExpressionCompilers import org.apache.daffodil.core.dsom.InitiatedTerminatedMixin +import org.apache.daffodil.core.dsom.ModelGroup import org.apache.daffodil.core.dsom.PrefixLengthQuasiElementDecl +import org.apache.daffodil.core.dsom.Root +import org.apache.daffodil.core.dsom.SequenceTermBase import org.apache.daffodil.core.grammar.primitives.* import org.apache.daffodil.core.runtime1.ElementBaseRuntime1Mixin import org.apache.daffodil.lib.exceptions.Assert @@ -52,6 +56,7 @@ trait ElementBaseGrammarMixin requiredEvaluationsIfActivated(checkPrefixedLengthElementDecl) requiredEvaluationsIfActivated(checkDelimitedLengthEVDP) + requiredEvaluationsIfActivated(checkEndOfParentElem) private val context = this @@ -252,6 +257,134 @@ trait ElementBaseGrammarMixin } final lazy val prefixedLengthBody = prefixedLengthElementDecl.parsedValue + final lazy val parentEffectiveLengthUnits: LengthUnits = + immediatelyEnclosingElementParent match { + case Some(parent: ElementBase) => { + parent.lengthKind match { + case LengthKind.Explicit | LengthKind.Prefixed => parent.lengthUnits + case LengthKind.Pattern => LengthUnits.Characters + case _ + if parent.isInstanceOf[ChoiceTermBase] && (parent + .asInstanceOf[ChoiceTermBase] + .choiceLengthKind == ChoiceLengthKind.Explicit) => + LengthUnits.Bytes + case LengthKind.EndOfParent => parent.parentEffectiveLengthUnits + case _ => + Assert.invariantFailed( + s"Could not figure effective length unit of parents of ${context}" + ) + } + } + case None if this.isInstanceOf[Root] => LengthUnits.Characters + case _ => + Assert.invariantFailed( + s"Could not figure effective length unit of parents of ${context}" + ) + } + final lazy val checkEndOfParentElem: Unit = { + if (lengthKind != LengthKind.EndOfParent) () + else { + schemaDefinitionWhen( + hasTerminator, + "%s is specified as dfdl:lengthKind=\"endOfParent\", but specifies a dfdl:terminator.", + context + ) + schemaDefinitionWhen( + trailingSkip != 0, + "%s is specified as dfdl:lengthKind=\"endOfParent\", but specifies a non-zero dfdl:trailingSkip.", + context + ) + schemaDefinitionWhen( + maxOccurs > 1, + "%s is specified as dfdl:lengthKind=\"endOfParent\", but specifies a maxOccurs greater than 1.", + context + ) + schemaDefinitionWhen( + nextSibling.isDefined && nextSibling.get.isInstanceOf[ModelGroup], + "%s is specified as dfdl:lengthKind=\"endOfParent\", but a model group is defined between this element and the end of the enclosing component", + context + ) + schemaDefinitionWhen( + nextSibling.isDefined && nextSibling.get.isRepresented, + "%s is specified as dfdl:lengthKind=\"endOfParent\", but a represented element is defined between this element and the end of the enclosing component", + context + ) + immediatelyEnclosingElementParent match { + case Some(parent: ElementBase) => + parent.lengthKind match { + case LengthKind.Implicit | LengthKind.Delimited => + schemaDefinitionError( + "%s is specified as dfdl:lengthKind=\"endOfParent\", but its parent is an element with dfdl:lengthKind 'implicit' or 'delimited'.", + context + ) + case _ => // do nothing + } + case _ => // do nothing + } + schemaDefinitionWhen( + representation == Representation.Text && knownEncodingWidthInBits != 8 && parentEffectiveLengthUnits != LengthUnits.Characters, + "%s is specified as dfdl:lengthKind=\"endOfParent\", but the element has text representation, does not have a single-byte character set encoding, and the effective length units of the parent is not 'characters'.", + context + ) + + immediatelyEnclosingModelGroup match { + case Some(s: SequenceTermBase) => { + schemaDefinitionWhen( + s.separatorPosition == SeparatorPosition.Postfix, + "%s is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with dfdl:separatorPosition defined as 'postfix'.", + context + ) + schemaDefinitionWhen( + s.sequenceKind != SequenceKind.Ordered, + "%s is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with dfdl:sequenceKind defined as 'unordered'.", + context + ) + schemaDefinitionWhen( + s.hasTerminator, + "%s is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with a dfdl:terminator.", + context + ) + schemaDefinitionWhen( + s.elementChildren.exists(e => e.floating == YesNo.Yes), + "%s is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with elements defining dfdl:floating='yes'.", + context + ) + schemaDefinitionWhen( + s.trailingSkip != 0, + "%s is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with a non-zero dfdl:trailingSkip." + ) + } + case Some(c: ChoiceTermBase) if c.choiceLengthKind == ChoiceLengthKind.Implicit => { + schemaDefinitionWhen( + c.hasTerminator, + "%s is specified as dfdl:lengthKind=\"endOfParent\", but is in a choice with a dfdl:terminator.", + context + ) + schemaDefinitionWhen( + c.trailingSkip != 0, + "%s is specified as dfdl:lengthKind=\"endOfParent\", but is in a choice with a non-zero dfdl:trailingSkip.", + context + ) + } + case _ => // do nothing + } + + if (isSimpleType) { + schemaDefinitionUnless( + (primType eq PrimType.String) + || (representation == Representation.Text) + || (primType eq PrimType.HexBinary) + || (representation == Representation.Binary + && Seq(BinaryNumberRep.Packed, BinaryNumberRep.Bcd, BinaryNumberRep.Ibm4690Packed) + .contains(binaryNumberRep)), + "%s is a simple type specified as dfdl:lengthKind=\"endOfParent\", but isn't a string type, doesn't have text representation, isn't a hexbinary type, or doesn't have binary representation with packed decimal representation.", + context + ) + } + + } + } + /** * Quite tricky when we add padding or fill * @@ -648,10 +781,7 @@ trait ElementBaseGrammarMixin Assert.invariant(pt == PrimType.String) StringOfSpecifiedLength(this) } - case LengthKind.EndOfParent if isComplexType => - notYetImplemented("lengthKind='endOfParent' for complex type") - case LengthKind.EndOfParent => - notYetImplemented("lengthKind='endOfParent' for simple type") + case LengthKind.EndOfParent => StringOfSpecifiedLength(this) } } @@ -667,6 +797,10 @@ trait ElementBaseGrammarMixin new HexBinaryLengthPrefixed(this) } + private lazy val hexBinaryLengthEndOfParent = prod("hexBinaryLengthEndOfParent") { + new HexBinaryLengthEndOfParent(this) + } + private lazy val hexBinaryValue = prod("hexBinaryValue") { schemaDefinitionWhen( lengthUnits == LengthUnits.Characters, @@ -678,10 +812,7 @@ trait ElementBaseGrammarMixin case LengthKind.Delimited => hexBinaryDelimitedEndOfData case LengthKind.Pattern => hexBinaryLengthPattern case LengthKind.Prefixed => hexBinaryLengthPrefixed - case LengthKind.EndOfParent if isComplexType => - notYetImplemented("lengthKind='endOfParent' for complex type") - case LengthKind.EndOfParent => - notYetImplemented("lengthKind='endOfParent' for simple type") + case LengthKind.EndOfParent => hexBinaryLengthEndOfParent } } @@ -1280,10 +1411,7 @@ trait ElementBaseGrammarMixin case LengthKind.Implicit => LiteralValueNilOfSpecifiedLength(this) case LengthKind.Prefixed => LiteralValueNilOfSpecifiedLength(this) - case LengthKind.EndOfParent if isComplexType => - notYetImplemented("lengthKind='endOfParent' for complex type") - case LengthKind.EndOfParent => - notYetImplemented("lengthKind='endOfParent' for simple type") + case LengthKind.EndOfParent => LiteralValueNilOfSpecifiedLength(this) } } case NilKind.LiteralCharacter => { @@ -1396,10 +1524,7 @@ trait ElementBaseGrammarMixin case LengthKind.Implicit if isSimpleType && impliedRepresentation == Representation.Binary => new SpecifiedLengthImplicit(this, body, implicitBinaryLengthInBits) - case LengthKind.EndOfParent if isComplexType => - notYetImplemented("lengthKind='endOfParent' for complex type") - case LengthKind.EndOfParent => - notYetImplemented("lengthKind='endOfParent' for simple type") + case LengthKind.EndOfParent => new SpecifiedLengthEndOfParent(this, body) case LengthKind.Delimited | LengthKind.Implicit => Assert.impossibleCase( "Delimited and ComplexType Implicit cases should not be reached" diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesLengthKind.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesLengthKind.scala index e069353bd5..fcda6161f8 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesLengthKind.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesLengthKind.scala @@ -191,6 +191,16 @@ case class HexBinaryLengthPrefixed(e: ElementBase) extends Terminal(e, true) { new HexBinaryMinLengthInBytesUnparser(e.minLength.longValue, e.elementRuntimeData) } +case class HexBinaryLengthEndOfParent(e: ElementBase) extends Terminal(e, true) { + + override lazy val parser: DaffodilParser = new HexBinaryEndOfBitLimitParser( + e.elementRuntimeData + ) + + override lazy val unparser: DaffodilUnparser = + new HexBinaryMinLengthInBytesUnparser(e.minLength.longValue, e.elementRuntimeData) +} + abstract class PackedIntegerDelimited( e: ElementBase, packedSignCodes: PackedSignCodes diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/SpecifiedLength.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/SpecifiedLength.scala index 0ae3e6a4d9..49baa728c2 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/SpecifiedLength.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/SpecifiedLength.scala @@ -132,6 +132,30 @@ class SpecifiedLengthExplicit(e: ElementBase, eGram: => Gram, bitsMultiplier: In } +class SpecifiedLengthEndOfParent(e: ElementBase, eGram: => Gram) + extends SpecifiedLengthCombinatorBase(e, eGram) { + + lazy val kind = "EndOfParent_" + e.lengthUnits.toString + + lazy val parser: Parser = { + if (eParser.isEmpty) eParser + else + new SpecifiedLengthEndOfParentParser( + eParser, + e.elementRuntimeData + ) + } + + lazy val unparser: Unparser = { + if (eUnparser.isEmpty) eUnparser + else + new SpecifiedLengthEndOfParentUnparser( + eUnparser, + e.elementRuntimeData + ) + } +} + class SpecifiedLengthImplicit(e: ElementBase, eGram: => Gram, nBits: Long) extends SpecifiedLengthCombinatorBase(e, eGram) with SpecifiedLengthExplicitImplicitUnparserMixin { diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/HexBinaryLengthParsers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/HexBinaryLengthParsers.scala index 7c9f50efca..0dc7b21996 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/HexBinaryLengthParsers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/HexBinaryLengthParsers.scala @@ -84,11 +84,8 @@ final class HexBinarySpecifiedLengthParser(erd: ElementRuntimeData, lengthEv: Le } final class HexBinaryEndOfBitLimitParser(erd: ElementRuntimeData) - extends HexBinaryLengthParser(erd) { + extends HexBinaryLengthParser(erd), + BitLengthFromBitLimitMixin { override def runtimeDependencies = Vector() - - override def getLengthInBits(pstate: PState): Long = { - pstate.bitLimit0b.get - pstate.bitPos0b - } } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BinaryNumberTraits.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/ParserTraits.scala similarity index 100% rename from daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BinaryNumberTraits.scala rename to daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/ParserTraits.scala diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala index 2f3ee1f5dd..1d2bb11521 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala @@ -132,6 +132,17 @@ class SpecifiedLengthPatternParser( } } +class SpecifiedLengthEndOfParentParser( + eParser: Parser, + erd: ElementRuntimeData +) extends SpecifiedLengthParserBase(eParser, erd), + BitLengthFromBitLimitMixin { + + override protected def getBitLength(s: PState): MaybeULong = { + MaybeULong(super[BitLengthFromBitLimitMixin].getBitLength(s)) + } +} + class SpecifiedLengthExplicitParser( eParser: Parser, erd: ElementRuntimeData, diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/SpecifiedLengthUnparsers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/SpecifiedLengthUnparsers.scala index 44af8e887a..071c810a04 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/SpecifiedLengthUnparsers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/SpecifiedLengthUnparsers.scala @@ -72,6 +72,20 @@ final class SpecifiedLengthExplicitImplicitUnparser( } } +final class SpecifiedLengthEndOfParentUnparser( + eUnparser: Unparser, + erd: ElementRuntimeData +) extends CombinatorUnparser(erd) { + + override def runtimeDependencies = Vector() + + override def childProcessors = Vector(eUnparser) + + override final def unparse(state: UState): Unit = { + eUnparser.unparse1(state) + } +} + /** * This trait is to be used with prefixed length unparsers where the length * must be calculated based on the content length of the data. This means the diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section12/aligned_data/Aligned_Data.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section12/aligned_data/Aligned_Data.tdml index c2be97ef3f..e97a545c6b 100644 --- a/daffodil-test/src/test/resources/org/apache/daffodil/section12/aligned_data/Aligned_Data.tdml +++ b/daffodil-test/src/test/resources/org/apache/daffodil/section12/aligned_data/Aligned_Data.tdml @@ -639,6 +639,28 @@ + + + + + + + + + + + + + + + @@ -664,6 +686,30 @@ + + + + + + + + + + + + + + + + @@ -4357,6 +4403,21 @@ + + IaaBbb + + + + + + aa + bb + + + + + + IAaabbee @@ -4373,6 +4434,22 @@ + + IAaabbee + + + + + + aa + bb + + ee + + + + + aaTBbb @@ -4613,4 +4690,6 @@ + + diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/EndOfParentTests.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/EndOfParentTests.tdml index fdbcc2e32d..29aa3214f0 100644 --- a/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/EndOfParentTests.tdml +++ b/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/EndOfParentTests.tdml @@ -20,9 +20,9 @@ description="Section 12 - lengthKind=endOfParent" xmlns:tdml="http://www.ibm.com/xmlns/dfdl/testData" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:dfdl="http://www.ogf.org/dfdl/dfdl-1.0/" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:fn="http://www.w3.org/2005/xpath-functions" - xmlns:ex="http://example.com" defaultRoundTrip="true"> + xmlns:ex="http://example.com" defaultRoundTrip="onePass"> - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Schema Definition Error + endOfParent + but its parent + lengthKind + delimited + + + + + + + Schema Definition Error + endOfParent + but its parent + lengthKind + delimited + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Schema Definition Error + endOfParent + but its parent + lengthKind + implicit + + + + + + + Schema Definition Error + endOfParent + but its parent + lengthKind + implicit + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + 1 + + + + 2 + 2 + + + 3 + 3 + + + + A + + + + + + + + + + + + X + YZ + + A + + + + + + + + + - + + + + + + + + + + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + 1 + + + + 2 + 2 + + + 3 + 3 + + + + A + + + + + + + + + + + + X + YZ + + A + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + 1 + + + + 2 + 2 + + + 3 + 3 + + + + A + + + + + + + + + + + + X + YZ + + A + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + 1 + + + + 2 + 2 + + + 3 + 3 + + + + + + + + + + + + + + + X + YZ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + endOfParent + represented element + between this element and + end of the enclosing component + + + + + + + endOfParent + represented element + between this element and + end of the enclosing component + + + + - + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - Schema Definition Error - not - implemented endOfParent - complex - type + model group + between this element and + end of the enclosing component - - + + - Schema Definition Error - not - implemented endOfParent - simple - type + model group + between this element and + end of the enclosing component + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + 1 + + + + 2 + 2 + + + 3 + 3 + + + A + + + + + + + + + + + + + X + YZ + A + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + endOfParent + specifies + terminator + + + + + + + endOfParent + specifies + terminator + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + endOfParent + specifies + non-zero + trailingSkip + + + + + + + endOfParent + specifies + non-zero + trailingSkip + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + endOfParent + specifies + maxOccurs + greater than 1 + + + + + + + endOfParent + specifies + maxOccurs + greater than 1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + endOfParent + does not have + single-byte character set encoding + + + + + + + endOfParent + does not have + single-byte character set encoding + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + endOfParent + in a sequence with + separatorPosition + postfix + + + + + + + endOfParent + in a sequence with + separatorPosition + postfix + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + endOfParent + in a sequence with + sequenceKind + unordered + + + + + + + endOfParent + in a sequence with + sequenceKind + unordered + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + endOfParent + in a sequence with + elements + floating='yes + + + + + + + endOfParent + in a sequence with + elements + floating='yes + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + endOfParent + in a sequence with + non-zero + trailingSkip + + + + + + + endOfParent + in a sequence with + non-zero + trailingSkip + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + endOfParent + in a choice with + non-zero + trailingSkip + + + + + + + endOfParent + in a choice with + non-zero + trailingSkip + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + endOfParent + in a choice with + terminator + + + + + + + endOfParent + in a choice with + terminator + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + simple type + endOfParent + isn't a string type + + + + + + + simple type + endOfParent + doesn't have binary representation with packed decimal representation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + X + 3132 + + + + + + + + + + + + + X + 12 + + + + + + + diff --git a/daffodil-test/src/test/scala/org/apache/daffodil/section12/aligned_data/TestAlignedData.scala b/daffodil-test/src/test/scala/org/apache/daffodil/section12/aligned_data/TestAlignedData.scala index 3c4de0add4..ad587d2a09 100644 --- a/daffodil-test/src/test/scala/org/apache/daffodil/section12/aligned_data/TestAlignedData.scala +++ b/daffodil-test/src/test/scala/org/apache/daffodil/section12/aligned_data/TestAlignedData.scala @@ -209,6 +209,10 @@ class TestAlignedData extends TdmlTests { @Test def prior_siblings_3 = test @Test def prior_siblings_4 = test @Test def prior_siblings_5 = test + + // DAFFODIL-238 + @Test def test_init_alignment_1_eop = test + @Test def test_init_alignment_2_eop = test } class TestBinaryInput extends TdmlTests { diff --git a/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindEndOfParent.scala b/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindEndOfParent.scala new file mode 100644 index 0000000000..53f25c3720 --- /dev/null +++ b/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindEndOfParent.scala @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.daffodil.section12.lengthKind + +import org.apache.daffodil.junit.tdml.TdmlSuite +import org.apache.daffodil.junit.tdml.TdmlTests + +import org.junit.Test + +object TestLengthKindEndOfParent extends TdmlSuite { + val tdmlResource = "/org/apache/daffodil/section12/lengthKind/EndOfParentTests.tdml" +} + +class TestLengthKindEndOfParent extends TdmlTests { + val tdmlSuite = TestLengthKindEndOfParent + + @Test def TestEndOfParentComplexTypesDelimited = test + @Test def TestEndOfParentSimpleTypesDelimited = test + @Test def TestEndOfParentComplexTypesImplicit = test + @Test def TestEndOfParentSimpleTypesImplicit = test + @Test def TestEndOfParentComplexTypesExplicit = test + @Test def TestEndOfParentSimpleTypesExplicit = test + @Test def TestEndOfParentComplexTypesPrefixed = test + @Test def TestEndOfParentSimpleTypesPrefixed = test + @Test def TestEndOfParentComplexTypesPattern = test + @Test def TestEndOfParentSimpleTypesPattern = test + @Test def TestEndOfParentComplexTypesEOP = test + @Test def TestEndOfParentSimpleTypesEOP = test + @Test def TestEndOfParentComplexTypes1 = test + @Test def TestEndOfParentSimpleTypes1 = test + @Test def TestEndOfParentComplexTypes2 = test + @Test def TestEndOfParentSimpleTypes2 = test + @Test def TestEndOfParentComplexTypes3 = test + @Test def TestEndOfParentSimpleTypes3 = test + @Test def TestEndOfParentComplexTypes4 = test + @Test def TestEndOfParentSimpleTypes4 = test + @Test def TestEndOfParentComplexTypes5 = test + @Test def TestEndOfParentSimpleTypes5 = test + @Test def TestEndOfParentComplexTypes6 = test + @Test def TestEndOfParentSimpleTypes6 = test + @Test def TestEndOfParentComplexTypes7 = test + @Test def TestEndOfParentSimpleTypes7 = test + @Test def TestEndOfParentComplexTypes8 = test + @Test def TestEndOfParentSimpleTypes8 = test + @Test def TestEndOfParentComplexTypes9 = test + @Test def TestEndOfParentSimpleTypes9 = test + @Test def TestEndOfParentComplexTypes10 = test + @Test def TestEndOfParentSimpleTypes10 = test + @Test def TestEndOfParentComplexTypes11 = test + @Test def TestEndOfParentSimpleTypes11 = test + @Test def TestEndOfParentComplexTypes12 = test + @Test def TestEndOfParentSimpleTypes12 = test + @Test def TestEndOfParentComplexTypes13 = test + @Test def TestEndOfParentSimpleTypes13 = test + @Test def TestEndOfParentSimpleTypes14 = test + @Test def TestEndOfParentSimpleTypes15 = test + @Test def TestEndOfParentSimpleTypes16 = test + @Test def TestEndOfParentSimpleTypes17 = test +} diff --git a/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindEndOfParent2.scala b/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindEndOfParent2.scala deleted file mode 100644 index 5928ed5f71..0000000000 --- a/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindEndOfParent2.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.daffodil.section12.lengthKind - -import org.apache.daffodil.junit.tdml.TdmlSuite -import org.apache.daffodil.junit.tdml.TdmlTests - -import org.junit.Test - -object TestLengthKindEndOfParent2 extends TdmlSuite { - val tdmlResource = "/org/apache/daffodil/section12/lengthKind/EndOfParentTests.tdml" -} - -class TestLengthKindEndOfParent2 extends TdmlTests { - val tdmlSuite = TestLengthKindEndOfParent2 - - @Test def TestEndOfParentNYIComplexTypes = test - @Test def TestEndOfParentNYISimpleTypes = test -} From df8100e79bf54c360a6674dddaffb597bd37956c Mon Sep 17 00:00:00 2001 From: olabusayoT <50379531+olabusayoT@users.noreply.github.com> Date: Thu, 7 May 2026 09:58:59 -0400 Subject: [PATCH 2/9] fixup! - Add checkEndOfParentRestriction to choice/sequence/elements - employ top-down approach looking from parents to their children instead of lexical child to parent approach - address checks for root elements - add support for EOP for packed decimals - add support for bytesTillEndOfDataStream for BBIS/BIS - update tests --- .../daffodil/core/dsom/ChoiceGroup.scala | 32 +- .../core/dsom/LocalElementMixin.scala | 2 +- .../apache/daffodil/core/dsom/SchemaSet.scala | 2 + .../daffodil/core/dsom/SequenceGroup.scala | 34 + .../org/apache/daffodil/core/dsom/Term.scala | 38 +- .../daffodil/core/grammar/AlignedMixin.scala | 8 +- .../grammar/ElementBaseGrammarMixin.scala | 288 +++-- .../grammar/primitives/PrimitivesBCD.scala | 30 +- .../primitives/PrimitivesBinaryBoolean.scala | 18 + .../primitives/PrimitivesIBM4690Packed.scala | 27 +- .../primitives/PrimitivesLengthKind.scala | 9 +- .../grammar/primitives/PrimitivesPacked.scala | 42 +- .../grammar/primitives/SpecifiedLength.scala | 10 +- .../org/apache/daffodil/io/InputSource.scala | 29 + .../io/InputSourceDataInputStream.scala | 2 + .../runtime1/processors/BCDParsers.scala | 9 +- .../IBM4690PackedDecimalParsers.scala | 9 +- .../processors/PackedDecimalParsers.scala | 10 +- .../parsers/BinaryBooleanParsers.scala | 5 +- .../parsers/HexBinaryLengthParsers.scala | 4 +- .../processors/parsers/ParserTraits.scala | 13 +- .../parsers/SpecifiedLengthParsers.scala | 2 +- .../runtime1/SpecifiedLengthUnparsers.scala | 14 - .../lengthKind/EndOfParentTests.tdml | 1035 ++++++++++++++++- .../daffodil/section13/packed/packed.tdml | 245 ++++ .../choice_groups/ChoiceLengthExplicit.tdml | 76 ++ .../TestLengthKindEndOfParent.scala | 47 +- .../section13/packed/TestPacked.scala | 14 + .../section15/choice_groups/TestChoice.scala | 2 + 29 files changed, 1840 insertions(+), 216 deletions(-) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/ChoiceGroup.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/ChoiceGroup.scala index 927bb38987..8af289b2b9 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/ChoiceGroup.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/ChoiceGroup.scala @@ -24,7 +24,9 @@ import org.apache.daffodil.core.dsom.walker.ChoiceView import org.apache.daffodil.core.grammar.ChoiceGrammarMixin import org.apache.daffodil.lib.schema.annotation.props.Found import org.apache.daffodil.lib.schema.annotation.props.gen.ChoiceAGMixin +import org.apache.daffodil.lib.schema.annotation.props.gen.ChoiceLengthKind import org.apache.daffodil.lib.schema.annotation.props.gen.Choice_AnnotationMixin +import org.apache.daffodil.lib.schema.annotation.props.gen.LengthUnits import org.apache.daffodil.lib.schema.annotation.props.gen.YesNo /** @@ -103,6 +105,7 @@ abstract class ChoiceTermBase( requiredEvaluationsIfActivated(noBranchesFound) requiredEvaluationsIfActivated(branchesAreNonOptional) requiredEvaluationsIfActivated(branchesAreNotIVCElements) + requiredEvaluationsIfActivated(checkEndOfParentRestrictions()) final protected lazy val optionChoiceDispatchKeyRaw = findPropertyOption("choiceDispatchKey", expressionAllowed = true) @@ -159,7 +162,7 @@ abstract class ChoiceTermBase( * Open issues: * 1) Is alignment or leading/trailing skip to be considered syntax. Alignment might not be there. * 2) What about an empty sequence that only carries statement annotations such as dfdl:assert or - * dfdl:setVariable + * dfdl:setVariable * * This latter need to be allowed, because while they do not have known required syntax they do * have to be executed for side-effect. @@ -189,6 +192,33 @@ abstract class ChoiceTermBase( } assuming(branchesOk.forall { x => x }) }.value + + lazy val optMyEffectiveLengthUnits: Option[LengthUnits] = + this match { + case self: ChoiceTermBase if self.choiceLengthKind == ChoiceLengthKind.Explicit => + Some(LengthUnits.Bytes) + case _ => None + } + + def checkEndOfParentRestrictions() = { + val parent = this + val eopChildren = this.childrenEndOfParent + if (eopChildren.isEmpty) {} else { + parent match { + case c: ChoiceTermBase if c.choiceLengthKind == ChoiceLengthKind.Implicit => { + schemaDefinitionWhen( + c.hasTerminator, + "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a choice with a dfdl:terminator." + ) + schemaDefinitionWhen( + c.trailingSkip != 0, + "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a choice with a non-zero dfdl:trailingSkip." + ) + } + case _ => // do nothing + } + } + } } object Choice { diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/LocalElementMixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/LocalElementMixin.scala index 4756afcc0f..d00cbc2b3f 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/LocalElementMixin.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/LocalElementMixin.scala @@ -61,7 +61,7 @@ trait LocalElementMixin extends ParticleMixin with LocalElementGrammarMixin { else if (representation =:= Representation.Binary) true else false } - // per DFDL Spec 9.3.2, endOfParent is already positioned at parent's end so length is zero + // we can rarely statically know if an endOfParent element must have non-zero length case LengthKind.EndOfParent => false } res diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/SchemaSet.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/SchemaSet.scala index 18a2483fac..ca76780db9 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/SchemaSet.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/SchemaSet.scala @@ -30,6 +30,7 @@ import org.apache.daffodil.lib.iapi.Diagnostic import org.apache.daffodil.lib.iapi.UnitTestSchemaSource import org.apache.daffodil.lib.oolag.OOLAG import org.apache.daffodil.lib.schema.annotation.props.LookupLocation +import org.apache.daffodil.lib.schema.annotation.props.gen.LengthUnits import org.apache.daffodil.lib.util.TransitiveClosure import org.apache.daffodil.lib.xml.* @@ -116,6 +117,7 @@ final class SchemaSet private ( requiredEvaluationsAlways(root) requiredEvaluationsAlways(checkForDuplicateTopLevels()) + requiredEvaluationsAlways(root.checkEndOfParentRestrictions(LengthUnits.Characters)) lazy val resolver = DFDLCatalogResolver.get diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/SequenceGroup.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/SequenceGroup.scala index a6a9bc0037..ef510f2a2f 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/SequenceGroup.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/SequenceGroup.scala @@ -34,6 +34,7 @@ import org.apache.daffodil.lib.schema.annotation.props.gen.OccursCountKind import org.apache.daffodil.lib.schema.annotation.props.gen.SeparatorPosition import org.apache.daffodil.lib.schema.annotation.props.gen.SequenceKind import org.apache.daffodil.lib.schema.annotation.props.gen.TestKind +import org.apache.daffodil.lib.schema.annotation.props.gen.YesNo import org.apache.daffodil.lib.xml.RefQName import org.apache.daffodil.lib.xml.XMLUtils import org.apache.daffodil.runtime1.layers.LayerRuntimeData @@ -105,6 +106,7 @@ abstract class SequenceGroupTermBase(xml: Node, lexicalParent: SchemaComponent, requiredEvaluationsIfActivated(checkValidityOccursCountKind) requiredEvaluationsIfActivated(checkIfNonEmptyAndDiscrimsOrAsserts) requiredEvaluationsIfActivated(checkIfMultipleChildrenWithSameName) + requiredEvaluationsIfActivated(checkEndOfParentRestrictions()) protected def apparentXMLChildren: Seq[Node] @@ -270,6 +272,38 @@ abstract class SequenceGroupTermBase(xml: Node, lexicalParent: SchemaComponent, case SequenceKind.Unordered => false } + def checkEndOfParentRestrictions() = { + val parent = this + val eopChildren = this.childrenEndOfParent + if (eopChildren.isEmpty) {} else { + parent match { + case s: SequenceTermBase => { + schemaDefinitionWhen( + s.separatorPosition == SeparatorPosition.Postfix, + "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with dfdl:separatorPosition defined as 'postfix'." + ) + schemaDefinitionWhen( + s.sequenceKind != SequenceKind.Ordered, + "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with dfdl:sequenceKind defined as 'unordered'." + ) + schemaDefinitionWhen( + s.hasTerminator, + "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with a dfdl:terminator." + ) + schemaDefinitionWhen( + s.realElementChildren.exists(e => e.floating == YesNo.Yes), + "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with elements defining dfdl:floating='yes'." + ) + schemaDefinitionWhen( + s.trailingSkip != 0, + "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with a non-zero dfdl:trailingSkip." + ) + } + case null => // do nothing + } + } + } + } /** diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/Term.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/Term.scala index 4e6495b3d1..d8db9d33a9 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/Term.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/Term.scala @@ -27,9 +27,7 @@ import org.apache.daffodil.lib.iapi.WarnID import org.apache.daffodil.lib.schema.annotation.props.Found import org.apache.daffodil.lib.schema.annotation.props.NotFound import org.apache.daffodil.lib.schema.annotation.props.SeparatorSuppressionPolicy -import org.apache.daffodil.lib.schema.annotation.props.gen.LengthKind -import org.apache.daffodil.lib.schema.annotation.props.gen.OccursCountKind -import org.apache.daffodil.lib.schema.annotation.props.gen.YesNo +import org.apache.daffodil.lib.schema.annotation.props.gen.* /** * Mixin for objects that are shared, but have consistency checks to be run @@ -96,6 +94,17 @@ trait Term requiredEvaluationsIfActivated(defaultPropertySources) requiredEvaluationsIfActivated(termChecks) + final lazy val childrenEndOfParent: Seq[ElementBase] = LV(Symbol("childrenEndOfParent")) { + val gms = termChildren + val chls = gms.flatMap { + case eb: ElementBase if eb.lengthKind == LengthKind.EndOfParent => Seq(eb) + case eb: ElementBase => Nil + case c: Choice => Nil + case mg: ModelGroup => mg.childrenEndOfParent + } + chls + }.value + private lazy val termChecks = { statements.foreach { _.checkTerm(this) } } @@ -269,12 +278,12 @@ trait Term .getOrElse(false) } - final lazy val immediatelyEnclosingElementParent: Option[ElementBase] = { + final lazy val immediatelyEnclosingParentForEOPElem: Option[Term] = { val p = optLexicalParent.flatMap { case e: ElementBase => Some(e) case ge: GlobalElementDecl => Some(ge.asRoot) - case s: SequenceTermBase => s.immediatelyEnclosingElementParent - case c: ChoiceTermBase => c.immediatelyEnclosingElementParent + case s: SequenceTermBase => s.immediatelyEnclosingParentForEOPElem + case c: ChoiceTermBase => Some(c) case ct: ComplexTypeBase => { ct.optLexicalParent.flatMap { case e: ElementBase => Some(e) @@ -582,4 +591,21 @@ trait Term } } + final protected lazy val realElementChildren: Seq[ElementBase] = { + termChildren.flatMap { + case eb: ElementBase => Seq(eb) + case c: Choice => Nil + case mg: ModelGroup => mg.realElementChildren + } + } + + lazy val flattenedChildren: IndexedSeq[Term] = termChildren.flatMap { c => + c match { + case eb: ElementBase => IndexedSeq(eb) + case mg: ModelGroup if mg.groupMembers.isEmpty => IndexedSeq(mg) + case s: SequenceTermBase => s.flattenedChildren + case c: ChoiceTermBase => c.flattenedChildren + case _ => Nil + } + }.toIndexedSeq } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/AlignedMixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/AlignedMixin.scala index 63c7936eb3..9b0942e9b1 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/AlignedMixin.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/AlignedMixin.scala @@ -457,10 +457,10 @@ trait AlignedMixin extends GrammarMixin { self: Term => case LengthKind.Delimited => encodingLengthApprox case LengthKind.Pattern => encodingLengthApprox case LengthKind.EndOfParent => - eb.immediatelyEnclosingElementParent match { - case Some(parent) => parent.elementSpecifiedLengthApprox - case _ => LengthMultipleOf(1) - } + // technically the alignment of an EndOfParent element would be the + // alignment of its parent minus our current alignment (i.e alignment of + // prior sibs) but since nothing can follow + LengthMultipleOf(1) // If an element is lengthKind="prefixed", the element's length is the length // of the value of the prefix element, which can't be known till runtime case LengthKind.Prefixed => LengthMultipleOf(1) // NYI (see DAFFODIL-3066) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala index 011f323b23..16b09e2873 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala @@ -19,14 +19,12 @@ package org.apache.daffodil.core.grammar import java.lang.Long as JLong -import org.apache.daffodil.core.dsom.ChoiceTermBase import org.apache.daffodil.core.dsom.ElementBase import org.apache.daffodil.core.dsom.ExpressionCompilers import org.apache.daffodil.core.dsom.InitiatedTerminatedMixin import org.apache.daffodil.core.dsom.ModelGroup import org.apache.daffodil.core.dsom.PrefixLengthQuasiElementDecl import org.apache.daffodil.core.dsom.Root -import org.apache.daffodil.core.dsom.SequenceTermBase import org.apache.daffodil.core.grammar.primitives.* import org.apache.daffodil.core.runtime1.ElementBaseRuntime1Mixin import org.apache.daffodil.lib.exceptions.Assert @@ -56,7 +54,6 @@ trait ElementBaseGrammarMixin requiredEvaluationsIfActivated(checkPrefixedLengthElementDecl) requiredEvaluationsIfActivated(checkDelimitedLengthEVDP) - requiredEvaluationsIfActivated(checkEndOfParentElem) private val context = this @@ -257,131 +254,124 @@ trait ElementBaseGrammarMixin } final lazy val prefixedLengthBody = prefixedLengthElementDecl.parsedValue - final lazy val parentEffectiveLengthUnits: LengthUnits = - immediatelyEnclosingElementParent match { - case Some(parent: ElementBase) => { - parent.lengthKind match { - case LengthKind.Explicit | LengthKind.Prefixed => parent.lengthUnits - case LengthKind.Pattern => LengthUnits.Characters - case _ - if parent.isInstanceOf[ChoiceTermBase] && (parent - .asInstanceOf[ChoiceTermBase] - .choiceLengthKind == ChoiceLengthKind.Explicit) => - LengthUnits.Bytes - case LengthKind.EndOfParent => parent.parentEffectiveLengthUnits - case _ => - Assert.invariantFailed( - s"Could not figure effective length unit of parents of ${context}" - ) - } - } - case None if this.isInstanceOf[Root] => LengthUnits.Characters + def myEffectiveLengthUnits(lastNonEOPLU: LengthUnits): LengthUnits = { + lengthKind match { + case LengthKind.EndOfParent => lastNonEOPLU + case LengthKind.Explicit | LengthKind.Prefixed => lengthUnits + case LengthKind.Pattern => LengthUnits.Characters case _ => Assert.invariantFailed( - s"Could not figure effective length unit of parents of ${context}" + "Delimited and Implicit are illegal for the parent of EndOfParent element" ) } - final lazy val checkEndOfParentElem: Unit = { - if (lengthKind != LengthKind.EndOfParent) () - else { - schemaDefinitionWhen( - hasTerminator, - "%s is specified as dfdl:lengthKind=\"endOfParent\", but specifies a dfdl:terminator.", - context - ) - schemaDefinitionWhen( - trailingSkip != 0, - "%s is specified as dfdl:lengthKind=\"endOfParent\", but specifies a non-zero dfdl:trailingSkip.", - context + } + + def checkChildrenForSiblingsAfterEOPElement( + parent: ElementBase, + specificChild: ElementBase + ) = { + lazy val foundPosition = flattenedChildren.indexOf(specificChild) + lazy val lastIndexOfChildren = flattenedChildren.length - 1 + if (flattenedChildren.isEmpty || foundPosition < 0) { + // not found amongst children + Assert.impossible("EndOfParent element not found amongst term children of parent") + } else if (foundPosition != lastIndexOfChildren) { + // get the following children after the EOP element+ 1 + val followingChildrenAfter = + flattenedChildren.slice(foundPosition + 1, lastIndexOfChildren + 1) + followingChildrenAfter.foreach { + case m: ModelGroup => { + specificChild.SDE( + "element is specified as dfdl:lengthKind=\"endOfParent\", but a model group is defined between this element and the end of the enclosing component" + ) + } + case r if r.isRepresented => { + specificChild.SDE( + "element is specified as dfdl:lengthKind=\"endOfParent\", but a represented element is defined between this element and the end of the enclosing component" + ) + } + case _ => // do nothing + } + } + } + + def checkEndOfParentRestrictions(lastNonEOPLU: LengthUnits): Unit = { + val parent = this + val eopChildren = this.childrenEndOfParent + // checks + this match { + case rootElem: Root if lengthKind == LengthKind.EndOfParent => { + rootElem.checkEndOfParentRestrictionsOnCurrentElement(lastNonEOPLU) + } + case _ => // do nothing + } + + if (eopChildren.isEmpty) + ( + this.elementChildren.foreach(_.checkEndOfParentRestrictions(lastNonEOPLU)) ) + else { + eopChildren.foreach { eopChild => + lazy val parentELU = myEffectiveLengthUnits(lastNonEOPLU) + checkChildrenForSiblingsAfterEOPElement(parent, eopChild) + parent.lengthKind match { + case LengthKind.Implicit | LengthKind.Delimited => + schemaDefinitionError( + "element is specified as dfdl:lengthKind=\"endOfParent\", but its parent is an element with dfdl:lengthKind 'implicit' or 'delimited'." + ) + case _ => // do nothing + } + eopChild.checkEndOfParentRestrictionsOnCurrentElement(parentELU) + if (parent.lengthKind != LengthKind.EndOfParent) { + eopChild.checkEndOfParentRestrictions(parentELU) + } else { + eopChild.checkEndOfParentRestrictions(lastNonEOPLU) + } + } + } + // end checks + } + + def checkEndOfParentRestrictionsOnCurrentElement(parentELU: LengthUnits): Unit = { + val currentElement: ElementBase = this + if (currentElement.lengthKind != LengthKind.EndOfParent) {} else { schemaDefinitionWhen( - maxOccurs > 1, - "%s is specified as dfdl:lengthKind=\"endOfParent\", but specifies a maxOccurs greater than 1.", - context + currentElement.hasTerminator, + "element is specified as dfdl:lengthKind=\"endOfParent\", but specifies a dfdl:terminator." ) schemaDefinitionWhen( - nextSibling.isDefined && nextSibling.get.isInstanceOf[ModelGroup], - "%s is specified as dfdl:lengthKind=\"endOfParent\", but a model group is defined between this element and the end of the enclosing component", - context + currentElement.trailingSkip != 0, + "element is specified as dfdl:lengthKind=\"endOfParent\", but specifies a non-zero dfdl:trailingSkip." ) schemaDefinitionWhen( - nextSibling.isDefined && nextSibling.get.isRepresented, - "%s is specified as dfdl:lengthKind=\"endOfParent\", but a represented element is defined between this element and the end of the enclosing component", - context + currentElement.maxOccurs > 1, + "element is specified as dfdl:lengthKind=\"endOfParent\", but specifies a maxOccurs greater than 1." ) - immediatelyEnclosingElementParent match { - case Some(parent: ElementBase) => - parent.lengthKind match { - case LengthKind.Implicit | LengthKind.Delimited => - schemaDefinitionError( - "%s is specified as dfdl:lengthKind=\"endOfParent\", but its parent is an element with dfdl:lengthKind 'implicit' or 'delimited'.", - context - ) - case _ => // do nothing - } - case _ => // do nothing - } schemaDefinitionWhen( - representation == Representation.Text && knownEncodingWidthInBits != 8 && parentEffectiveLengthUnits != LengthUnits.Characters, - "%s is specified as dfdl:lengthKind=\"endOfParent\", but the element has text representation, does not have a single-byte character set encoding, and the effective length units of the parent is not 'characters'.", - context + currentElement.impliedRepresentation == Representation.Text + && (!currentElement.isKnownEncoding || !currentElement.knownEncodingIsFixedWidth || currentElement.knownEncodingWidthInBits != 8) + && (parentELU != LengthUnits.Characters), + "element is specified as dfdl:lengthKind=\"endOfParent\", but the element has text representation, and does not have a single-byte character set encoding, and the effective length units of the parent is not 'characters'." ) - - immediatelyEnclosingModelGroup match { - case Some(s: SequenceTermBase) => { - schemaDefinitionWhen( - s.separatorPosition == SeparatorPosition.Postfix, - "%s is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with dfdl:separatorPosition defined as 'postfix'.", - context - ) - schemaDefinitionWhen( - s.sequenceKind != SequenceKind.Ordered, - "%s is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with dfdl:sequenceKind defined as 'unordered'.", - context - ) - schemaDefinitionWhen( - s.hasTerminator, - "%s is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with a dfdl:terminator.", - context - ) - schemaDefinitionWhen( - s.elementChildren.exists(e => e.floating == YesNo.Yes), - "%s is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with elements defining dfdl:floating='yes'.", - context - ) - schemaDefinitionWhen( - s.trailingSkip != 0, - "%s is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with a non-zero dfdl:trailingSkip." - ) - } - case Some(c: ChoiceTermBase) if c.choiceLengthKind == ChoiceLengthKind.Implicit => { - schemaDefinitionWhen( - c.hasTerminator, - "%s is specified as dfdl:lengthKind=\"endOfParent\", but is in a choice with a dfdl:terminator.", - context - ) - schemaDefinitionWhen( - c.trailingSkip != 0, - "%s is specified as dfdl:lengthKind=\"endOfParent\", but is in a choice with a non-zero dfdl:trailingSkip.", - context - ) - } - case _ => // do nothing - } - - if (isSimpleType) { + if (currentElement.isSimpleType) { schemaDefinitionUnless( - (primType eq PrimType.String) - || (representation == Representation.Text) - || (primType eq PrimType.HexBinary) - || (representation == Representation.Binary + (currentElement.primType eq PrimType.String) + || (currentElement.representation == Representation.Text) + || (currentElement.primType eq PrimType.HexBinary) + || (currentElement.representation == Representation.Binary && Seq(BinaryNumberRep.Packed, BinaryNumberRep.Bcd, BinaryNumberRep.Ibm4690Packed) - .contains(binaryNumberRep)), - "%s is a simple type specified as dfdl:lengthKind=\"endOfParent\", but isn't a string type, doesn't have text representation, isn't a hexbinary type, or doesn't have binary representation with packed decimal representation.", - context + .contains(currentElement.binaryNumberRep)) + || (currentElement.representation == Representation.Binary + && optionBinaryCalendarRep.isDefined + && Seq( + BinaryCalendarRep.Packed, + BinaryCalendarRep.Bcd, + BinaryCalendarRep.Ibm4690Packed + ) + .contains(currentElement.binaryCalendarRep)), + "element is a simple type specified as dfdl:lengthKind=\"endOfParent\", but isn't a string type, doesn't have text representation, isn't a hexbinary type, or doesn't have binary representation with packed decimal representation." ) } - } } @@ -738,7 +728,19 @@ trait ElementBaseGrammarMixin case LengthKind.Pattern => schemaDefinitionError("Binary data elements cannot have lengthKind='pattern'.") case LengthKind.EndOfParent => - schemaDefinitionError("Binary data elements cannot have lengthKind='endOfParent'.") + // only for packed binary data, length must be computed at runtime. + if ( + representation == Representation.Binary + && optionBinaryCalendarRep.isDefined + && Seq(BinaryCalendarRep.Packed, BinaryCalendarRep.Bcd, BinaryCalendarRep.Ibm4690Packed) + .contains(binaryCalendarRep) + || representation == Representation.Binary + && optionBinaryNumberRep.isDefined + && Seq(BinaryNumberRep.Packed, BinaryNumberRep.Bcd, BinaryNumberRep.Ibm4690Packed) + .contains(binaryNumberRep) + ) -1 + else + SDE("lengthKind='endOfParent' only supported for packed binary formats.") } private def explicitBinaryLengthInBits() = { @@ -924,6 +926,10 @@ trait ElementBaseGrammarMixin prod("bcdPrefixedLengthCalendar", binaryCalendarRep == BinaryCalendarRep.Bcd) { ConvertZonedCombinator(this, new BCDIntegerPrefixedLength(this), textConverter) } + private lazy val bcdEndOfParentLengthCalendar = + prod("bcdEndOfParentLengthCalendar", binaryCalendarRep == BinaryCalendarRep.Bcd) { + ConvertZonedCombinator(this, new BCDIntegerEndOfParentLength(this), textConverter) + } private lazy val ibm4690PackedKnownLengthCalendar = prod( "ibm4690PackedKnownLengthCalendar", @@ -965,6 +971,16 @@ trait ElementBaseGrammarMixin textConverter ) } + private lazy val ibm4690PackedEndOfParentLengthCalendar = prod( + "ibm4690PackedEndOfParentLengthCalendar", + binaryCalendarRep == BinaryCalendarRep.Ibm4690Packed + ) { + ConvertZonedCombinator( + this, + new IBM4690PackedIntegerEndOfParentLength(this), + textConverter + ) + } private lazy val packedKnownLengthCalendar = prod("packedKnownLengthCalendar", binaryCalendarRep == BinaryCalendarRep.Packed) { @@ -1002,6 +1018,14 @@ trait ElementBaseGrammarMixin textConverter ) } + private lazy val packedEndOfParentLengthCalendar = + prod("packedPrefixedLengthCalendar", binaryCalendarRep == BinaryCalendarRep.Packed) { + ConvertZonedCombinator( + this, + new PackedIntegerEndOfParentLength(this, packedSignCodes), + textConverter + ) + } def primType: PrimType @@ -1053,6 +1077,8 @@ trait ElementBaseGrammarMixin byteOrderRaw // must be defined or SDE } (binaryNumberRep, lengthKind, binaryNumberKnownLengthInBits) match { + case (BinaryNumberRep.Binary, LengthKind.EndOfParent, _) => + SDE("lengthKind='endOfParent' is not allowed with binary number representation") case (BinaryNumberRep.Binary, LengthKind.Prefixed, _) => new BinaryIntegerPrefixedLength(this) case (BinaryNumberRep.Binary, _, -1) => new BinaryIntegerRuntimeLength(this) @@ -1070,6 +1096,8 @@ trait ElementBaseGrammarMixin new PackedIntegerDelimitedEndOfData(this, packedSignCodes) case (BinaryNumberRep.Packed, LengthKind.Prefixed, -1) => new PackedIntegerPrefixedLength(this, packedSignCodes) + case (BinaryNumberRep.Packed, LengthKind.EndOfParent, -1) => + new PackedIntegerEndOfParentLength(this, packedSignCodes) case (BinaryNumberRep.Packed, _, -1) => new PackedIntegerRuntimeLength(this, packedSignCodes) case (BinaryNumberRep.Packed, _, _) => @@ -1082,6 +1110,8 @@ trait ElementBaseGrammarMixin new IBM4690PackedIntegerDelimitedEndOfData(this) case (BinaryNumberRep.Ibm4690Packed, LengthKind.Prefixed, -1) => new IBM4690PackedIntegerPrefixedLength(this) + case (BinaryNumberRep.Ibm4690Packed, LengthKind.EndOfParent, -1) => + new IBM4690PackedIntegerEndOfParentLength(this) case (BinaryNumberRep.Ibm4690Packed, _, -1) => new IBM4690PackedIntegerRuntimeLength(this) case (BinaryNumberRep.Ibm4690Packed, _, _) => @@ -1094,6 +1124,7 @@ trait ElementBaseGrammarMixin (lengthKind, binaryNumberKnownLengthInBits) match { case (LengthKind.Delimited, -1) => new BCDIntegerDelimitedEndOfData(this) case (LengthKind.Prefixed, -1) => new BCDIntegerPrefixedLength(this) + case (LengthKind.EndOfParent, -1) => new BCDIntegerEndOfParentLength(this) case (_, -1) => new BCDIntegerRuntimeLength(this) case (_, _) => new BCDIntegerKnownLength(this, binaryNumberKnownLengthInBits) } @@ -1161,6 +1192,8 @@ trait ElementBaseGrammarMixin ) byteOrderRaw // must have or SDE (binaryNumberRep, lengthKind, binaryNumberKnownLengthInBits) match { + case (BinaryNumberRep.Binary, LengthKind.EndOfParent, _) => + SDE("lengthKind='endOfParent' is not allowed with binary number representation") case (BinaryNumberRep.Binary, LengthKind.Prefixed, _) => new BinaryDecimalPrefixedLength(this) case (BinaryNumberRep.Binary, _, -1) => new BinaryDecimalRuntimeLength(this) @@ -1178,6 +1211,8 @@ trait ElementBaseGrammarMixin new PackedDecimalDelimitedEndOfData(this, packedSignCodes) case (BinaryNumberRep.Packed, LengthKind.Prefixed, _) => new PackedDecimalPrefixedLength(this, packedSignCodes) + case (BinaryNumberRep.Packed, LengthKind.EndOfParent, _) => + new PackedDecimalEndOfParentLength(this, packedSignCodes) case (BinaryNumberRep.Packed, _, -1) => new PackedDecimalRuntimeLength(this, packedSignCodes) case (BinaryNumberRep.Packed, _, _) => @@ -1186,6 +1221,8 @@ trait ElementBaseGrammarMixin new BCDDecimalDelimitedEndOfData(this) case (BinaryNumberRep.Bcd, LengthKind.Prefixed, _) => new BCDDecimalPrefixedLength(this) + case (BinaryNumberRep.Bcd, LengthKind.EndOfParent, _) => + new BCDDecimalEndOfParentLength(this) case (BinaryNumberRep.Bcd, _, -1) => new BCDDecimalRuntimeLength(this) case (BinaryNumberRep.Bcd, _, _) => new BCDDecimalKnownLength(this, binaryNumberKnownLengthInBits) @@ -1193,6 +1230,8 @@ trait ElementBaseGrammarMixin new IBM4690PackedDecimalDelimitedEndOfData(this) case (BinaryNumberRep.Ibm4690Packed, LengthKind.Prefixed, _) => new IBM4690PackedDecimalPrefixedLength(this) + case (BinaryNumberRep.Ibm4690Packed, LengthKind.EndOfParent, _) => + new IBM4690PackedDecimalEndOfParentLength(this) case (BinaryNumberRep.Ibm4690Packed, _, -1) => new IBM4690PackedDecimalRuntimeLength(this) case (BinaryNumberRep.Ibm4690Packed, _, _) => @@ -1203,6 +1242,7 @@ trait ElementBaseGrammarMixin case PrimType.Boolean => { lengthKind match { case LengthKind.Prefixed => new BinaryBooleanPrefixedLength(this) + case LengthKind.EndOfParent => new BinaryBooleanEndOfParentLength(this) case _ => new BinaryBoolean(this) } } @@ -1256,6 +1296,7 @@ trait ElementBaseGrammarMixin (lengthKind, binaryNumberKnownLengthInBits) match { case (LengthKind.Delimited, -1) => bcdDelimitedLengthCalendar case (LengthKind.Prefixed, -1) => bcdPrefixedLengthCalendar + case (LengthKind.EndOfParent, -1) => bcdEndOfParentLengthCalendar case (_, -1) => bcdRuntimeLengthCalendar case (_, _) => bcdKnownLengthCalendar } @@ -1264,6 +1305,7 @@ trait ElementBaseGrammarMixin (lengthKind, binaryNumberKnownLengthInBits) match { case (LengthKind.Delimited, -1) => ibm4690PackedDelimitedLengthCalendar case (LengthKind.Prefixed, -1) => ibm4690PackedPrefixedLengthCalendar + case (LengthKind.EndOfParent, -1) => ibm4690PackedEndOfParentLengthCalendar case (_, -1) => ibm4690PackedRuntimeLengthCalendar case (_, _) => ibm4690PackedKnownLengthCalendar } @@ -1272,6 +1314,7 @@ trait ElementBaseGrammarMixin (lengthKind, binaryNumberKnownLengthInBits) match { case (LengthKind.Delimited, -1) => packedDelimitedLengthCalendar case (LengthKind.Prefixed, -1) => packedPrefixedLengthCalendar + case (LengthKind.EndOfParent, -1) => packedEndOfParentLengthCalendar case (_, -1) => packedRuntimeLengthCalendar case (_, _) => packedKnownLengthCalendar } @@ -1474,13 +1517,15 @@ trait ElementBaseGrammarMixin // processor to calculate the length and set the bit limit which this processor will use as // the length. The following determines if this element requires another processor to // calculate and set the bit limit, and if so adds the appropriate grammar to do that - val bodyRequiresSpecifiedLengthBitLimit = lengthKind != LengthKind.Delimited && ( - isSimpleType && impliedRepresentation == Representation.Text || - isSimpleType && isNillable || - isComplexType && lengthKind != LengthKind.Implicit || - lengthKind == LengthKind.Prefixed || - isSimpleType && primType == PrimType.HexBinary && lengthKind == LengthKind.Pattern - ) + val bodyRequiresSpecifiedLengthBitLimit = lengthKind != LengthKind.Delimited + && !(isSimpleType && lengthKind == LengthKind.EndOfParent && !this.isInstanceOf[Root]) + && ( + isSimpleType && impliedRepresentation == Representation.Text || + isSimpleType && isNillable || + isComplexType && lengthKind != LengthKind.Implicit || + lengthKind == LengthKind.Prefixed || + isSimpleType && primType == PrimType.HexBinary && lengthKind == LengthKind.Pattern + ) if (!bodyRequiresSpecifiedLengthBitLimit) { body } else { @@ -1524,8 +1569,9 @@ trait ElementBaseGrammarMixin case LengthKind.Implicit if isSimpleType && impliedRepresentation == Representation.Binary => new SpecifiedLengthImplicit(this, body, implicitBinaryLengthInBits) - case LengthKind.EndOfParent => new SpecifiedLengthEndOfParent(this, body) - case LengthKind.Delimited | LengthKind.Implicit => + case LengthKind.EndOfParent if (isComplexType || this.isInstanceOf[Root]) => + new SpecifiedLengthEndOfParent(this, body) + case LengthKind.Delimited | LengthKind.Implicit | LengthKind.EndOfParent => Assert.impossibleCase( "Delimited and ComplexType Implicit cases should not be reached" ) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesBCD.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesBCD.scala index e9e0996b00..5dda4397b1 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesBCD.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesBCD.scala @@ -52,7 +52,18 @@ class BCDIntegerKnownLength(val e: ElementBase, lengthInBits: Long) extends Term class BCDIntegerPrefixedLength(val e: ElementBase) extends Terminal(e, true) { - override lazy val parser = new BCDIntegerBitLimitLengthParser(e.elementRuntimeData) + override lazy val parser = + new BCDIntegerBitLimitLengthParser(e.elementRuntimeData, isEndOfParent = false) + + override lazy val unparser: Unparser = new BCDIntegerMinimumLengthUnparser( + e.elementRuntimeData + ) +} + +class BCDIntegerEndOfParentLength(val e: ElementBase) extends Terminal(e, true) { + + override lazy val parser = + new BCDIntegerBitLimitLengthParser(e.elementRuntimeData, isEndOfParent = true) override lazy val unparser: Unparser = new BCDIntegerMinimumLengthUnparser( e.elementRuntimeData @@ -94,9 +105,26 @@ class BCDDecimalPrefixedLength(val e: ElementBase) extends Terminal(e, true) { override lazy val parser = new BCDDecimalBitLimitLengthParser( + e.elementRuntimeData, + e.binaryDecimalVirtualPoint, + isEndOfParent = false + ) + + override lazy val unparser: Unparser = + new BCDDecimalMinimumLengthUnparser( e.elementRuntimeData, e.binaryDecimalVirtualPoint ) +} + +class BCDDecimalEndOfParentLength(val e: ElementBase) extends Terminal(e, true) { + + override lazy val parser = + new BCDDecimalBitLimitLengthParser( + e.elementRuntimeData, + e.binaryDecimalVirtualPoint, + isEndOfParent = true + ) override lazy val unparser: Unparser = new BCDDecimalMinimumLengthUnparser( diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesBinaryBoolean.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesBinaryBoolean.scala index b934a24fb5..a5be07b9c1 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesBinaryBoolean.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesBinaryBoolean.scala @@ -47,11 +47,29 @@ class BinaryBoolean(val e: ElementBase) extends Terminal(e, true) { class BinaryBooleanPrefixedLength(val e: ElementBase) extends Terminal(e, true) { override lazy val parser = new BinaryBooleanBitLimitLengthParser( + e.elementRuntimeData, + e.binaryBooleanTrueRep, + e.binaryBooleanFalseRep, + e.lengthUnits, + isEndOfParent = false + ) + + override lazy val unparser: Unparser = new BinaryBooleanMinimumLengthUnparser( e.elementRuntimeData, e.binaryBooleanTrueRep, e.binaryBooleanFalseRep, e.lengthUnits ) +} + +class BinaryBooleanEndOfParentLength(val e: ElementBase) extends Terminal(e, true) { + override lazy val parser = new BinaryBooleanBitLimitLengthParser( + e.elementRuntimeData, + e.binaryBooleanTrueRep, + e.binaryBooleanFalseRep, + e.lengthUnits, + isEndOfParent = true + ) override lazy val unparser: Unparser = new BinaryBooleanMinimumLengthUnparser( e.elementRuntimeData, diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesIBM4690Packed.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesIBM4690Packed.scala index 91d0d25efe..6c04df2801 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesIBM4690Packed.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesIBM4690Packed.scala @@ -59,7 +59,16 @@ class IBM4690PackedIntegerKnownLength(val e: ElementBase, lengthInBits: Long) class IBM4690PackedIntegerPrefixedLength(val e: ElementBase) extends Terminal(e, true) { override lazy val parser = - new IBM4690PackedIntegerBitLimitLengthParser(e.elementRuntimeData) + new IBM4690PackedIntegerBitLimitLengthParser(e.elementRuntimeData, isEndOfParent = false) + + override lazy val unparser: Unparser = new IBM4690PackedIntegerMinimumLengthUnparser( + e.elementRuntimeData + ) +} + +class IBM4690PackedIntegerEndOfParentLength(val e: ElementBase) extends Terminal(e, true) { + override lazy val parser = + new IBM4690PackedIntegerBitLimitLengthParser(e.elementRuntimeData, isEndOfParent = true) override lazy val unparser: Unparser = new IBM4690PackedIntegerMinimumLengthUnparser( e.elementRuntimeData @@ -104,10 +113,26 @@ class IBM4690PackedDecimalKnownLength(val e: ElementBase, lengthInBits: Long) class IBM4690PackedDecimalPrefixedLength(val e: ElementBase) extends Terminal(e, true) { override lazy val parser = new IBM4690PackedDecimalBitLimitLengthParser( + e.elementRuntimeData, + e.binaryDecimalVirtualPoint, + e.decimalSigned, + isEndOfParent = false + ) + + override lazy val unparser: Unparser = new IBM4690PackedDecimalMinimumLengthUnparser( e.elementRuntimeData, e.binaryDecimalVirtualPoint, e.decimalSigned ) +} + +class IBM4690PackedDecimalEndOfParentLength(val e: ElementBase) extends Terminal(e, true) { + override lazy val parser = new IBM4690PackedDecimalBitLimitLengthParser( + e.elementRuntimeData, + e.binaryDecimalVirtualPoint, + e.decimalSigned, + isEndOfParent = true + ) override lazy val unparser: Unparser = new IBM4690PackedDecimalMinimumLengthUnparser( e.elementRuntimeData, diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesLengthKind.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesLengthKind.scala index fcda6161f8..521e1370b7 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesLengthKind.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesLengthKind.scala @@ -174,7 +174,8 @@ case class HexBinaryDelimitedEndOfData(e: ElementBase) extends HexBinaryDelimite case class HexBinaryEndOfBitLimit(e: ElementBase) extends Terminal(e, true) { override lazy val parser: DaffodilParser = new HexBinaryEndOfBitLimitParser( - e.elementRuntimeData + e.elementRuntimeData, + isEndOfParent = false ) override lazy val unparser: DaffodilUnparser = @@ -184,7 +185,8 @@ case class HexBinaryEndOfBitLimit(e: ElementBase) extends Terminal(e, true) { case class HexBinaryLengthPrefixed(e: ElementBase) extends Terminal(e, true) { override lazy val parser: DaffodilParser = new HexBinaryEndOfBitLimitParser( - e.elementRuntimeData + e.elementRuntimeData, + isEndOfParent = false ) override lazy val unparser: DaffodilUnparser = @@ -194,7 +196,8 @@ case class HexBinaryLengthPrefixed(e: ElementBase) extends Terminal(e, true) { case class HexBinaryLengthEndOfParent(e: ElementBase) extends Terminal(e, true) { override lazy val parser: DaffodilParser = new HexBinaryEndOfBitLimitParser( - e.elementRuntimeData + e.elementRuntimeData, + isEndOfParent = true ) override lazy val unparser: DaffodilUnparser = diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesPacked.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesPacked.scala index f5c81be1f4..bb4b010cd2 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesPacked.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesPacked.scala @@ -78,7 +78,27 @@ class PackedIntegerPrefixedLength( ) extends Terminal(e, true) { override lazy val parser = - new PackedIntegerBitLimitLengthParser(e.elementRuntimeData, packedSignCodes) + new PackedIntegerBitLimitLengthParser( + e.elementRuntimeData, + packedSignCodes, + isEndOfParent = false + ) + + override lazy val unparser: Unparser = + new PackedIntegerMinimumLengthUnparser(e.elementRuntimeData, packedSignCodes) +} + +class PackedIntegerEndOfParentLength( + val e: ElementBase, + packedSignCodes: PackedSignCodes +) extends Terminal(e, true) { + + override lazy val parser = + new PackedIntegerBitLimitLengthParser( + e.elementRuntimeData, + packedSignCodes, + isEndOfParent = true + ) override lazy val unparser: Unparser = new PackedIntegerMinimumLengthUnparser(e.elementRuntimeData, packedSignCodes) @@ -132,11 +152,31 @@ class PackedDecimalPrefixedLength(val e: ElementBase, packedSignCodes: PackedSig extends Terminal(e, true) { override lazy val parser = new PackedDecimalBitLimitLengthParser( + e.elementRuntimeData, + e.binaryDecimalVirtualPoint, + packedSignCodes, + e.decimalSigned, + isEndOfParent = false + ) + + override lazy val unparser: Unparser = new PackedDecimalMinimumLengthUnparser( e.elementRuntimeData, e.binaryDecimalVirtualPoint, packedSignCodes, e.decimalSigned ) +} + +class PackedDecimalEndOfParentLength(val e: ElementBase, packedSignCodes: PackedSignCodes) + extends Terminal(e, true) { + + override lazy val parser = new PackedDecimalBitLimitLengthParser( + e.elementRuntimeData, + e.binaryDecimalVirtualPoint, + packedSignCodes, + e.decimalSigned, + isEndOfParent = true + ) override lazy val unparser: Unparser = new PackedDecimalMinimumLengthUnparser( e.elementRuntimeData, diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/SpecifiedLength.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/SpecifiedLength.scala index 49baa728c2..5c2bde7f84 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/SpecifiedLength.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/SpecifiedLength.scala @@ -146,14 +146,8 @@ class SpecifiedLengthEndOfParent(e: ElementBase, eGram: => Gram) ) } - lazy val unparser: Unparser = { - if (eUnparser.isEmpty) eUnparser - else - new SpecifiedLengthEndOfParentUnparser( - eUnparser, - e.elementRuntimeData - ) - } + lazy val unparser: Unparser = eUnparser + } class SpecifiedLengthImplicit(e: ElementBase, eGram: => Gram, nBits: Long) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSource.scala b/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSource.scala index c951fb2c7f..627c48ee2a 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSource.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSource.scala @@ -181,6 +181,12 @@ abstract class InputSource { */ def releasePosition(bytePos0b: Long): Unit + /** + * Get the number of bytes that are available to be read until the end of + * the data stream. + */ + lazy val bytesTillEndOfDataStream: Long = -1L + /** * Alerts the implementation to attempt to free data that is no longer used, * if possible. If possible, this should free any unlocked bytes. @@ -564,6 +570,22 @@ class BucketingInputSource( headBucketBytePosition0b += bytesRemoved oldestBucketIndex = 0 } + + override lazy val bytesTillEndOfDataStream: Long = { + val initialBytePosition = position() + val endOfDataNotReached = fillBucketsToIndex(maxNumberOfNonNullBuckets, 8) + if (!endOfDataNotReached) { + val numBytesFilled = totalBytesBucketed - initialBytePosition + // reset the byte position to the initial byte position + position(initialBytePosition) + numBytesFilled + } else { + position(initialBytePosition) + throw new Exception( + s"Attempted to fill to end of data stream, but did not reach end of data stream before maxCacheSizeInBytes: ${maxCacheSizeInBytes}." + ) + } + } } /** @@ -645,4 +667,11 @@ class ByteBufferInputSource(byteBuffer: ByteBuffer) extends InputSource { override def close(): Unit = { // do nothing. No resources to release. } + + override lazy val bytesTillEndOfDataStream: Long = { + val initialPosition = position() + val br = byteBuffer.remaining + position(initialPosition) + br + } } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSourceDataInputStream.scala b/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSourceDataInputStream.scala index 8e39d7db43..5e1bc1ff5d 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSourceDataInputStream.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSourceDataInputStream.scala @@ -136,6 +136,8 @@ final class InputSourceDataInputStream private (val inputSource: InputSource) */ def hasReachedEndOfData: Boolean = inputSource.hasReachedEndOfData + lazy val bytesTillEndOfDataStream: Long = inputSource.bytesTillEndOfDataStream + /** * Return the number of currently available bytes. * diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/BCDParsers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/BCDParsers.scala index 36fc5ddeda..f6c285d04a 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/BCDParsers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/BCDParsers.scala @@ -52,9 +52,10 @@ class BCDDecimalRuntimeLengthParser( class BCDDecimalBitLimitLengthParser( e: ElementRuntimeData, - binaryDecimalVirtualPoint: Int + binaryDecimalVirtualPoint: Int, + isEndOfParent: Boolean ) extends PackedBinaryDecimalBaseParser(e, binaryDecimalVirtualPoint, None) - with BitLengthFromBitLimitMixin { + with BitLengthFromBitLimitMixin(isEndOfParent) { override def toNumber(num: Array[Byte]): JBigDecimal = DecimalUtils.bcdToBigDecimal(num, binaryDecimalVirtualPoint) @@ -79,9 +80,9 @@ class BCDIntegerKnownLengthParser(e: ElementRuntimeData, val lengthInBits: Int) } -class BCDIntegerBitLimitLengthParser(e: ElementRuntimeData) +class BCDIntegerBitLimitLengthParser(e: ElementRuntimeData, isEndOfParent: Boolean) extends PackedBinaryIntegerBaseParser(e) - with BitLengthFromBitLimitMixin { + with BitLengthFromBitLimitMixin(isEndOfParent) { override def toNumber(num: Array[Byte]): JBigInteger = DecimalUtils.bcdToBigInteger(num) } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/IBM4690PackedDecimalParsers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/IBM4690PackedDecimalParsers.scala index 6ea612a57d..e9761a4349 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/IBM4690PackedDecimalParsers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/IBM4690PackedDecimalParsers.scala @@ -56,9 +56,10 @@ class IBM4690PackedDecimalRuntimeLengthParser( class IBM4690PackedDecimalBitLimitLengthParser( e: ElementRuntimeData, binaryDecimalVirtualPoint: Int, - decimalSigned: YesNo + decimalSigned: YesNo, + isEndOfParent: Boolean ) extends PackedBinaryDecimalBaseParser(e, binaryDecimalVirtualPoint, Some(decimalSigned)) - with BitLengthFromBitLimitMixin { + with BitLengthFromBitLimitMixin(isEndOfParent) { override def toNumber(num: Array[Byte]): JBigDecimal = DecimalUtils.ibm4690ToBigDecimal(num, binaryDecimalVirtualPoint) @@ -88,9 +89,9 @@ class IBM4690PackedIntegerKnownLengthParser( } -class IBM4690PackedIntegerBitLimitLengthParser(e: ElementRuntimeData) +class IBM4690PackedIntegerBitLimitLengthParser(e: ElementRuntimeData, isEndOfParent: Boolean) extends PackedBinaryIntegerBaseParser(e) - with BitLengthFromBitLimitMixin { + with BitLengthFromBitLimitMixin(isEndOfParent) { override def toNumber(num: Array[Byte]): JBigInteger = DecimalUtils.ibm4690ToBigInteger(num) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/PackedDecimalParsers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/PackedDecimalParsers.scala index 46fc6c091d..59779e1a84 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/PackedDecimalParsers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/PackedDecimalParsers.scala @@ -58,9 +58,10 @@ class PackedDecimalBitLimitLengthParser( e: ElementRuntimeData, binaryDecimalVirtualPoint: Int, packedSignCodes: PackedSignCodes, - decimalSigned: YesNo + decimalSigned: YesNo, + isEndOfParent: Boolean ) extends PackedBinaryDecimalBaseParser(e, binaryDecimalVirtualPoint, Some(decimalSigned)) - with BitLengthFromBitLimitMixin { + with BitLengthFromBitLimitMixin(isEndOfParent) { override def toNumber(num: Array[Byte]): JBigDecimal = DecimalUtils.packedToBigDecimal(num, binaryDecimalVirtualPoint, packedSignCodes) @@ -93,9 +94,10 @@ class PackedIntegerKnownLengthParser( class PackedIntegerBitLimitLengthParser( e: ElementRuntimeData, - packedSignCodes: PackedSignCodes + packedSignCodes: PackedSignCodes, + isEndOfParent: Boolean ) extends PackedBinaryIntegerBaseParser(e) - with BitLengthFromBitLimitMixin { + with BitLengthFromBitLimitMixin(isEndOfParent) { override def toNumber(num: Array[Byte]): JBigInteger = DecimalUtils.packedToBigInteger(num, packedSignCodes) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BinaryBooleanParsers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BinaryBooleanParsers.scala index a4a30de0bd..72584d234a 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BinaryBooleanParsers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BinaryBooleanParsers.scala @@ -109,9 +109,10 @@ class BinaryBooleanBitLimitLengthParser( override val context: ElementRuntimeData, binaryBooleanTrueRep: MaybeULong, binaryBooleanFalseRep: ULong, - lengthUnits: LengthUnits + lengthUnits: LengthUnits, + isEndOfParent: Boolean ) extends BinaryBooleanParserBase(binaryBooleanTrueRep, binaryBooleanFalseRep, lengthUnits) - with BitLengthFromBitLimitMixin { + with BitLengthFromBitLimitMixin(isEndOfParent) { override def runtimeDependencies = Vector() diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/HexBinaryLengthParsers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/HexBinaryLengthParsers.scala index 0dc7b21996..2e496d59df 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/HexBinaryLengthParsers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/HexBinaryLengthParsers.scala @@ -83,9 +83,9 @@ final class HexBinarySpecifiedLengthParser(erd: ElementRuntimeData, lengthEv: Le } -final class HexBinaryEndOfBitLimitParser(erd: ElementRuntimeData) +final class HexBinaryEndOfBitLimitParser(erd: ElementRuntimeData, isEndOfParent: Boolean) extends HexBinaryLengthParser(erd), - BitLengthFromBitLimitMixin { + BitLengthFromBitLimitMixin(isEndOfParent) { override def runtimeDependencies = Vector() } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/ParserTraits.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/ParserTraits.scala index 814a9ee31b..93feb7d6de 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/ParserTraits.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/ParserTraits.scala @@ -172,7 +172,7 @@ trait PrefixedLengthParserMixin { * An example of this is prefix length parsers. This trait can be used by those * parsers to do determine the length based on the bitLimit and position. */ -trait BitLengthFromBitLimitMixin { +trait BitLengthFromBitLimitMixin(val isEndOfParent: Boolean = false) { def getBitLength(s: ParseOrUnparseState): Int = { val pState = s.asInstanceOf[PState] @@ -181,7 +181,14 @@ trait BitLengthFromBitLimitMixin { } def getLengthInBits(pstate: PState): Long = { - val len = pstate.bitLimit0b.get - pstate.bitPos0b - len + if (pstate.bitLimit0b.isDefined) { + val len = pstate.bitLimit0b.get - pstate.bitPos0b + len + } else if (isEndOfParent) { + val byteLimit = pstate.dataInputStream.bytesTillEndOfDataStream + byteLimit * 8 + } else { + Assert.invariantFailed("BitLimit not set for parser.") + } } } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala index 1d2bb11521..a2fec40539 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala @@ -136,7 +136,7 @@ class SpecifiedLengthEndOfParentParser( eParser: Parser, erd: ElementRuntimeData ) extends SpecifiedLengthParserBase(eParser, erd), - BitLengthFromBitLimitMixin { + BitLengthFromBitLimitMixin(true) { override protected def getBitLength(s: PState): MaybeULong = { MaybeULong(super[BitLengthFromBitLimitMixin].getBitLength(s)) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/SpecifiedLengthUnparsers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/SpecifiedLengthUnparsers.scala index 071c810a04..44af8e887a 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/SpecifiedLengthUnparsers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/SpecifiedLengthUnparsers.scala @@ -72,20 +72,6 @@ final class SpecifiedLengthExplicitImplicitUnparser( } } -final class SpecifiedLengthEndOfParentUnparser( - eUnparser: Unparser, - erd: ElementRuntimeData -) extends CombinatorUnparser(erd) { - - override def runtimeDependencies = Vector() - - override def childProcessors = Vector(eUnparser) - - override final def unparse(state: UState): Unit = { - eUnparser.unparse1(state) - } -} - /** * This trait is to be used with prefixed length unparsers where the length * must be calculated based on the content length of the data. This means the diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/EndOfParentTests.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/EndOfParentTests.tdml index 29aa3214f0..6b2348fee5 100644 --- a/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/EndOfParentTests.tdml +++ b/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/EndOfParentTests.tdml @@ -20,7 +20,8 @@ description="Section 12 - lengthKind=endOfParent" xmlns:tdml="http://www.ibm.com/xmlns/dfdl/testData" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:dfdl="http://www.ogf.org/dfdl/dfdl-1.0/" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:fn="http://www.w3.org/2005/xpath-functions" - xmlns:ex="http://example.com" defaultRoundTrip="onePass"> + xmlns:ex="http://example.com" defaultRoundTrip="onePass" + xmlns:dfdlx="http://www.ogf.org/dfdl/dfdl-1.0/extensions"> @@ -253,6 +254,14 @@ + + + + + + + + + + + + endOfParent + does not have + single-byte character set encoding + + + @@ -1247,7 +1267,7 @@ + dfdl:lengthKind="endOfParent" dfdl:representation="text" dfdl:encoding="UTF-8"> @@ -1278,7 +1298,7 @@ - @@ -1675,6 +1695,77 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1862,8 +1953,7 @@ separator="" leadingSkip='0' encoding="US-ASCII" ignoreCase='no' initiator="" terminator="" initiatedContent="no" textNumberRep="standard" separatorSuppressionPolicy="anyEmpty" separatorPosition="infix" - documentFinalTerminatorCanBeMissing='yes' byteOrder="bigEndian" - binaryNumberRep='binary' /> + documentFinalTerminatorCanBeMissing='yes' byteOrder="bigEndian" /> @@ -1881,42 +1971,15 @@ - - - - - - - - - - - - - - - - simple type - endOfParent - isn't a string type - - - - - - - simple type endOfParent - doesn't have binary representation with packed decimal representation + supported for + packed binary @@ -1995,5 +2058,909 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + 1 + + + + 2 + 2 + + + 3 + 3 + + + + + + + + + + + + + X + YZ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + abcde + + + + abcde + + + + + + + ábcde + + + endOfParent + represented element + between this element and + end of the enclosing component + + + + + + abcde + + + + + abcde + + + + + + + + abcde + + + + + + abcde + + + + + + + + + abcde + + + + abcde + + + + + + + abcde + + + + abcde + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 12 + + + + 12 + + + + + + + 123 + + + endOfParent + represented element + between this element and + end of the enclosing component + + + + + + 12 + + + + 12 + + + + + + + 12 + + + + 12 + + + + + + + 123.45 + + + + 123.45 + + + + + + 123.45 + + + + 123.45 + + + + + + + 123.45 + + + + 123.45 + + + + + + + 2017-08-28T18:29:00 + + + + 2017-08-28T18:29:00 + + + + + + + 2017-08-28T18:29:00 + + + + 2017-08-28T18:29:00 + + + + + + + 2017-08-28T18:29:00 + + + + 2017-08-28T18:29:00 + + + + + + + TRUE + + + + true + + + + + + + TRUE + + + + true + + + + + + + + TRUE + + + + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 01234C + + + + 1234 + + + + + + + + 1234 + + + + 1234 + + + + + + + 1234 + + + + 1234 + + + + + + + 30 39 + + + endOfParent + only supported + packed binary formats + + + + + + 01234C + + + + 12.34 + + + + + + + 1234 + + + + 12.34 + + + + + + + 1234 + + + + 12.34 + + + + + + + + DE AD BE EF + + + + DEADBEEF + + + + + + + + 01 + + + endOfParent + doesn't have + packed decimal representation + + + + + + + 01 22 51 64 51 93 65 0C + + + + 1645-12-25T19:36:50 + + + + + + + 06 14 20 04 18 56 03 + + + + 2004-06-14T18:56:03 + + + + + + + 11 30 20 07 04 15 08 + + + + 2007-11-30T04:15:08 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + hello + + + + hello + + + + + + + hello + + + + hello + + + + + + + hello} + + + endOfParent + sequence with + terminator + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + helff + + + + hel + + + + + + + helff + + + + hel + + + diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section13/packed/packed.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section13/packed/packed.tdml index 6714a288c5..ce71d4d347 100644 --- a/daffodil-test/src/test/resources/org/apache/daffodil/section13/packed/packed.tdml +++ b/daffodil-test/src/test/resources/org/apache/daffodil/section13/packed/packed.tdml @@ -259,6 +259,68 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1050,4 +1112,187 @@ + + + + 003C 123D + + + + + 3 + -123 + + + + + + + + + 003C 123D + + + + + 0.03 + -1.23 + + + + + + + + + + 3 + -123 + + + + + + 003C 123D + + + + + + + + + 0.03 + -1.23 + + + + + + 003C 123D + + + + + + + 0003 0123 + + + + + 3 + 123 + + + + + + + + + 0003 0123 + + + + + 0.03 + 1.23 + + + + + + + + + + 3 + 123 + + + + + + 0003 0123 + + + + + + + + 0.03 + 1.23 + + + + + + 0003 0123 + + + + + + + + FFF3 D123 + + + + + 3 + -123 + + + + + + + + + FFF3 D123 + + + + + 0.03 + -1.23 + + + + + + + + + + + 3 + -123 + + + + + + FFF3 D123 + + + + + + + + 0.03 + -1.23 + + + + + + FFF3 D123 + + + diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section15/choice_groups/ChoiceLengthExplicit.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section15/choice_groups/ChoiceLengthExplicit.tdml index 099c6c5a1b..ff00047748 100644 --- a/daffodil-test/src/test/resources/org/apache/daffodil/section15/choice_groups/ChoiceLengthExplicit.tdml +++ b/daffodil-test/src/test/resources/org/apache/daffodil/section15/choice_groups/ChoiceLengthExplicit.tdml @@ -84,6 +84,45 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + A + BC + + + + + + + + + + + + + + + + + A + BC + + + + + + + diff --git a/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindEndOfParent.scala b/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindEndOfParent.scala index 53f25c3720..606daefff3 100644 --- a/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindEndOfParent.scala +++ b/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindEndOfParent.scala @@ -35,6 +35,7 @@ class TestLengthKindEndOfParent extends TdmlTests { @Test def TestEndOfParentSimpleTypesImplicit = test @Test def TestEndOfParentComplexTypesExplicit = test @Test def TestEndOfParentSimpleTypesExplicit = test + @Test def TestEndOfParentCSVExplicit = test @Test def TestEndOfParentComplexTypesPrefixed = test @Test def TestEndOfParentSimpleTypesPrefixed = test @Test def TestEndOfParentComplexTypesPattern = test @@ -63,12 +64,56 @@ class TestLengthKindEndOfParent extends TdmlTests { @Test def TestEndOfParentSimpleTypes10 = test @Test def TestEndOfParentComplexTypes11 = test @Test def TestEndOfParentSimpleTypes11 = test +// @Test def TestEndOfParentComplexTypesRootChoice = test +// @Test def TestEndOfParentSimpleTypesRootChoice = test @Test def TestEndOfParentComplexTypes12 = test @Test def TestEndOfParentSimpleTypes12 = test @Test def TestEndOfParentComplexTypes13 = test @Test def TestEndOfParentSimpleTypes13 = test @Test def TestEndOfParentSimpleTypes14 = test - @Test def TestEndOfParentSimpleTypes15 = test @Test def TestEndOfParentSimpleTypes16 = test @Test def TestEndOfParentSimpleTypes17 = test + @Test def TestEndOfParentComplexTypesRootEOP = test + @Test def TestEndOfParentSimpleTypesRootEOP = test + + @Test def text_string_txt_bytes = test + @Test def text_string_txt_bits = test + @Test def text_string_txt_chars = test + @Test def text_string_txt_ref1 = test + @Test def text_string_txt_ref2 = test + @Test def text_string_txt_ref3 = test + @Test def text_string_txt_bytes_nil = test + @Test def text_string_txt_bits_nil = test + @Test def text_string_txt_chars_nil = test + @Test def text_int_txt_bytes = test + @Test def text_int_txt_bytes_group_ref = test + @Test def text_int_txt_bits = test + @Test def text_int_txt_chars = test + @Test def text_dec_txt_bytes = test + @Test def text_dec_txt_bits = test + @Test def text_dec_txt_chars = test + @Test def text_date_txt_bytes = test + @Test def text_date_txt_bits = test + @Test def text_date_txt_chars = test + @Test def text_bool_txt_bytes = test + @Test def text_bool_txt_bits = test + @Test def text_bool_txt_chars = test + @Test def bin_int_bin_bytes_packed = test + @Test def bin_int_bin_bytes_bcd = test + @Test def bin_int_bin_bytes_ibm4690 = test + @Test def bin_dec_bin_bytes = test + @Test def bin_dec_bin_bytes_packed = test + @Test def bin_dec_bin_bytes_bcd = test + @Test def bin_dec_bin_bytes_ibm4690 = test + @Test def bin_hex_bytes = test + @Test def bin_bool_bin_bytes = test + @Test def bin_date_bin_bytes_packed = test + @Test def bin_date_bin_bytes_bcd = test + @Test def bin_date_bin_bytes_ibm4690 = test + + @Test def nested_01 = test + @Test def nested_02 = test + @Test def nested_03 = test + @Test def checks_01 = test + @Test def checks_02 = test } diff --git a/daffodil-test/src/test/scala/org/apache/daffodil/section13/packed/TestPacked.scala b/daffodil-test/src/test/scala/org/apache/daffodil/section13/packed/TestPacked.scala index 48fbcf95cf..4c7b516881 100644 --- a/daffodil-test/src/test/scala/org/apache/daffodil/section13/packed/TestPacked.scala +++ b/daffodil-test/src/test/scala/org/apache/daffodil/section13/packed/TestPacked.scala @@ -95,4 +95,18 @@ class TestPacked extends TdmlTests { // Daffodil-2961 @Test def bcdBigIntToLongExpr = test + + // Daffodil-238 + @Test def EOPPackedIntSeq = test + @Test def EOPPackedDecSeq = test + @Test def EOPPackedIntSeqUnparser = test + @Test def EOPPackedDecSeqUnparser = test + @Test def EOPBCDIntSeq = test + @Test def EOPBCDDecSeq = test + @Test def EOPBCDIntSeqUnparser = test + @Test def EOPBCDDecSeqUnparser = test + @Test def EOPIBM4690IntSeq = test + @Test def EOPIBM4690DecSeq = test + @Test def EOPIBM4690IntSeqUnparser = test + @Test def EOPIBM4690DecSeqUnparser = test } diff --git a/daffodil-test/src/test/scala/org/apache/daffodil/section15/choice_groups/TestChoice.scala b/daffodil-test/src/test/scala/org/apache/daffodil/section15/choice_groups/TestChoice.scala index cfbb16a117..075f3a5183 100644 --- a/daffodil-test/src/test/scala/org/apache/daffodil/section15/choice_groups/TestChoice.scala +++ b/daffodil-test/src/test/scala/org/apache/daffodil/section15/choice_groups/TestChoice.scala @@ -133,6 +133,8 @@ class TestChoiceLengthExplicit extends TdmlTests { @Test def explicit_07 = test @Test def explicit_08 = test @Test def explicit_09 = test + @Test def explicit_10 = test + @Test def explicit_11 = test @Test def explicit_multiple_choices = test From 4a974ba0ba93716d12b4b11259021f64566a992e Mon Sep 17 00:00:00 2001 From: olabusayoT <50379531+olabusayoT@users.noreply.github.com> Date: Tue, 26 May 2026 10:41:51 -0400 Subject: [PATCH 3/9] fixup! fixup! fixup! fixup! fixup! fixup! fixup! - consolidate checkEndOfParentRestrictions in TermGrammarMixin - add more tests - clean up commented code --- .../daffodil/core/dsom/ChoiceGroup.scala | 30 --- .../apache/daffodil/core/dsom/SchemaSet.scala | 2 +- .../daffodil/core/dsom/SequenceGroup.scala | 35 ---- .../org/apache/daffodil/core/dsom/Term.scala | 13 +- .../grammar/ElementBaseGrammarMixin.scala | 85 +------- .../core/grammar/TermGrammarMixin.scala | 189 +++++++++++++++++ .../lengthKind/EndOfParentTests.tdml | 194 +++++++++++------- .../TestLengthKindEndOfParent.scala | 5 +- 8 files changed, 315 insertions(+), 238 deletions(-) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/ChoiceGroup.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/ChoiceGroup.scala index 8af289b2b9..e662e1ce6b 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/ChoiceGroup.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/ChoiceGroup.scala @@ -24,9 +24,7 @@ import org.apache.daffodil.core.dsom.walker.ChoiceView import org.apache.daffodil.core.grammar.ChoiceGrammarMixin import org.apache.daffodil.lib.schema.annotation.props.Found import org.apache.daffodil.lib.schema.annotation.props.gen.ChoiceAGMixin -import org.apache.daffodil.lib.schema.annotation.props.gen.ChoiceLengthKind import org.apache.daffodil.lib.schema.annotation.props.gen.Choice_AnnotationMixin -import org.apache.daffodil.lib.schema.annotation.props.gen.LengthUnits import org.apache.daffodil.lib.schema.annotation.props.gen.YesNo /** @@ -105,7 +103,6 @@ abstract class ChoiceTermBase( requiredEvaluationsIfActivated(noBranchesFound) requiredEvaluationsIfActivated(branchesAreNonOptional) requiredEvaluationsIfActivated(branchesAreNotIVCElements) - requiredEvaluationsIfActivated(checkEndOfParentRestrictions()) final protected lazy val optionChoiceDispatchKeyRaw = findPropertyOption("choiceDispatchKey", expressionAllowed = true) @@ -192,33 +189,6 @@ abstract class ChoiceTermBase( } assuming(branchesOk.forall { x => x }) }.value - - lazy val optMyEffectiveLengthUnits: Option[LengthUnits] = - this match { - case self: ChoiceTermBase if self.choiceLengthKind == ChoiceLengthKind.Explicit => - Some(LengthUnits.Bytes) - case _ => None - } - - def checkEndOfParentRestrictions() = { - val parent = this - val eopChildren = this.childrenEndOfParent - if (eopChildren.isEmpty) {} else { - parent match { - case c: ChoiceTermBase if c.choiceLengthKind == ChoiceLengthKind.Implicit => { - schemaDefinitionWhen( - c.hasTerminator, - "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a choice with a dfdl:terminator." - ) - schemaDefinitionWhen( - c.trailingSkip != 0, - "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a choice with a non-zero dfdl:trailingSkip." - ) - } - case _ => // do nothing - } - } - } } object Choice { diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/SchemaSet.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/SchemaSet.scala index ca76780db9..dcd6a87ee9 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/SchemaSet.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/SchemaSet.scala @@ -117,7 +117,7 @@ final class SchemaSet private ( requiredEvaluationsAlways(root) requiredEvaluationsAlways(checkForDuplicateTopLevels()) - requiredEvaluationsAlways(root.checkEndOfParentRestrictions(LengthUnits.Characters)) + requiredEvaluationsAlways(root.checkEndOfParentRestrictions(Some(LengthUnits.Characters))) lazy val resolver = DFDLCatalogResolver.get diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/SequenceGroup.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/SequenceGroup.scala index ef510f2a2f..210797ed01 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/SequenceGroup.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/SequenceGroup.scala @@ -34,7 +34,6 @@ import org.apache.daffodil.lib.schema.annotation.props.gen.OccursCountKind import org.apache.daffodil.lib.schema.annotation.props.gen.SeparatorPosition import org.apache.daffodil.lib.schema.annotation.props.gen.SequenceKind import org.apache.daffodil.lib.schema.annotation.props.gen.TestKind -import org.apache.daffodil.lib.schema.annotation.props.gen.YesNo import org.apache.daffodil.lib.xml.RefQName import org.apache.daffodil.lib.xml.XMLUtils import org.apache.daffodil.runtime1.layers.LayerRuntimeData @@ -106,7 +105,6 @@ abstract class SequenceGroupTermBase(xml: Node, lexicalParent: SchemaComponent, requiredEvaluationsIfActivated(checkValidityOccursCountKind) requiredEvaluationsIfActivated(checkIfNonEmptyAndDiscrimsOrAsserts) requiredEvaluationsIfActivated(checkIfMultipleChildrenWithSameName) - requiredEvaluationsIfActivated(checkEndOfParentRestrictions()) protected def apparentXMLChildren: Seq[Node] @@ -271,39 +269,6 @@ abstract class SequenceGroupTermBase(xml: Node, lexicalParent: SchemaComponent, case SequenceKind.Ordered => true case SequenceKind.Unordered => false } - - def checkEndOfParentRestrictions() = { - val parent = this - val eopChildren = this.childrenEndOfParent - if (eopChildren.isEmpty) {} else { - parent match { - case s: SequenceTermBase => { - schemaDefinitionWhen( - s.separatorPosition == SeparatorPosition.Postfix, - "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with dfdl:separatorPosition defined as 'postfix'." - ) - schemaDefinitionWhen( - s.sequenceKind != SequenceKind.Ordered, - "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with dfdl:sequenceKind defined as 'unordered'." - ) - schemaDefinitionWhen( - s.hasTerminator, - "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with a dfdl:terminator." - ) - schemaDefinitionWhen( - s.realElementChildren.exists(e => e.floating == YesNo.Yes), - "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with elements defining dfdl:floating='yes'." - ) - schemaDefinitionWhen( - s.trailingSkip != 0, - "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with a non-zero dfdl:trailingSkip." - ) - } - case null => // do nothing - } - } - } - } /** diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/Term.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/Term.scala index d8db9d33a9..1172f35d56 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/Term.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/Term.scala @@ -94,17 +94,6 @@ trait Term requiredEvaluationsIfActivated(defaultPropertySources) requiredEvaluationsIfActivated(termChecks) - final lazy val childrenEndOfParent: Seq[ElementBase] = LV(Symbol("childrenEndOfParent")) { - val gms = termChildren - val chls = gms.flatMap { - case eb: ElementBase if eb.lengthKind == LengthKind.EndOfParent => Seq(eb) - case eb: ElementBase => Nil - case c: Choice => Nil - case mg: ModelGroup => mg.childrenEndOfParent - } - chls - }.value - private lazy val termChecks = { statements.foreach { _.checkTerm(this) } } @@ -591,7 +580,7 @@ trait Term } } - final protected lazy val realElementChildren: Seq[ElementBase] = { + final lazy val realElementChildren: Seq[ElementBase] = { termChildren.flatMap { case eb: ElementBase => Seq(eb) case c: Choice => Nil diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala index 16b09e2873..ba759e23f7 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala @@ -22,7 +22,6 @@ import java.lang.Long as JLong import org.apache.daffodil.core.dsom.ElementBase import org.apache.daffodil.core.dsom.ExpressionCompilers import org.apache.daffodil.core.dsom.InitiatedTerminatedMixin -import org.apache.daffodil.core.dsom.ModelGroup import org.apache.daffodil.core.dsom.PrefixLengthQuasiElementDecl import org.apache.daffodil.core.dsom.Root import org.apache.daffodil.core.grammar.primitives.* @@ -254,85 +253,7 @@ trait ElementBaseGrammarMixin } final lazy val prefixedLengthBody = prefixedLengthElementDecl.parsedValue - def myEffectiveLengthUnits(lastNonEOPLU: LengthUnits): LengthUnits = { - lengthKind match { - case LengthKind.EndOfParent => lastNonEOPLU - case LengthKind.Explicit | LengthKind.Prefixed => lengthUnits - case LengthKind.Pattern => LengthUnits.Characters - case _ => - Assert.invariantFailed( - "Delimited and Implicit are illegal for the parent of EndOfParent element" - ) - } - } - - def checkChildrenForSiblingsAfterEOPElement( - parent: ElementBase, - specificChild: ElementBase - ) = { - lazy val foundPosition = flattenedChildren.indexOf(specificChild) - lazy val lastIndexOfChildren = flattenedChildren.length - 1 - if (flattenedChildren.isEmpty || foundPosition < 0) { - // not found amongst children - Assert.impossible("EndOfParent element not found amongst term children of parent") - } else if (foundPosition != lastIndexOfChildren) { - // get the following children after the EOP element+ 1 - val followingChildrenAfter = - flattenedChildren.slice(foundPosition + 1, lastIndexOfChildren + 1) - followingChildrenAfter.foreach { - case m: ModelGroup => { - specificChild.SDE( - "element is specified as dfdl:lengthKind=\"endOfParent\", but a model group is defined between this element and the end of the enclosing component" - ) - } - case r if r.isRepresented => { - specificChild.SDE( - "element is specified as dfdl:lengthKind=\"endOfParent\", but a represented element is defined between this element and the end of the enclosing component" - ) - } - case _ => // do nothing - } - } - } - - def checkEndOfParentRestrictions(lastNonEOPLU: LengthUnits): Unit = { - val parent = this - val eopChildren = this.childrenEndOfParent - // checks - this match { - case rootElem: Root if lengthKind == LengthKind.EndOfParent => { - rootElem.checkEndOfParentRestrictionsOnCurrentElement(lastNonEOPLU) - } - case _ => // do nothing - } - - if (eopChildren.isEmpty) - ( - this.elementChildren.foreach(_.checkEndOfParentRestrictions(lastNonEOPLU)) - ) - else { - eopChildren.foreach { eopChild => - lazy val parentELU = myEffectiveLengthUnits(lastNonEOPLU) - checkChildrenForSiblingsAfterEOPElement(parent, eopChild) - parent.lengthKind match { - case LengthKind.Implicit | LengthKind.Delimited => - schemaDefinitionError( - "element is specified as dfdl:lengthKind=\"endOfParent\", but its parent is an element with dfdl:lengthKind 'implicit' or 'delimited'." - ) - case _ => // do nothing - } - eopChild.checkEndOfParentRestrictionsOnCurrentElement(parentELU) - if (parent.lengthKind != LengthKind.EndOfParent) { - eopChild.checkEndOfParentRestrictions(parentELU) - } else { - eopChild.checkEndOfParentRestrictions(lastNonEOPLU) - } - } - } - // end checks - } - - def checkEndOfParentRestrictionsOnCurrentElement(parentELU: LengthUnits): Unit = { + def checkEndOfParentRestrictionsOnCurrentElement(optParentELU: Option[LengthUnits]): Unit = { val currentElement: ElementBase = this if (currentElement.lengthKind != LengthKind.EndOfParent) {} else { schemaDefinitionWhen( @@ -350,7 +271,7 @@ trait ElementBaseGrammarMixin schemaDefinitionWhen( currentElement.impliedRepresentation == Representation.Text && (!currentElement.isKnownEncoding || !currentElement.knownEncodingIsFixedWidth || currentElement.knownEncodingWidthInBits != 8) - && (parentELU != LengthUnits.Characters), + && (!optParentELU.contains(LengthUnits.Characters)), "element is specified as dfdl:lengthKind=\"endOfParent\", but the element has text representation, and does not have a single-byte character set encoding, and the effective length units of the parent is not 'characters'." ) if (currentElement.isSimpleType) { @@ -362,7 +283,7 @@ trait ElementBaseGrammarMixin && Seq(BinaryNumberRep.Packed, BinaryNumberRep.Bcd, BinaryNumberRep.Ibm4690Packed) .contains(currentElement.binaryNumberRep)) || (currentElement.representation == Representation.Binary - && optionBinaryCalendarRep.isDefined + && currentElement.optionBinaryCalendarRep.isDefined && Seq( BinaryCalendarRep.Packed, BinaryCalendarRep.Bcd, diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/TermGrammarMixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/TermGrammarMixin.scala index 995a97cd87..c12a2b264d 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/TermGrammarMixin.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/TermGrammarMixin.scala @@ -17,9 +17,21 @@ package org.apache.daffodil.core.grammar +import org.apache.daffodil.core.dsom.ChoiceTermBase +import org.apache.daffodil.core.dsom.ElementBase +import org.apache.daffodil.core.dsom.ModelGroup +import org.apache.daffodil.core.dsom.Root +import org.apache.daffodil.core.dsom.SequenceTermBase import org.apache.daffodil.core.dsom.Term import org.apache.daffodil.core.grammar.primitives.MandatoryTextAlignment import org.apache.daffodil.core.runtime1.TermRuntime1Mixin +import org.apache.daffodil.lib.exceptions.Assert +import org.apache.daffodil.lib.schema.annotation.props.gen.ChoiceLengthKind +import org.apache.daffodil.lib.schema.annotation.props.gen.LengthKind +import org.apache.daffodil.lib.schema.annotation.props.gen.LengthUnits +import org.apache.daffodil.lib.schema.annotation.props.gen.SeparatorPosition +import org.apache.daffodil.lib.schema.annotation.props.gen.SequenceKind +import org.apache.daffodil.lib.schema.annotation.props.gen.YesNo ///////////////////////////////////////////////////////////////// // Common to all Terms (Elements and ModelGroups) @@ -84,4 +96,181 @@ trait TermGrammarMixin extends AlignedMixin with BitOrderMixin with TermRuntime1 MandatoryTextAlignment(this, knownEncodingAlignmentInBits, true) } + def myEffectiveLengthUnits(optLastNonEOPLU: Option[LengthUnits]): Option[LengthUnits] = { + val elu = this match { + case e: ElementBase => + e.lengthKind match { + case LengthKind.EndOfParent => optLastNonEOPLU + case LengthKind.Explicit | LengthKind.Prefixed => Some(e.lengthUnits) + case LengthKind.Pattern => Some(LengthUnits.Characters) + case _ => None + } + case c: ChoiceTermBase if c.choiceLengthKind == ChoiceLengthKind.Explicit => + Some(LengthUnits.Bytes) + // the spec doesn't actually account for this case, but logically it makes + // sense that ELU of a sequence is the ELU of its parent that technically is the + // last non-EndOfParent ELU. + case s: SequenceTermBase => optLastNonEOPLU + case _ => None + } + elu + } + + final lazy val childrenEndOfParent: Seq[Term] = LV(Symbol("childrenEndOfParent")) { + val gms = termChildren + val chls = gms.flatMap { + case eb: ElementBase if eb.lengthKind == LengthKind.EndOfParent => Seq(eb) + case eb: ElementBase => Nil + case c: ChoiceTermBase => Nil + case mg: ModelGroup => mg.childrenEndOfParent + } + chls + }.value + + def checkEndOfParentRestrictions(optLastNonEOPLU: Option[LengthUnits]): Unit = { + val term = this + lazy val eopChildren = this.childrenEndOfParent + // checks + term match { + case rootElem: Root if rootElem.lengthKind == LengthKind.EndOfParent => { + rootElem.checkEndOfParentRestrictionsOnCurrentElement(Some(LengthUnits.Characters)) + eopChildren.foreach { + case e: ElementBase => { + rootElem.checkChildrenForSiblingsAfterEOPElement(e) + Assert.invariant( + optLastNonEOPLU.isDefined, + "Effective Length Units of parent should not be None" + ) + e.checkEndOfParentRestrictionsOnCurrentElement(optLastNonEOPLU) + } + case _ => // do nothing + } + } + case parent: ElementBase if eopChildren.nonEmpty => { + eopChildren.foreach { + case e: ElementBase => { + val optParentELU = parent.myEffectiveLengthUnits(optLastNonEOPLU) + parent.lengthKind match { + case LengthKind.Implicit | LengthKind.Delimited => + schemaDefinitionError( + "element is specified as dfdl:lengthKind=\"endOfParent\", but its parent is an element with dfdl:lengthKind 'implicit' or 'delimited'." + ) + case _ => // do nothing + } + parent.checkChildrenForSiblingsAfterEOPElement(e) + Assert.invariant( + optParentELU.isDefined, + "Effective Length Units of parent should not be None" + ) + e.checkEndOfParentRestrictionsOnCurrentElement(optParentELU) + } + case _ => // do nothing + } + } + case s: SequenceTermBase if eopChildren.nonEmpty => { + schemaDefinitionWhen( + s.separatorPosition == SeparatorPosition.Postfix, + "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with dfdl:separatorPosition defined as 'postfix'." + ) + schemaDefinitionWhen( + s.sequenceKind != SequenceKind.Ordered, + "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with dfdl:sequenceKind defined as 'unordered'." + ) + schemaDefinitionWhen( + s.hasTerminator, + "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with a dfdl:terminator." + ) + schemaDefinitionWhen( + s.realElementChildren.exists(e => e.floating == YesNo.Yes), + "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with elements defining dfdl:floating='yes'." + ) + schemaDefinitionWhen( + s.trailingSkip != 0, + "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with a non-zero dfdl:trailingSkip." + ) + eopChildren.foreach { + case e: ElementBase => { + s.checkChildrenForSiblingsAfterEOPElement(e) + e.checkEndOfParentRestrictionsOnCurrentElement(optLastNonEOPLU) + } + case _ => // do nothing + } + } + case c: ChoiceTermBase if eopChildren.nonEmpty => { + // TODO: The DFDL spec (12.3.6) explicitly mentions that an EndOfParent element + // can be terminated by a choice with choiceLengthKind='explicit', with no reference + // to implicit length choices. Later on in 12.3.6, it specified what the parent Effective + // Length Units would be for an explicit length choice, again with no reference to + // implicit length choices. The only way to get an EndOfParent element within an + // implicit length choice would be to have it actually be terminated by the choice's + // parent, similar to how we treat elements within an element that is also EndOfParent. + // The only mention of implicit length choices in the DFDL Spec is to mention SDEs when + // an EndOfParent element is enclosed by a choice with choiceLengthKind='implicit'. We + // think that may be a typo, so for now we disallow ChoiceLengthKind.Implicit + // enclosing an EndOfParent element. + // See Daffodil-3080 + schemaDefinitionWhen( + c.choiceLengthKind == ChoiceLengthKind.Implicit, + "element is specified as dfdl:lengthKind=\"endOfParent\", but its parent is a choice with dfdl:choiceLengthKind 'implicit'." + ) + schemaDefinitionWhen( + c.hasTerminator, + "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a choice with a dfdl:terminator." + ) + schemaDefinitionWhen( + c.trailingSkip != 0, + "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a choice with a non-zero dfdl:trailingSkip." + ) + val optParentELU = c.myEffectiveLengthUnits(optLastNonEOPLU) + eopChildren.foreach { + case e: ElementBase => { + c.checkChildrenForSiblingsAfterEOPElement(e) + Assert.invariant( + optParentELU.isDefined, + "Effective Length Units of parent should not be None" + ) + e.checkEndOfParentRestrictionsOnCurrentElement(optParentELU) + } + case _ => // do nothing + } + } + case _ => // do nothing + } + // end checks + term.termChildren.foreach { e => + lazy val optParentELU = term.myEffectiveLengthUnits(optLastNonEOPLU) + term match { + case parent @ (_: ElementBase | _: ChoiceTermBase | _: SequenceTermBase) => { + e.checkEndOfParentRestrictions(optParentELU) + } + case _ => // do nothing + } + } + } + + def checkChildrenForSiblingsAfterEOPElement(specificChild: ElementBase) = { + lazy val foundPosition = flattenedChildren.indexOf(specificChild) + lazy val lastIndexOfChildren = flattenedChildren.length - 1 + if (flattenedChildren.isEmpty || foundPosition < 0) { + // not found amongst children + Assert.impossible("EndOfParent element not found amongst term children of parent") + } else if (foundPosition != lastIndexOfChildren) { + // get the following children after the EOP element+ 1 + val followingChildrenAfter = + flattenedChildren.slice(foundPosition + 1, lastIndexOfChildren + 1) + followingChildrenAfter.foreach { + case m: ModelGroup => { + specificChild.SDE( + "element is specified as dfdl:lengthKind=\"endOfParent\", but a model group is defined between this element and the end of the enclosing component" + ) + } + case r if r.isRepresented => { + specificChild.SDE( + "element is specified as dfdl:lengthKind=\"endOfParent\", but a represented element is defined between this element and the end of the enclosing component" + ) + } + case _ => // do nothing + } + } + } } diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/EndOfParentTests.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/EndOfParentTests.tdml index 6b2348fee5..47a77e97c6 100644 --- a/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/EndOfParentTests.tdml +++ b/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/EndOfParentTests.tdml @@ -1695,77 +1695,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -1776,7 +1705,7 @@ documentFinalTerminatorCanBeMissing='yes' byteOrder="bigEndian" binaryNumberRep='binary' /> - + @@ -1817,7 +1746,7 @@ - + @@ -1833,6 +1762,64 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + endOfParent + a choice with + choiceLengthKind + implicit + + + + + + + endOfParent + a choice with + choiceLengthKind + implicit + + + @@ -1872,7 +1881,7 @@ - + @@ -1913,7 +1922,7 @@ - + + + + + + + + + + + + + + + + + + + + + @@ -2894,6 +2923,19 @@ + + + hello + + + + hello + + + + @@ -2918,7 +2960,7 @@ - + diff --git a/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindEndOfParent.scala b/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindEndOfParent.scala index 606daefff3..0f5d8071ec 100644 --- a/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindEndOfParent.scala +++ b/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindEndOfParent.scala @@ -64,10 +64,10 @@ class TestLengthKindEndOfParent extends TdmlTests { @Test def TestEndOfParentSimpleTypes10 = test @Test def TestEndOfParentComplexTypes11 = test @Test def TestEndOfParentSimpleTypes11 = test -// @Test def TestEndOfParentComplexTypesRootChoice = test -// @Test def TestEndOfParentSimpleTypesRootChoice = test @Test def TestEndOfParentComplexTypes12 = test @Test def TestEndOfParentSimpleTypes12 = test + @Test def TestEndOfParentComplexTypes12Implicit = test + @Test def TestEndOfParentSimpleTypes12Implicit = test @Test def TestEndOfParentComplexTypes13 = test @Test def TestEndOfParentSimpleTypes13 = test @Test def TestEndOfParentSimpleTypes14 = test @@ -114,6 +114,7 @@ class TestLengthKindEndOfParent extends TdmlTests { @Test def nested_01 = test @Test def nested_02 = test @Test def nested_03 = test + @Test def nested_04 = test @Test def checks_01 = test @Test def checks_02 = test } From 799d9e1ec8edeedb40fbede5bc31ebf947fcfdc6 Mon Sep 17 00:00:00 2001 From: olabusayoT <50379531+olabusayoT@users.noreply.github.com> Date: Thu, 4 Jun 2026 15:03:13 -0400 Subject: [PATCH 4/9] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! - cleanup/refactor code so we call checkEndOfParentRestrictions from RootGrammarMixin and no longer use flattenedChildren instead leverage the recursion we already do down the terms for the sibling after EOP element check - binary booleans are not allowed to be EOP, remove classes and update SDEs/tests - clarify comments for length of EOP elements in AlignedMixin - add EndOfDataPosition instead of bytesTillEndOfDataStream with making it a lazy val in the bucketinginputsource implementations since it shouldn't need to be recalculated each time, but for optEndOfData, it should be calculated each time since position() can change - removed isEndOfParent parameter and the BitLengthFromBitLimitMixin(isEndOfParent) parameterized mixin. Each class that previously accepted true got a new ...EndOfParentLengthParser sibling that mixes in EndOfParentBitLengthMixin. - fix bug with fillBucketToIndex where reffed oldestbucket could be nulled while still being reffed; with test - fix bug with EOP complex elements where we were causing the underlying input source too fill a bunch of buckets for the full length of the complex types even when there were limits on how much we could bucket - update tests Deprecation/Compatibility Worth noting is EndOfParent opens up a potentially interesting technique, if one wishes to ever discard left over data during parsing instead of capturing/warning or erroring out, one could instead make the complex root element EndOfParent, and that would silently discard any left over data. During unparse, a complex EndOfParent root element, will fill in, with the fillByte, any missing data. --- .../apache/daffodil/core/dsom/SchemaSet.scala | 2 - .../org/apache/daffodil/core/dsom/Term.scala | 30 - .../daffodil/core/grammar/AlignedMixin.scala | 10 +- .../grammar/ElementBaseGrammarMixin.scala | 141 +++-- .../grammar/ElementDeclGrammarMixin.scala | 3 + .../core/grammar/TermGrammarMixin.scala | 211 +++---- .../grammar/primitives/PrimitivesBCD.scala | 18 +- .../primitives/PrimitivesBinaryBoolean.scala | 18 - .../primitives/PrimitivesIBM4690Packed.scala | 14 +- .../primitives/PrimitivesLengthKind.scala | 12 +- .../grammar/primitives/PrimitivesPacked.scala | 22 +- .../grammar/primitives/SpecifiedLength.scala | 3 + .../org/apache/daffodil/io/InputSource.scala | 63 +- .../io/InputSourceDataInputStream.scala | 2 +- .../runtime1/processors/BCDParsers.scala | 28 +- .../IBM4690PackedDecimalParsers.scala | 30 +- .../processors/PackedDecimalParsers.scala | 33 +- .../parsers/BinaryBooleanParsers.scala | 9 +- .../parsers/HexBinaryLengthParsers.scala | 11 +- .../processors/parsers/ParserTraits.scala | 80 ++- .../parsers/SpecifiedLengthParsers.scala | 128 +++- .../io/TestDaffodilDataInputSource.scala | 35 ++ .../lengthKind/EndOfParentTests.tdml | 561 +++++++++++++++++- .../TestLengthKindEndOfParent.scala | 15 + 24 files changed, 1094 insertions(+), 385 deletions(-) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/SchemaSet.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/SchemaSet.scala index dcd6a87ee9..18a2483fac 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/SchemaSet.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/SchemaSet.scala @@ -30,7 +30,6 @@ import org.apache.daffodil.lib.iapi.Diagnostic import org.apache.daffodil.lib.iapi.UnitTestSchemaSource import org.apache.daffodil.lib.oolag.OOLAG import org.apache.daffodil.lib.schema.annotation.props.LookupLocation -import org.apache.daffodil.lib.schema.annotation.props.gen.LengthUnits import org.apache.daffodil.lib.util.TransitiveClosure import org.apache.daffodil.lib.xml.* @@ -117,7 +116,6 @@ final class SchemaSet private ( requiredEvaluationsAlways(root) requiredEvaluationsAlways(checkForDuplicateTopLevels()) - requiredEvaluationsAlways(root.checkEndOfParentRestrictions(Some(LengthUnits.Characters))) lazy val resolver = DFDLCatalogResolver.get diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/Term.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/Term.scala index 1172f35d56..e093bbf223 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/Term.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/Term.scala @@ -267,26 +267,6 @@ trait Term .getOrElse(false) } - final lazy val immediatelyEnclosingParentForEOPElem: Option[Term] = { - val p = optLexicalParent.flatMap { - case e: ElementBase => Some(e) - case ge: GlobalElementDecl => Some(ge.asRoot) - case s: SequenceTermBase => s.immediatelyEnclosingParentForEOPElem - case c: ChoiceTermBase => Some(c) - case ct: ComplexTypeBase => { - ct.optLexicalParent.flatMap { - case e: ElementBase => Some(e) - case ge: GlobalElementDecl => Some(ge.asRoot) - case _ => { - None - } - } - } - case _ => None - } - p - } - final lazy val immediatelyEnclosingGroupDef: Option[GroupDefLike] = { optLexicalParent.flatMap { lexicalParent => val res: Option[GroupDefLike] = lexicalParent match { @@ -587,14 +567,4 @@ trait Term case mg: ModelGroup => mg.realElementChildren } } - - lazy val flattenedChildren: IndexedSeq[Term] = termChildren.flatMap { c => - c match { - case eb: ElementBase => IndexedSeq(eb) - case mg: ModelGroup if mg.groupMembers.isEmpty => IndexedSeq(mg) - case s: SequenceTermBase => s.flattenedChildren - case c: ChoiceTermBase => c.flattenedChildren - case _ => Nil - } - }.toIndexedSeq } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/AlignedMixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/AlignedMixin.scala index 9b0942e9b1..721211a71b 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/AlignedMixin.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/AlignedMixin.scala @@ -457,9 +457,13 @@ trait AlignedMixin extends GrammarMixin { self: Term => case LengthKind.Delimited => encodingLengthApprox case LengthKind.Pattern => encodingLengthApprox case LengthKind.EndOfParent => - // technically the alignment of an EndOfParent element would be the - // alignment of its parent minus our current alignment (i.e alignment of - // prior sibs) but since nothing can follow + // Technically, the approximate length of an EndOfParent element is the length + // of its parent minus the length of prior siblings. However, this value is only + // used to compute the ending alignment available to subsequent represented terms. + // Per spec (12.3.6), no represented element or model group may follow an + // EndOfParent element (only unrepresented elements such as IVC elements are + // permitted), so the ending alignment is never consumed. LengthMultipleOf(1) + // is a safe conservative value to return. LengthMultipleOf(1) // If an element is lengthKind="prefixed", the element's length is the length // of the value of the prefix element, which can't be known till runtime diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala index ba759e23f7..0a27f8db80 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala @@ -254,45 +254,55 @@ trait ElementBaseGrammarMixin final lazy val prefixedLengthBody = prefixedLengthElementDecl.parsedValue def checkEndOfParentRestrictionsOnCurrentElement(optParentELU: Option[LengthUnits]): Unit = { + Assert.invariant(optParentELU.isDefined) + val parentELU = optParentELU.get val currentElement: ElementBase = this - if (currentElement.lengthKind != LengthKind.EndOfParent) {} else { - schemaDefinitionWhen( - currentElement.hasTerminator, - "element is specified as dfdl:lengthKind=\"endOfParent\", but specifies a dfdl:terminator." - ) - schemaDefinitionWhen( - currentElement.trailingSkip != 0, - "element is specified as dfdl:lengthKind=\"endOfParent\", but specifies a non-zero dfdl:trailingSkip." - ) - schemaDefinitionWhen( - currentElement.maxOccurs > 1, - "element is specified as dfdl:lengthKind=\"endOfParent\", but specifies a maxOccurs greater than 1." - ) + Assert.invariant(currentElement.lengthKind == LengthKind.EndOfParent) + schemaDefinitionWhen( + currentElement.hasTerminator, + "element is specified as dfdl:lengthKind=\"endOfParent\", but specifies a dfdl:terminator." + ) + schemaDefinitionWhen( + currentElement.trailingSkip != 0, + "element is specified as dfdl:lengthKind=\"endOfParent\", but specifies a non-zero dfdl:trailingSkip." + ) + schemaDefinitionWhen( + currentElement.maxOccurs > 1, + "element is specified as dfdl:lengthKind=\"endOfParent\", but specifies a maxOccurs greater than 1." + ) + schemaDefinitionWhen( + currentElement.impliedRepresentation == Representation.Text + && (!currentElement.isKnownEncoding || !currentElement.knownEncodingIsFixedWidth || currentElement.knownEncodingWidthInBits != 8) + && (parentELU != LengthUnits.Characters), + "element is specified as dfdl:lengthKind=\"endOfParent\", but the element has text representation, and does not have a single-byte character set encoding, and the effective length units of the parent is not 'characters'." + ) + if (currentElement.isSimpleType) { + val meetsSimpleTypeRestrictions = (currentElement.primType eq PrimType.String) + || (currentElement.representation == Representation.Text) + || (currentElement.primType eq PrimType.HexBinary) + || (currentElement.representation == Representation.Binary + && ((currentElement.binaryNumberRep match { + case BinaryNumberRep.Packed => true + case BinaryNumberRep.Bcd => true + case BinaryNumberRep.Ibm4690Packed => true + case _ => false + }) || ( + currentElement.optionBinaryCalendarRep match { + case Some(bcr) => + bcr match { + case BinaryCalendarRep.Packed => true + case BinaryCalendarRep.Bcd => true + case BinaryCalendarRep.Ibm4690Packed => true + case _ => false + } + case None => + false // if no binary calendar rep, then we dont care about what it returns + } + ))) schemaDefinitionWhen( - currentElement.impliedRepresentation == Representation.Text - && (!currentElement.isKnownEncoding || !currentElement.knownEncodingIsFixedWidth || currentElement.knownEncodingWidthInBits != 8) - && (!optParentELU.contains(LengthUnits.Characters)), - "element is specified as dfdl:lengthKind=\"endOfParent\", but the element has text representation, and does not have a single-byte character set encoding, and the effective length units of the parent is not 'characters'." + !meetsSimpleTypeRestrictions, + "element is a simple type specified as dfdl:lengthKind=\"endOfParent\", but isn't a string type, doesn't have text representation, isn't a hexbinary type, or doesn't have binary representation with packed decimal representation." ) - if (currentElement.isSimpleType) { - schemaDefinitionUnless( - (currentElement.primType eq PrimType.String) - || (currentElement.representation == Representation.Text) - || (currentElement.primType eq PrimType.HexBinary) - || (currentElement.representation == Representation.Binary - && Seq(BinaryNumberRep.Packed, BinaryNumberRep.Bcd, BinaryNumberRep.Ibm4690Packed) - .contains(currentElement.binaryNumberRep)) - || (currentElement.representation == Representation.Binary - && currentElement.optionBinaryCalendarRep.isDefined - && Seq( - BinaryCalendarRep.Packed, - BinaryCalendarRep.Bcd, - BinaryCalendarRep.Ibm4690Packed - ) - .contains(currentElement.binaryCalendarRep)), - "element is a simple type specified as dfdl:lengthKind=\"endOfParent\", but isn't a string type, doesn't have text representation, isn't a hexbinary type, or doesn't have binary representation with packed decimal representation." - ) - } } } @@ -648,20 +658,20 @@ trait ElementBaseGrammarMixin } case LengthKind.Pattern => schemaDefinitionError("Binary data elements cannot have lengthKind='pattern'.") + case LengthKind.EndOfParent if optionBinaryCalendarRep.isDefined => + binaryCalendarRep match { + case BinaryCalendarRep.BinaryMilliseconds | BinaryCalendarRep.BinarySeconds => + SDE("lengthKind='endOfParent' only supported for packed binary formats.") + case _ => -1 + } + case LengthKind.EndOfParent if optionBinaryNumberRep.isDefined => + binaryNumberRep match { + case BinaryNumberRep.Binary => + SDE("lengthKind='endOfParent' only supported for packed binary formats.") + case _ => -1 + } case LengthKind.EndOfParent => - // only for packed binary data, length must be computed at runtime. - if ( - representation == Representation.Binary - && optionBinaryCalendarRep.isDefined - && Seq(BinaryCalendarRep.Packed, BinaryCalendarRep.Bcd, BinaryCalendarRep.Ibm4690Packed) - .contains(binaryCalendarRep) - || representation == Representation.Binary - && optionBinaryNumberRep.isDefined - && Seq(BinaryNumberRep.Packed, BinaryNumberRep.Bcd, BinaryNumberRep.Ibm4690Packed) - .contains(binaryNumberRep) - ) -1 - else - SDE("lengthKind='endOfParent' only supported for packed binary formats.") + SDE("lengthKind='endOfParent' only supported for packed binary formats.") } private def explicitBinaryLengthInBits() = { @@ -998,8 +1008,6 @@ trait ElementBaseGrammarMixin byteOrderRaw // must be defined or SDE } (binaryNumberRep, lengthKind, binaryNumberKnownLengthInBits) match { - case (BinaryNumberRep.Binary, LengthKind.EndOfParent, _) => - SDE("lengthKind='endOfParent' is not allowed with binary number representation") case (BinaryNumberRep.Binary, LengthKind.Prefixed, _) => new BinaryIntegerPrefixedLength(this) case (BinaryNumberRep.Binary, _, -1) => new BinaryIntegerRuntimeLength(this) @@ -1163,7 +1171,13 @@ trait ElementBaseGrammarMixin case PrimType.Boolean => { lengthKind match { case LengthKind.Prefixed => new BinaryBooleanPrefixedLength(this) - case LengthKind.EndOfParent => new BinaryBooleanEndOfParentLength(this) + case LengthKind.EndOfParent => + // xs:boolean is not one of the allowed simple types for lengthKind='endOfParent' + // (spec 12.3.6: only xs:string, text representation, xs:hexBinary, or binary + // with packed decimal representation are permitted). + SDE( + "lengthKind='endOfParent' is not supported for xs:boolean with binary representation." + ) case _ => new BinaryBoolean(this) } } @@ -1433,13 +1447,22 @@ trait ElementBaseGrammarMixin // non-explicit lengthKind val body = bodyArg + val eopSimpleTypeElementThatNeedsBitLimit = + (isSimpleType && lengthKind == LengthKind.EndOfParent) + && !this.isInstanceOf[Root] + && impliedRepresentation != Representation.Text + && !isNillable + && primType != PrimType.HexBinary // there are essentially two categories of processors that read/write data input/output - // stream: those that calculate lengths themselves and those that expect another - // processor to calculate the length and set the bit limit which this processor will use as - // the length. The following determines if this element requires another processor to - // calculate and set the bit limit, and if so adds the appropriate grammar to do that + // stream: those that calculate lengths themselves (ex: binary numeric parsers) and those + // that expect another processor to calculate the length and set the bit limit which + // this processor will use as the length (such as text parsers). The following determines + // if this element requires another processor to calculate and set the bit limit, and if so + // adds the appropriate grammar to do that val bodyRequiresSpecifiedLengthBitLimit = lengthKind != LengthKind.Delimited - && !(isSimpleType && lengthKind == LengthKind.EndOfParent && !this.isInstanceOf[Root]) + // Note for non-root EndOfParent simple types, we don't wish to duplicate the length + // calculation efforts unless we know it needs the bit limit set by a parent + && !eopSimpleTypeElementThatNeedsBitLimit && ( isSimpleType && impliedRepresentation == Representation.Text || isSimpleType && isNillable || @@ -1490,9 +1513,9 @@ trait ElementBaseGrammarMixin case LengthKind.Implicit if isSimpleType && impliedRepresentation == Representation.Binary => new SpecifiedLengthImplicit(this, body, implicitBinaryLengthInBits) - case LengthKind.EndOfParent if (isComplexType || this.isInstanceOf[Root]) => + case LengthKind.EndOfParent => new SpecifiedLengthEndOfParent(this, body) - case LengthKind.Delimited | LengthKind.Implicit | LengthKind.EndOfParent => + case LengthKind.Delimited | LengthKind.Implicit => Assert.impossibleCase( "Delimited and ComplexType Implicit cases should not be reached" ) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementDeclGrammarMixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementDeclGrammarMixin.scala index c1c3febadb..0d2733f441 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementDeclGrammarMixin.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementDeclGrammarMixin.scala @@ -19,11 +19,14 @@ package org.apache.daffodil.core.grammar import org.apache.daffodil.core.dsom.Root import org.apache.daffodil.core.grammar.primitives.UnicodeByteOrderMark +import org.apache.daffodil.lib.schema.annotation.props.gen.LengthUnits trait RootGrammarMixin extends LocalElementGrammarMixin // can be repeating if not root { self: Root => + requiredEvaluationsAlways(checkEndOfParentRestrictions(Some(LengthUnits.Characters))) + final lazy val document = prod("document") { schemaDefinitionUnless(isScalar, "The document element cannot be an array.") UnicodeByteOrderMark(this) ~ documentElement diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/TermGrammarMixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/TermGrammarMixin.scala index c12a2b264d..f489adf7a8 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/TermGrammarMixin.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/TermGrammarMixin.scala @@ -25,7 +25,6 @@ import org.apache.daffodil.core.dsom.SequenceTermBase import org.apache.daffodil.core.dsom.Term import org.apache.daffodil.core.grammar.primitives.MandatoryTextAlignment import org.apache.daffodil.core.runtime1.TermRuntime1Mixin -import org.apache.daffodil.lib.exceptions.Assert import org.apache.daffodil.lib.schema.annotation.props.gen.ChoiceLengthKind import org.apache.daffodil.lib.schema.annotation.props.gen.LengthKind import org.apache.daffodil.lib.schema.annotation.props.gen.LengthUnits @@ -96,27 +95,25 @@ trait TermGrammarMixin extends AlignedMixin with BitOrderMixin with TermRuntime1 MandatoryTextAlignment(this, knownEncodingAlignmentInBits, true) } - def myEffectiveLengthUnits(optLastNonEOPLU: Option[LengthUnits]): Option[LengthUnits] = { - val elu = this match { + def optEffectiveLengthUnits(optLastNonEOPELU: Option[LengthUnits]): Option[LengthUnits] = { + this match { case e: ElementBase => e.lengthKind match { - case LengthKind.EndOfParent => optLastNonEOPLU + case LengthKind.EndOfParent => optLastNonEOPELU case LengthKind.Explicit | LengthKind.Prefixed => Some(e.lengthUnits) case LengthKind.Pattern => Some(LengthUnits.Characters) - case _ => None + case LengthKind.Implicit | LengthKind.Delimited => + None // invalid parent; SDE fires separately } case c: ChoiceTermBase if c.choiceLengthKind == ChoiceLengthKind.Explicit => Some(LengthUnits.Bytes) - // the spec doesn't actually account for this case, but logically it makes - // sense that ELU of a sequence is the ELU of its parent that technically is the - // last non-EndOfParent ELU. - case s: SequenceTermBase => optLastNonEOPLU - case _ => None + // Sequences are transparent — the ELU is inherited from the nearest enclosing box. + case _: SequenceTermBase => optLastNonEOPELU + case _: ChoiceTermBase => None // implicit-length choice; SDE fires separately } - elu } - final lazy val childrenEndOfParent: Seq[Term] = LV(Symbol("childrenEndOfParent")) { + final lazy val childrenEndOfParent: Seq[ElementBase] = LV(Symbol("childrenEndOfParent")) { val gms = termChildren val chls = gms.flatMap { case eb: ElementBase if eb.lengthKind == LengthKind.EndOfParent => Seq(eb) @@ -127,74 +124,51 @@ trait TermGrammarMixin extends AlignedMixin with BitOrderMixin with TermRuntime1 chls }.value - def checkEndOfParentRestrictions(optLastNonEOPLU: Option[LengthUnits]): Unit = { + def checkEndOfParentRestrictions(lastNonEOPELU: Option[LengthUnits]): Boolean = { val term = this lazy val eopChildren = this.childrenEndOfParent + lazy val optParentELU = term.optEffectiveLengthUnits(lastNonEOPELU) // checks term match { case rootElem: Root if rootElem.lengthKind == LengthKind.EndOfParent => { rootElem.checkEndOfParentRestrictionsOnCurrentElement(Some(LengthUnits.Characters)) - eopChildren.foreach { - case e: ElementBase => { - rootElem.checkChildrenForSiblingsAfterEOPElement(e) - Assert.invariant( - optLastNonEOPLU.isDefined, - "Effective Length Units of parent should not be None" - ) - e.checkEndOfParentRestrictionsOnCurrentElement(optLastNonEOPLU) - } - case _ => // do nothing + eopChildren.foreach { child => + child.checkEndOfParentRestrictionsOnCurrentElement(lastNonEOPELU) } } - case parent: ElementBase if eopChildren.nonEmpty => { - eopChildren.foreach { - case e: ElementBase => { - val optParentELU = parent.myEffectiveLengthUnits(optLastNonEOPLU) - parent.lengthKind match { - case LengthKind.Implicit | LengthKind.Delimited => - schemaDefinitionError( - "element is specified as dfdl:lengthKind=\"endOfParent\", but its parent is an element with dfdl:lengthKind 'implicit' or 'delimited'." - ) - case _ => // do nothing - } - parent.checkChildrenForSiblingsAfterEOPElement(e) - Assert.invariant( - optParentELU.isDefined, - "Effective Length Units of parent should not be None" - ) - e.checkEndOfParentRestrictionsOnCurrentElement(optParentELU) - } - case _ => // do nothing + case e: ElementBase if eopChildren.nonEmpty => { + eopChildren.foreach { child => + child.schemaDefinitionWhen( + e.lengthKind == LengthKind.Implicit || e.lengthKind == LengthKind.Delimited, + "element is specified as dfdl:lengthKind=\"endOfParent\", but its parent is an element with dfdl:lengthKind 'implicit' or 'delimited'." + ) + child.checkEndOfParentRestrictionsOnCurrentElement(optParentELU) } } case s: SequenceTermBase if eopChildren.nonEmpty => { - schemaDefinitionWhen( - s.separatorPosition == SeparatorPosition.Postfix, - "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with dfdl:separatorPosition defined as 'postfix'." - ) - schemaDefinitionWhen( - s.sequenceKind != SequenceKind.Ordered, - "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with dfdl:sequenceKind defined as 'unordered'." - ) - schemaDefinitionWhen( - s.hasTerminator, - "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with a dfdl:terminator." - ) - schemaDefinitionWhen( - s.realElementChildren.exists(e => e.floating == YesNo.Yes), - "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with elements defining dfdl:floating='yes'." - ) - schemaDefinitionWhen( - s.trailingSkip != 0, - "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with a non-zero dfdl:trailingSkip." - ) - eopChildren.foreach { - case e: ElementBase => { - s.checkChildrenForSiblingsAfterEOPElement(e) - e.checkEndOfParentRestrictionsOnCurrentElement(optLastNonEOPLU) - } - case _ => // do nothing + eopChildren.foreach { child => + child.schemaDefinitionWhen( + s.separatorPosition == SeparatorPosition.Postfix, + "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with dfdl:separatorPosition defined as 'postfix'." + ) + child.schemaDefinitionWhen( + s.sequenceKind != SequenceKind.Ordered, + "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with dfdl:sequenceKind defined as 'unordered'." + ) + child.schemaDefinitionWhen( + s.hasTerminator, + "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with a dfdl:terminator." + ) + child.schemaDefinitionWhen( + s.realElementChildren.exists(e => e.floating == YesNo.Yes), + "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with elements defining dfdl:floating='yes'." + ) + child.schemaDefinitionWhen( + s.trailingSkip != 0, + "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with a non-zero dfdl:trailingSkip." + ) } + } case c: ChoiceTermBase if eopChildren.nonEmpty => { // TODO: The DFDL spec (12.3.6) explicitly mentions that an EndOfParent element @@ -209,68 +183,61 @@ trait TermGrammarMixin extends AlignedMixin with BitOrderMixin with TermRuntime1 // think that may be a typo, so for now we disallow ChoiceLengthKind.Implicit // enclosing an EndOfParent element. // See Daffodil-3080 - schemaDefinitionWhen( - c.choiceLengthKind == ChoiceLengthKind.Implicit, - "element is specified as dfdl:lengthKind=\"endOfParent\", but its parent is a choice with dfdl:choiceLengthKind 'implicit'." - ) - schemaDefinitionWhen( - c.hasTerminator, - "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a choice with a dfdl:terminator." - ) - schemaDefinitionWhen( - c.trailingSkip != 0, - "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a choice with a non-zero dfdl:trailingSkip." - ) - val optParentELU = c.myEffectiveLengthUnits(optLastNonEOPLU) - eopChildren.foreach { - case e: ElementBase => { - c.checkChildrenForSiblingsAfterEOPElement(e) - Assert.invariant( - optParentELU.isDefined, - "Effective Length Units of parent should not be None" - ) - e.checkEndOfParentRestrictionsOnCurrentElement(optParentELU) - } - case _ => // do nothing + eopChildren.foreach { child => + child.schemaDefinitionWhen( + c.choiceLengthKind == ChoiceLengthKind.Implicit, + "element is specified as dfdl:lengthKind=\"endOfParent\", but its parent is a choice with dfdl:choiceLengthKind 'implicit'." + ) + child.schemaDefinitionWhen( + c.hasTerminator, + "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a choice with a dfdl:terminator." + ) + child.schemaDefinitionWhen( + c.trailingSkip != 0, + "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a choice with a non-zero dfdl:trailingSkip." + ) + child.checkEndOfParentRestrictionsOnCurrentElement(optParentELU) } } case _ => // do nothing } // end checks - term.termChildren.foreach { e => - lazy val optParentELU = term.myEffectiveLengthUnits(optLastNonEOPLU) - term match { - case parent @ (_: ElementBase | _: ChoiceTermBase | _: SequenceTermBase) => { - e.checkEndOfParentRestrictions(optParentELU) + val sawEOP = term.termChildren.foldLeft(false) { case (sawEOP, child) => + if (sawEOP) { + // Choice branches are alternatives, not sequential data — the after-EOP SDE must + // not fire across branches. Only sequences and elements have sequential ordering. + term match { + case _: ChoiceTermBase => // has alternatives which can all be EOP; skip + case _ => + child.schemaDefinitionWhen( + child.isInstanceOf[ModelGroup], + "element is specified as dfdl:lengthKind=\"endOfParent\", but a model group is defined between this element and the end of the enclosing component" + ) + child.schemaDefinitionWhen( + child.isRepresented, + "element is specified as dfdl:lengthKind=\"endOfParent\", but a represented element is defined between this element and the end of the enclosing component" + ) } - case _ => // do nothing } - } - } - - def checkChildrenForSiblingsAfterEOPElement(specificChild: ElementBase) = { - lazy val foundPosition = flattenedChildren.indexOf(specificChild) - lazy val lastIndexOfChildren = flattenedChildren.length - 1 - if (flattenedChildren.isEmpty || foundPosition < 0) { - // not found amongst children - Assert.impossible("EndOfParent element not found amongst term children of parent") - } else if (foundPosition != lastIndexOfChildren) { - // get the following children after the EOP element+ 1 - val followingChildrenAfter = - flattenedChildren.slice(foundPosition + 1, lastIndexOfChildren + 1) - followingChildrenAfter.foreach { - case m: ModelGroup => { - specificChild.SDE( - "element is specified as dfdl:lengthKind=\"endOfParent\", but a model group is defined between this element and the end of the enclosing component" - ) - } - case r if r.isRepresented => { - specificChild.SDE( - "element is specified as dfdl:lengthKind=\"endOfParent\", but a represented element is defined between this element and the end of the enclosing component" - ) - } - case _ => // do nothing + val res = child.checkEndOfParentRestrictions(optParentELU) + term match { + // Branches of a choice are alternatives, so should never carry sawEOP from one branch to the next. + case _: ChoiceTermBase => false + case _ => + child match { + case e: ElementBase if e.lengthKind == LengthKind.EndOfParent => true + // A non-EOP element forms a hard length boundary. EOP elements nested inside it are + // scoped to that element's length, so they must not affect sibling scanning here. + case _: ElementBase => false + // An explicit-length choice owns a fixed byte span, so EOP elements inside it are + // scoped to that span. After the choice ends the parent continues normally. + case c: ChoiceTermBase if c.choiceLengthKind == ChoiceLengthKind.Explicit => false + // Sequences are transparent (no own length boundary), + // so EOP state propagates through them. + case _ => res + } } } + sawEOP } } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesBCD.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesBCD.scala index 5dda4397b1..3052f25e4d 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesBCD.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesBCD.scala @@ -20,9 +20,11 @@ package org.apache.daffodil.core.grammar.primitives import org.apache.daffodil.core.dsom.ElementBase import org.apache.daffodil.core.grammar.Terminal import org.apache.daffodil.runtime1.processors.parsers.BCDDecimalBitLimitLengthParser +import org.apache.daffodil.runtime1.processors.parsers.BCDDecimalEndOfParentLengthParser import org.apache.daffodil.runtime1.processors.parsers.BCDDecimalKnownLengthParser import org.apache.daffodil.runtime1.processors.parsers.BCDDecimalRuntimeLengthParser import org.apache.daffodil.runtime1.processors.parsers.BCDIntegerBitLimitLengthParser +import org.apache.daffodil.runtime1.processors.parsers.BCDIntegerEndOfParentLengthParser import org.apache.daffodil.runtime1.processors.parsers.BCDIntegerKnownLengthParser import org.apache.daffodil.runtime1.processors.parsers.BCDIntegerRuntimeLengthParser import org.apache.daffodil.runtime1.processors.unparsers.Unparser @@ -53,7 +55,7 @@ class BCDIntegerKnownLength(val e: ElementBase, lengthInBits: Long) extends Term class BCDIntegerPrefixedLength(val e: ElementBase) extends Terminal(e, true) { override lazy val parser = - new BCDIntegerBitLimitLengthParser(e.elementRuntimeData, isEndOfParent = false) + new BCDIntegerBitLimitLengthParser(e.elementRuntimeData) override lazy val unparser: Unparser = new BCDIntegerMinimumLengthUnparser( e.elementRuntimeData @@ -63,7 +65,7 @@ class BCDIntegerPrefixedLength(val e: ElementBase) extends Terminal(e, true) { class BCDIntegerEndOfParentLength(val e: ElementBase) extends Terminal(e, true) { override lazy val parser = - new BCDIntegerBitLimitLengthParser(e.elementRuntimeData, isEndOfParent = true) + new BCDIntegerEndOfParentLengthParser(e.elementRuntimeData) override lazy val unparser: Unparser = new BCDIntegerMinimumLengthUnparser( e.elementRuntimeData @@ -104,11 +106,7 @@ class BCDDecimalKnownLength(val e: ElementBase, lengthInBits: Long) extends Term class BCDDecimalPrefixedLength(val e: ElementBase) extends Terminal(e, true) { override lazy val parser = - new BCDDecimalBitLimitLengthParser( - e.elementRuntimeData, - e.binaryDecimalVirtualPoint, - isEndOfParent = false - ) + new BCDDecimalBitLimitLengthParser(e.elementRuntimeData, e.binaryDecimalVirtualPoint) override lazy val unparser: Unparser = new BCDDecimalMinimumLengthUnparser( @@ -120,11 +118,7 @@ class BCDDecimalPrefixedLength(val e: ElementBase) extends Terminal(e, true) { class BCDDecimalEndOfParentLength(val e: ElementBase) extends Terminal(e, true) { override lazy val parser = - new BCDDecimalBitLimitLengthParser( - e.elementRuntimeData, - e.binaryDecimalVirtualPoint, - isEndOfParent = true - ) + new BCDDecimalEndOfParentLengthParser(e.elementRuntimeData, e.binaryDecimalVirtualPoint) override lazy val unparser: Unparser = new BCDDecimalMinimumLengthUnparser( diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesBinaryBoolean.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesBinaryBoolean.scala index a5be07b9c1..b934a24fb5 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesBinaryBoolean.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesBinaryBoolean.scala @@ -47,29 +47,11 @@ class BinaryBoolean(val e: ElementBase) extends Terminal(e, true) { class BinaryBooleanPrefixedLength(val e: ElementBase) extends Terminal(e, true) { override lazy val parser = new BinaryBooleanBitLimitLengthParser( - e.elementRuntimeData, - e.binaryBooleanTrueRep, - e.binaryBooleanFalseRep, - e.lengthUnits, - isEndOfParent = false - ) - - override lazy val unparser: Unparser = new BinaryBooleanMinimumLengthUnparser( e.elementRuntimeData, e.binaryBooleanTrueRep, e.binaryBooleanFalseRep, e.lengthUnits ) -} - -class BinaryBooleanEndOfParentLength(val e: ElementBase) extends Terminal(e, true) { - override lazy val parser = new BinaryBooleanBitLimitLengthParser( - e.elementRuntimeData, - e.binaryBooleanTrueRep, - e.binaryBooleanFalseRep, - e.lengthUnits, - isEndOfParent = true - ) override lazy val unparser: Unparser = new BinaryBooleanMinimumLengthUnparser( e.elementRuntimeData, diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesIBM4690Packed.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesIBM4690Packed.scala index 6c04df2801..78332ef333 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesIBM4690Packed.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesIBM4690Packed.scala @@ -20,9 +20,11 @@ package org.apache.daffodil.core.grammar.primitives import org.apache.daffodil.core.dsom.ElementBase import org.apache.daffodil.core.grammar.Terminal import org.apache.daffodil.runtime1.processors.parsers.IBM4690PackedDecimalBitLimitLengthParser +import org.apache.daffodil.runtime1.processors.parsers.IBM4690PackedDecimalEndOfParentLengthParser import org.apache.daffodil.runtime1.processors.parsers.IBM4690PackedDecimalKnownLengthParser import org.apache.daffodil.runtime1.processors.parsers.IBM4690PackedDecimalRuntimeLengthParser import org.apache.daffodil.runtime1.processors.parsers.IBM4690PackedIntegerBitLimitLengthParser +import org.apache.daffodil.runtime1.processors.parsers.IBM4690PackedIntegerEndOfParentLengthParser import org.apache.daffodil.runtime1.processors.parsers.IBM4690PackedIntegerKnownLengthParser import org.apache.daffodil.runtime1.processors.parsers.IBM4690PackedIntegerRuntimeLengthParser import org.apache.daffodil.runtime1.processors.unparsers.Unparser @@ -59,7 +61,7 @@ class IBM4690PackedIntegerKnownLength(val e: ElementBase, lengthInBits: Long) class IBM4690PackedIntegerPrefixedLength(val e: ElementBase) extends Terminal(e, true) { override lazy val parser = - new IBM4690PackedIntegerBitLimitLengthParser(e.elementRuntimeData, isEndOfParent = false) + new IBM4690PackedIntegerBitLimitLengthParser(e.elementRuntimeData) override lazy val unparser: Unparser = new IBM4690PackedIntegerMinimumLengthUnparser( e.elementRuntimeData @@ -68,7 +70,7 @@ class IBM4690PackedIntegerPrefixedLength(val e: ElementBase) extends Terminal(e, class IBM4690PackedIntegerEndOfParentLength(val e: ElementBase) extends Terminal(e, true) { override lazy val parser = - new IBM4690PackedIntegerBitLimitLengthParser(e.elementRuntimeData, isEndOfParent = true) + new IBM4690PackedIntegerEndOfParentLengthParser(e.elementRuntimeData) override lazy val unparser: Unparser = new IBM4690PackedIntegerMinimumLengthUnparser( e.elementRuntimeData @@ -115,8 +117,7 @@ class IBM4690PackedDecimalPrefixedLength(val e: ElementBase) extends Terminal(e, override lazy val parser = new IBM4690PackedDecimalBitLimitLengthParser( e.elementRuntimeData, e.binaryDecimalVirtualPoint, - e.decimalSigned, - isEndOfParent = false + e.decimalSigned ) override lazy val unparser: Unparser = new IBM4690PackedDecimalMinimumLengthUnparser( @@ -127,11 +128,10 @@ class IBM4690PackedDecimalPrefixedLength(val e: ElementBase) extends Terminal(e, } class IBM4690PackedDecimalEndOfParentLength(val e: ElementBase) extends Terminal(e, true) { - override lazy val parser = new IBM4690PackedDecimalBitLimitLengthParser( + override lazy val parser = new IBM4690PackedDecimalEndOfParentLengthParser( e.elementRuntimeData, e.binaryDecimalVirtualPoint, - e.decimalSigned, - isEndOfParent = true + e.decimalSigned ) override lazy val unparser: Unparser = new IBM4690PackedDecimalMinimumLengthUnparser( diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesLengthKind.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesLengthKind.scala index 521e1370b7..3c9ae08786 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesLengthKind.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesLengthKind.scala @@ -34,6 +34,7 @@ import org.apache.daffodil.runtime1.processors.parsers.BCDIntegerDelimitedParser import org.apache.daffodil.runtime1.processors.parsers.BlobSpecifiedLengthParser import org.apache.daffodil.runtime1.processors.parsers.HexBinaryDelimitedParser import org.apache.daffodil.runtime1.processors.parsers.HexBinaryEndOfBitLimitParser +import org.apache.daffodil.runtime1.processors.parsers.HexBinaryEndOfParentParser import org.apache.daffodil.runtime1.processors.parsers.HexBinarySpecifiedLengthParser import org.apache.daffodil.runtime1.processors.parsers.IBM4690PackedDecimalDelimitedParser import org.apache.daffodil.runtime1.processors.parsers.IBM4690PackedIntegerDelimitedParser @@ -174,8 +175,7 @@ case class HexBinaryDelimitedEndOfData(e: ElementBase) extends HexBinaryDelimite case class HexBinaryEndOfBitLimit(e: ElementBase) extends Terminal(e, true) { override lazy val parser: DaffodilParser = new HexBinaryEndOfBitLimitParser( - e.elementRuntimeData, - isEndOfParent = false + e.elementRuntimeData ) override lazy val unparser: DaffodilUnparser = @@ -185,8 +185,7 @@ case class HexBinaryEndOfBitLimit(e: ElementBase) extends Terminal(e, true) { case class HexBinaryLengthPrefixed(e: ElementBase) extends Terminal(e, true) { override lazy val parser: DaffodilParser = new HexBinaryEndOfBitLimitParser( - e.elementRuntimeData, - isEndOfParent = false + e.elementRuntimeData ) override lazy val unparser: DaffodilUnparser = @@ -195,9 +194,8 @@ case class HexBinaryLengthPrefixed(e: ElementBase) extends Terminal(e, true) { case class HexBinaryLengthEndOfParent(e: ElementBase) extends Terminal(e, true) { - override lazy val parser: DaffodilParser = new HexBinaryEndOfBitLimitParser( - e.elementRuntimeData, - isEndOfParent = true + override lazy val parser: DaffodilParser = new HexBinaryEndOfParentParser( + e.elementRuntimeData ) override lazy val unparser: DaffodilUnparser = diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesPacked.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesPacked.scala index bb4b010cd2..80d1280f17 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesPacked.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesPacked.scala @@ -21,9 +21,11 @@ import org.apache.daffodil.core.dsom.ElementBase import org.apache.daffodil.core.grammar.Terminal import org.apache.daffodil.lib.util.PackedSignCodes import org.apache.daffodil.runtime1.processors.parsers.PackedDecimalBitLimitLengthParser +import org.apache.daffodil.runtime1.processors.parsers.PackedDecimalEndOfParentLengthParser import org.apache.daffodil.runtime1.processors.parsers.PackedDecimalKnownLengthParser import org.apache.daffodil.runtime1.processors.parsers.PackedDecimalRuntimeLengthParser import org.apache.daffodil.runtime1.processors.parsers.PackedIntegerBitLimitLengthParser +import org.apache.daffodil.runtime1.processors.parsers.PackedIntegerEndOfParentLengthParser import org.apache.daffodil.runtime1.processors.parsers.PackedIntegerKnownLengthParser import org.apache.daffodil.runtime1.processors.parsers.PackedIntegerRuntimeLengthParser import org.apache.daffodil.runtime1.processors.unparsers.Unparser @@ -78,11 +80,7 @@ class PackedIntegerPrefixedLength( ) extends Terminal(e, true) { override lazy val parser = - new PackedIntegerBitLimitLengthParser( - e.elementRuntimeData, - packedSignCodes, - isEndOfParent = false - ) + new PackedIntegerBitLimitLengthParser(e.elementRuntimeData, packedSignCodes) override lazy val unparser: Unparser = new PackedIntegerMinimumLengthUnparser(e.elementRuntimeData, packedSignCodes) @@ -94,11 +92,7 @@ class PackedIntegerEndOfParentLength( ) extends Terminal(e, true) { override lazy val parser = - new PackedIntegerBitLimitLengthParser( - e.elementRuntimeData, - packedSignCodes, - isEndOfParent = true - ) + new PackedIntegerEndOfParentLengthParser(e.elementRuntimeData, packedSignCodes) override lazy val unparser: Unparser = new PackedIntegerMinimumLengthUnparser(e.elementRuntimeData, packedSignCodes) @@ -155,8 +149,7 @@ class PackedDecimalPrefixedLength(val e: ElementBase, packedSignCodes: PackedSig e.elementRuntimeData, e.binaryDecimalVirtualPoint, packedSignCodes, - e.decimalSigned, - isEndOfParent = false + e.decimalSigned ) override lazy val unparser: Unparser = new PackedDecimalMinimumLengthUnparser( @@ -170,12 +163,11 @@ class PackedDecimalPrefixedLength(val e: ElementBase, packedSignCodes: PackedSig class PackedDecimalEndOfParentLength(val e: ElementBase, packedSignCodes: PackedSignCodes) extends Terminal(e, true) { - override lazy val parser = new PackedDecimalBitLimitLengthParser( + override lazy val parser = new PackedDecimalEndOfParentLengthParser( e.elementRuntimeData, e.binaryDecimalVirtualPoint, packedSignCodes, - e.decimalSigned, - isEndOfParent = true + e.decimalSigned ) override lazy val unparser: Unparser = new PackedDecimalMinimumLengthUnparser( diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/SpecifiedLength.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/SpecifiedLength.scala index 5c2bde7f84..22edb1bccb 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/SpecifiedLength.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/SpecifiedLength.scala @@ -146,6 +146,9 @@ class SpecifiedLengthEndOfParent(e: ElementBase, eGram: => Gram) ) } + // No length-management wrapper needed: the enclosing element's + // unparser handles RightFill/ElementUnused for any space + // the EOP child doesn't fill lazy val unparser: Unparser = eUnparser } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSource.scala b/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSource.scala index 627c48ee2a..f393f337cf 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSource.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSource.scala @@ -182,10 +182,10 @@ abstract class InputSource { def releasePosition(bytePos0b: Long): Unit /** - * Get the number of bytes that are available to be read until the end of - * the data stream. + * Returns the 0-based exclusive byte position of the end of the data stream, + * or None if the end of data is not reachable within the implementation's cache size limit. */ - lazy val bytesTillEndOfDataStream: Long = -1L + def optEndOfDataPosition: Option[Long] /** * Alerts the implementation to attempt to free data that is no longer used, @@ -356,9 +356,19 @@ class BucketingInputSource( bytesFilledInLastBucket = 0 lastBucketIndex += 1 if ((lastBucketIndex - oldestBucketIndex) >= maxNumberOfNonNullBuckets) { - // This frees the oldest bucket, allowing it to be garbage collected. - buckets(oldestBucketIndex) = null - oldestBucketIndex += 1 + if (buckets(oldestBucketIndex).refCount == 0) { + // This frees the oldest bucket, allowing it to be garbage collected. + buckets(oldestBucketIndex) = null + oldestBucketIndex += 1 + } else { + // The oldest bucket has an active backtracking mark. We cannot evict + // it, and we cannot advance oldestBucketIndex past it (that would + // break the invariant checked in releasePosition). Stop filling here + // to prevent the window from growing without bound. Callers that need + // more data (e.g. optEndOfDataPosition) will receive a "goal not + // reached" result and handle the limitation appropriately. + needsMoreData = false + } } } @@ -571,19 +581,25 @@ class BucketingInputSource( oldestBucketIndex = 0 } - override lazy val bytesTillEndOfDataStream: Long = { - val initialBytePosition = position() - val endOfDataNotReached = fillBucketsToIndex(maxNumberOfNonNullBuckets, 8) - if (!endOfDataNotReached) { - val numBytesFilled = totalBytesBucketed - initialBytePosition - // reset the byte position to the initial byte position - position(initialBytePosition) - numBytesFilled + /** + * Returns the 0-based exclusive byte position of the end of the data stream + * by reading and bucketing data until EOF is reached. Does not affect the + * current read position. + * + * The scan window extends [[maxCacheSizeInBytes]] bytes beyond the last + * already-filled bucket (not from the current read position), maximising + * coverage without invalidating backtracking state. + * + * Returns None if EOF is not reached within that window. + */ + override lazy val optEndOfDataPosition: Option[Long] = { + val lastFilledBucketIndex = buckets.length - 1 + val goalBucketIndex = lastFilledBucketIndex + maxNumberOfNonNullBuckets - 1 + val endOfDataNotReached = fillBucketsToIndex(goalBucketIndex, bucketSize) + if (endOfDataNotReached) { + None } else { - position(initialBytePosition) - throw new Exception( - s"Attempted to fill to end of data stream, but did not reach end of data stream before maxCacheSizeInBytes: ${maxCacheSizeInBytes}." - ) + Some(totalBytesBucketed) } } } @@ -668,10 +684,11 @@ class ByteBufferInputSource(byteBuffer: ByteBuffer) extends InputSource { // do nothing. No resources to release. } - override lazy val bytesTillEndOfDataStream: Long = { - val initialPosition = position() - val br = byteBuffer.remaining - position(initialPosition) - br + /** + * Returns the 0-based exclusive byte position of the end of the buffer data. + */ + override def optEndOfDataPosition: Option[Long] = { + val finalPosition = position() + bb.remaining + Some(finalPosition) } } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSourceDataInputStream.scala b/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSourceDataInputStream.scala index 5e1bc1ff5d..d73f854301 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSourceDataInputStream.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSourceDataInputStream.scala @@ -136,7 +136,7 @@ final class InputSourceDataInputStream private (val inputSource: InputSource) */ def hasReachedEndOfData: Boolean = inputSource.hasReachedEndOfData - lazy val bytesTillEndOfDataStream: Long = inputSource.bytesTillEndOfDataStream + def optEndOfDataPosition: Option[Long] = inputSource.optEndOfDataPosition /** * Return the number of currently available bytes. diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/BCDParsers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/BCDParsers.scala index f6c285d04a..d832354c4c 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/BCDParsers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/BCDParsers.scala @@ -50,12 +50,17 @@ class BCDDecimalRuntimeLengthParser( } -class BCDDecimalBitLimitLengthParser( - e: ElementRuntimeData, - binaryDecimalVirtualPoint: Int, - isEndOfParent: Boolean -) extends PackedBinaryDecimalBaseParser(e, binaryDecimalVirtualPoint, None) - with BitLengthFromBitLimitMixin(isEndOfParent) { +class BCDDecimalBitLimitLengthParser(e: ElementRuntimeData, binaryDecimalVirtualPoint: Int) + extends PackedBinaryDecimalBaseParser(e, binaryDecimalVirtualPoint, None) + with BitLengthFromBitLimitMixin { + + override def toNumber(num: Array[Byte]): JBigDecimal = + DecimalUtils.bcdToBigDecimal(num, binaryDecimalVirtualPoint) +} + +class BCDDecimalEndOfParentLengthParser(e: ElementRuntimeData, binaryDecimalVirtualPoint: Int) + extends PackedBinaryDecimalBaseParser(e, binaryDecimalVirtualPoint, None) + with EndOfParentBitLengthMixin { override def toNumber(num: Array[Byte]): JBigDecimal = DecimalUtils.bcdToBigDecimal(num, binaryDecimalVirtualPoint) @@ -80,9 +85,16 @@ class BCDIntegerKnownLengthParser(e: ElementRuntimeData, val lengthInBits: Int) } -class BCDIntegerBitLimitLengthParser(e: ElementRuntimeData, isEndOfParent: Boolean) +class BCDIntegerBitLimitLengthParser(e: ElementRuntimeData) + extends PackedBinaryIntegerBaseParser(e) + with BitLengthFromBitLimitMixin { + + override def toNumber(num: Array[Byte]): JBigInteger = DecimalUtils.bcdToBigInteger(num) +} + +class BCDIntegerEndOfParentLengthParser(e: ElementRuntimeData) extends PackedBinaryIntegerBaseParser(e) - with BitLengthFromBitLimitMixin(isEndOfParent) { + with EndOfParentBitLengthMixin { override def toNumber(num: Array[Byte]): JBigInteger = DecimalUtils.bcdToBigInteger(num) } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/IBM4690PackedDecimalParsers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/IBM4690PackedDecimalParsers.scala index e9761a4349..4c4d58c06e 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/IBM4690PackedDecimalParsers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/IBM4690PackedDecimalParsers.scala @@ -56,10 +56,21 @@ class IBM4690PackedDecimalRuntimeLengthParser( class IBM4690PackedDecimalBitLimitLengthParser( e: ElementRuntimeData, binaryDecimalVirtualPoint: Int, - decimalSigned: YesNo, - isEndOfParent: Boolean + decimalSigned: YesNo +) extends PackedBinaryDecimalBaseParser(e, binaryDecimalVirtualPoint, Some(decimalSigned)) + with BitLengthFromBitLimitMixin { + + override def toNumber(num: Array[Byte]): JBigDecimal = + DecimalUtils.ibm4690ToBigDecimal(num, binaryDecimalVirtualPoint) + +} + +class IBM4690PackedDecimalEndOfParentLengthParser( + e: ElementRuntimeData, + binaryDecimalVirtualPoint: Int, + decimalSigned: YesNo ) extends PackedBinaryDecimalBaseParser(e, binaryDecimalVirtualPoint, Some(decimalSigned)) - with BitLengthFromBitLimitMixin(isEndOfParent) { + with EndOfParentBitLengthMixin { override def toNumber(num: Array[Byte]): JBigDecimal = DecimalUtils.ibm4690ToBigDecimal(num, binaryDecimalVirtualPoint) @@ -89,9 +100,18 @@ class IBM4690PackedIntegerKnownLengthParser( } -class IBM4690PackedIntegerBitLimitLengthParser(e: ElementRuntimeData, isEndOfParent: Boolean) +class IBM4690PackedIntegerBitLimitLengthParser(e: ElementRuntimeData) + extends PackedBinaryIntegerBaseParser(e) + with BitLengthFromBitLimitMixin { + + override def toNumber(num: Array[Byte]): JBigInteger = + DecimalUtils.ibm4690ToBigInteger(num) + +} + +class IBM4690PackedIntegerEndOfParentLengthParser(e: ElementRuntimeData) extends PackedBinaryIntegerBaseParser(e) - with BitLengthFromBitLimitMixin(isEndOfParent) { + with EndOfParentBitLengthMixin { override def toNumber(num: Array[Byte]): JBigInteger = DecimalUtils.ibm4690ToBigInteger(num) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/PackedDecimalParsers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/PackedDecimalParsers.scala index 59779e1a84..823db43d99 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/PackedDecimalParsers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/PackedDecimalParsers.scala @@ -58,10 +58,21 @@ class PackedDecimalBitLimitLengthParser( e: ElementRuntimeData, binaryDecimalVirtualPoint: Int, packedSignCodes: PackedSignCodes, - decimalSigned: YesNo, - isEndOfParent: Boolean + decimalSigned: YesNo +) extends PackedBinaryDecimalBaseParser(e, binaryDecimalVirtualPoint, Some(decimalSigned)) + with BitLengthFromBitLimitMixin { + + override def toNumber(num: Array[Byte]): JBigDecimal = + DecimalUtils.packedToBigDecimal(num, binaryDecimalVirtualPoint, packedSignCodes) +} + +class PackedDecimalEndOfParentLengthParser( + e: ElementRuntimeData, + binaryDecimalVirtualPoint: Int, + packedSignCodes: PackedSignCodes, + decimalSigned: YesNo ) extends PackedBinaryDecimalBaseParser(e, binaryDecimalVirtualPoint, Some(decimalSigned)) - with BitLengthFromBitLimitMixin(isEndOfParent) { + with EndOfParentBitLengthMixin { override def toNumber(num: Array[Byte]): JBigDecimal = DecimalUtils.packedToBigDecimal(num, binaryDecimalVirtualPoint, packedSignCodes) @@ -94,10 +105,20 @@ class PackedIntegerKnownLengthParser( class PackedIntegerBitLimitLengthParser( e: ElementRuntimeData, - packedSignCodes: PackedSignCodes, - isEndOfParent: Boolean + packedSignCodes: PackedSignCodes +) extends PackedBinaryIntegerBaseParser(e) + with BitLengthFromBitLimitMixin { + + override def toNumber(num: Array[Byte]): JBigInteger = + DecimalUtils.packedToBigInteger(num, packedSignCodes) + +} + +class PackedIntegerEndOfParentLengthParser( + e: ElementRuntimeData, + packedSignCodes: PackedSignCodes ) extends PackedBinaryIntegerBaseParser(e) - with BitLengthFromBitLimitMixin(isEndOfParent) { + with EndOfParentBitLengthMixin { override def toNumber(num: Array[Byte]): JBigInteger = DecimalUtils.packedToBigInteger(num, packedSignCodes) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BinaryBooleanParsers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BinaryBooleanParsers.scala index 72584d234a..2df8114f7e 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BinaryBooleanParsers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BinaryBooleanParsers.scala @@ -109,14 +109,11 @@ class BinaryBooleanBitLimitLengthParser( override val context: ElementRuntimeData, binaryBooleanTrueRep: MaybeULong, binaryBooleanFalseRep: ULong, - lengthUnits: LengthUnits, - isEndOfParent: Boolean + lengthUnits: LengthUnits ) extends BinaryBooleanParserBase(binaryBooleanTrueRep, binaryBooleanFalseRep, lengthUnits) - with BitLengthFromBitLimitMixin(isEndOfParent) { + with BitLengthFromBitLimitMixin { override def runtimeDependencies = Vector() - override def getBitLength(state: PState): Int = { - getLengthInBits(state).toInt - } + override def getBitLength(state: PState): Int = super.getBitLengthAsInt(state) } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/HexBinaryLengthParsers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/HexBinaryLengthParsers.scala index 2e496d59df..9d07194c0d 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/HexBinaryLengthParsers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/HexBinaryLengthParsers.scala @@ -83,9 +83,16 @@ final class HexBinarySpecifiedLengthParser(erd: ElementRuntimeData, lengthEv: Le } -final class HexBinaryEndOfBitLimitParser(erd: ElementRuntimeData, isEndOfParent: Boolean) +final class HexBinaryEndOfBitLimitParser(erd: ElementRuntimeData) extends HexBinaryLengthParser(erd), - BitLengthFromBitLimitMixin(isEndOfParent) { + BitLengthFromBitLimitMixin { + + override def runtimeDependencies = Vector() +} + +final class HexBinaryEndOfParentParser(erd: ElementRuntimeData) + extends HexBinaryLengthParser(erd), + EndOfParentBitLengthMixin { override def runtimeDependencies = Vector() } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/ParserTraits.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/ParserTraits.scala index 93feb7d6de..f37e3efc50 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/ParserTraits.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/ParserTraits.scala @@ -21,6 +21,7 @@ import java.lang.Long as JLong import org.apache.daffodil.lib.exceptions.Assert import org.apache.daffodil.lib.schema.annotation.props.gen.LengthUnits +import org.apache.daffodil.lib.util.Maybe.One import org.apache.daffodil.lib.util.MaybeULong import org.apache.daffodil.lib.util.Numbers import org.apache.daffodil.runtime1.infoset.DIComplex @@ -170,25 +171,82 @@ trait PrefixedLengthParserMixin { * Some parsers do not calculate their own length, but instead expect another parser * to set the bit limit, and then they use that bit limit as the length. * An example of this is prefix length parsers. This trait can be used by those - * parsers to do determine the length based on the bitLimit and position. + * parsers to determine the length based on the bitLimit and position. + * + * For dfdl:lengthKind='endOfParent' parsers that need to scan to end-of-stream + * when no bit limit is set, mix in [[EndOfParentBitLengthMixin]] instead. */ -trait BitLengthFromBitLimitMixin(val isEndOfParent: Boolean = false) { +trait BitLengthFromBitLimitMixin { - def getBitLength(s: ParseOrUnparseState): Int = { - val pState = s.asInstanceOf[PState] - val len = getLengthInBits(pState) - len.toInt + def getBitLength(s: ParseOrUnparseState): Int = getBitLengthAsInt(s.asInstanceOf[PState]) + + /** + * getLengthInBits converted to Int with a parse error on overflow. + * + * The default maxCacheSizeInBytes (256 MiB) yields a maximum bit length of + * exactly Int.MaxValue + 1, so a bare .toInt without this guard can overflow + * by one bit on a full-cache EOP element. + */ + def getBitLengthAsInt(pstate: PState): Int = { + val len = getLengthInBits(pstate) + if (pstate.processorStatus ne Success) return 0 + if (len > Int.MaxValue) { + pstate.setFailed( + new ParseError( + One(pstate.schemaFileLocation), + One(pstate.currentLocation), + "Bit length %d exceeds maximum (%d) for this parser.", + len, + Int.MaxValue + ) + ) + 0 + } else { + len.toInt + } } def getLengthInBits(pstate: PState): Long = { if (pstate.bitLimit0b.isDefined) { - val len = pstate.bitLimit0b.get - pstate.bitPos0b - len - } else if (isEndOfParent) { - val byteLimit = pstate.dataInputStream.bytesTillEndOfDataStream - byteLimit * 8 + pstate.bitLimit0b.get - pstate.bitPos0b } else { Assert.invariantFailed("BitLimit not set for parser.") } } } + +/** + * Extends [[BitLengthFromBitLimitMixin]] with dfdl:lengthKind='endOfParent' support. + * When no bit limit is set (i.e., the parser is at the root or outermost EOP element), + * falls back to [[org.apache.daffodil.io.InputSource#optEndOfDataPosition]] to locate + * the end of the data stream. + * + * Only mix this in for parsers that are instantiated specifically for EOP elements. + * Parsers for prefixed or explicitly-bounded elements should use the plain + * [[BitLengthFromBitLimitMixin]]; mixing in this trait by mistake would silently + * consume the entire remaining stream instead of failing at the missing bit limit. + */ +trait EndOfParentBitLengthMixin extends BitLengthFromBitLimitMixin { + + override def getLengthInBits(pstate: PState): Long = { + if (pstate.bitLimit0b.isDefined) { + pstate.bitLimit0b.get - pstate.bitPos0b + } else { + val dis = pstate.dataInputStream + dis.optEndOfDataPosition match { + case Some(endOfDataPosition) => + (endOfDataPosition * 8) - dis.bitPos0b + case None => + pstate.setFailed( + new ParseError( + One(pstate.schemaFileLocation), + One(pstate.currentLocation), + "Cannot determine end-of-data position for dfdl:lengthKind='endOfParent': " + + "data stream exceeds the maximum cache size. Consider increasing maxCacheSizeInBytes." + ) + ) + 0L // callers check pstate.processorStatus before using this value + } + } + } +} diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala index a2fec40539..3f72372c9c 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala @@ -20,6 +20,7 @@ package org.apache.daffodil.runtime1.processors.parsers import java.lang.Long as JLong import java.util.regex.Matcher +import org.apache.daffodil.io.InputSourceDataInputStream import org.apache.daffodil.lib.equality.* import org.apache.daffodil.lib.exceptions.Assert import org.apache.daffodil.lib.schema.annotation.props.gen.LengthUnits @@ -52,29 +53,18 @@ sealed abstract class SpecifiedLengthParserBase(eParser: Parser, erd: RuntimeDat */ protected def getBitLength(s: PState): MaybeULong - final def parse(pState: PState): Unit = { - - val maybeNBits = getBitLength(pState) - - if (pState.processorStatus._ne_(Success)) return - val nBits = maybeNBits.get - val dis = pState.dataInputStream - - val shouldCheckDefinedForLength = erd match { + def parse(pState: PState): Unit = { + lazy val shouldCheckDefinedForLength = erd match { case erd: ElementRuntimeData => !erd.isComplexType case _: ChoiceRuntimeData => false case _ => true } - if (shouldCheckDefinedForLength && !dis.isDefinedForLength(nBits)) { - PENotEnoughBits(pState, nBits, dis) - return - } - - val startingBitPos0b = dis.bitPos0b - dis.withBitLengthLimit(nBits) { - eParser.parse1(pState) - } + val (nBits: Long, dis: InputSourceDataInputStream, startingBitPos0b: Long) = + checkLengthAndParseWithinBitLimits(pState, shouldCheckDefinedForLength) match { + case Some(result) => result + case None => return + } // at this point the recursive parse of the children is finished // so if we're still successful we need to advance the position @@ -87,12 +77,39 @@ sealed abstract class SpecifiedLengthParserBase(eParser: Parser, erd: RuntimeDat // we want to capture the length before we do any skipping // value length of simple types is captured by the eParser if needed - // the SpecifiedLengthParserBase is extended by SpecifiedLengthChoiceParser which should not have its valueLength captured here + // the SpecifiedLengthParserBase is extended by SpecifiedLengthChoiceParser + // which should not have its valueLength captured here if (pState.infoset.isComplex && !erd.isInstanceOf[ChoiceRuntimeData]) captureValueLength(pState, ULong(startingBitPos0b), ULong(dis.bitPos0b)) Assert.invariant(dis eq pState.dataInputStream) val bitsToSkip = finalEndPos0b - dis.bitPos0b + skipBits(pState, bitsToSkip, dis) + } + + protected def checkLengthAndParseWithinBitLimits( + pState: PState, + shouldCheckDefinedForLength: Boolean + ): Option[(Long, InputSourceDataInputStream, Long)] = { + val maybeNBits = getBitLength(pState) + + if (pState.processorStatus._ne_(Success)) return None + val nBits = maybeNBits.get + val dis = pState.dataInputStream + + if (shouldCheckDefinedForLength && !dis.isDefinedForLength(nBits)) { + PENotEnoughBits(pState, nBits, dis) + return None + } + + val startingBitPos0b = dis.bitPos0b + dis.withBitLengthLimit(nBits) { + eParser.parse1(pState) + } + Some((nBits, dis, startingBitPos0b)) + } + + def skipBits(pState: PState, bitsToSkip: Long, dis: InputSourceDataInputStream): Unit = { Assert.invariant( bitsToSkip >= 0 ) // if this is < 0, then the parsing of children went past the limit, which it isn't supposed to. @@ -104,6 +121,7 @@ sealed abstract class SpecifiedLengthParserBase(eParser: Parser, erd: RuntimeDat } } } + } class SpecifiedLengthPatternParser( @@ -136,11 +154,79 @@ class SpecifiedLengthEndOfParentParser( eParser: Parser, erd: ElementRuntimeData ) extends SpecifiedLengthParserBase(eParser, erd), - BitLengthFromBitLimitMixin(true) { + EndOfParentBitLengthMixin { override protected def getBitLength(s: PState): MaybeULong = { - MaybeULong(super[BitLengthFromBitLimitMixin].getBitLength(s)) + MaybeULong(getLengthInBits(s)) + } + + override def parse(pState: PState): Unit = { + val dis = pState.dataInputStream + + if (erd.isComplexType) { + if (dis.bitLimit0b.isDefined) { + val (nBits: Long, dis: InputSourceDataInputStream, startingBitPos0b: Long) = + checkLengthAndParseWithinBitLimits( + pState, + shouldCheckDefinedForLength = false + ) match { + case Some(result) => result + case None => return + } + + // at this point the recursive parse of the children is finished + // so if we're still successful we need to advance the position + // to skip past any bits that the recursive child parse did not + // consume at the end. That is, the specified length can be an + // outer constraint, but the children may not use it all up, leaving + // a section at the end. + if (pState.processorStatus ne Success) return + val finalEndPos0b = startingBitPos0b + nBits + + // we want to capture the length before we do any skipping + // value length of simple types is captured by the eParser + if (pState.infoset.isComplex) + captureValueLength(pState, ULong(startingBitPos0b), ULong(dis.bitPos0b)) + + Assert.invariant(dis eq pState.dataInputStream) + val bitsToSkip = finalEndPos0b - dis.bitPos0b + // if this is < 0, then the parsing of children went past the limit, which it isn't supposed to. + skipBits(pState, bitsToSkip, dis) + } else { + val startingBitPos0b = dis.bitPos0b + + eParser.parse1(pState) + + // at this point the recursive parse of the children is finished + // so if we're still successful we need to advance the position + // to skip past any bits that the recursive child parse did not + // consume at the end. That is, the specified length can be an + // outer constraint, but the children may not use it all up, leaving + // a section at the end. + if (pState.processorStatus ne Success) return + + // we want to capture the length before we do any skipping + // value length of simple types is captured by the eParser + if (pState.infoset.isComplex) { + captureValueLength(pState, ULong(startingBitPos0b), ULong(dis.bitPos0b)) + } + + val maybeNBits = getBitLength(pState) + + if (pState.processorStatus._ne_(Success)) return + val nBits = maybeNBits.get + + if (!dis.isDefinedForLength(nBits)) { + PENotEnoughBits(pState, nBits, dis) + return + } + skipBits(pState, nBits, dis) + } + } else { + super.parse(pState) + } } + } class SpecifiedLengthExplicitParser( diff --git a/daffodil-core/src/test/scala/org/apache/daffodil/io/TestDaffodilDataInputSource.scala b/daffodil-core/src/test/scala/org/apache/daffodil/io/TestDaffodilDataInputSource.scala index b6220d65af..3b8caf80a9 100644 --- a/daffodil-core/src/test/scala/org/apache/daffodil/io/TestDaffodilDataInputSource.scala +++ b/daffodil-core/src/test/scala/org/apache/daffodil/io/TestDaffodilDataInputSource.scala @@ -256,6 +256,41 @@ class TestBucketingInputSource { assertEquals(6, bis.get()) assertEquals(-1, bis.get()) } + + /** + * Regression for: optEndOfDataPosition must not evict a bucket that has an + * active backtracking mark (refCount > 0). + * + * Configuration: bucketSize=2 bytes, maxNumberOfNonNullBuckets=2 (4 bytes total + * cache). Locking position 0 pins bucket 0. When optEndOfDataPosition tries to + * scan beyond the 2-bucket window it would need to evict bucket 0 — which is + * forbidden. The fix stops the scan and returns None rather than nulling the + * locked bucket. + * + * Before fix: fillBucketsToIndex evicted bucket 0 unconditionally; the + * subsequent bis.position(0)/bis.get() threw BacktrackingException. + * After fix: optEndOfDataPosition returns None; the locked position is still + * readable after the scan. + */ + @Test def testOptEndOfDataPositionDoesNotEvictLockedBucket(): Unit = { + // bucketSizeExponent=1 -> bucketSize=2; maxCacheSizeInBytes=4 -> maxNumberOfNonNullBuckets=2 + val tis = new TestInputStream() + tis.setEOF(12) + val bis = new BucketingInputSource(tis, bucketSizeExponent = 1, maxCacheSizeInBytes = 4) + + bis.lockPosition(0) // simulates a choice-parser mark at the start of the stream + + // The oldest bucket (index 0) is locked. fillBucketsToIndex would exceed + // maxNumberOfNonNullBuckets after filling 2 buckets and be unable to evict + // bucket 0, so it stops the scan and returns None. + assertEquals(None, bis.optEndOfDataPosition) + + // Bucket 0 must still be live: backtracking to position 0 must succeed. + bis.position(0) + assertEquals(0, bis.get()) + + bis.releasePosition(0) + } } class TestByteBufferInputSource { diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/EndOfParentTests.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/EndOfParentTests.tdml index 47a77e97c6..1cbec0c1fa 100644 --- a/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/EndOfParentTests.tdml +++ b/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/EndOfParentTests.tdml @@ -41,10 +41,9 @@ - - + + - - + + @@ -120,10 +118,9 @@ - - + + - - + + @@ -211,8 +207,7 @@ - + @@ -1955,6 +1950,245 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + xyzw + A + + + + + + + + + + + + xyzw + A + + + + + + + + + + + + + 1 + 1 + + A + + + + + + + + + + + + abcdef + A + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + X + YZ + COMPUTED + + A + + + + + + + + + + + + + + 1 + 1 + + + 2 + 2 + + COMPUTED + + A + + + + + @@ -2219,6 +2453,21 @@ + + + + + + + + + + + + + + + @@ -2233,8 +2482,7 @@ - + @@ -2576,11 +2824,7 @@ - - @@ -2593,6 +2837,51 @@ + + + + abcdefg + + + + + + abcd + + e + + + + + + + + + abcd + + + + + + abcd + + + + + + + @@ -2754,8 +3043,8 @@ endOfParent - doesn't have - packed decimal representation + not supported + boolean with binary representation @@ -2978,6 +3267,21 @@ + + + + + + + + + + + + + + + + + helloA + + + + helloA + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0111 + 10101011 + + + + + + 7 + AB + + + + + + + + + + 0111 + 10101011 1010 + + + + + + 7 + 101010111010 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + AA BB CC DD EE + + + + + + AABBCCDDEE + + + + + + + + + + AA BB 00 00 00 + + + + + + AABB + + + + + + diff --git a/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindEndOfParent.scala b/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindEndOfParent.scala index 0f5d8071ec..cd72b674b9 100644 --- a/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindEndOfParent.scala +++ b/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindEndOfParent.scala @@ -70,6 +70,12 @@ class TestLengthKindEndOfParent extends TdmlTests { @Test def TestEndOfParentSimpleTypes12Implicit = test @Test def TestEndOfParentComplexTypes13 = test @Test def TestEndOfParentSimpleTypes13 = test + @Test def eopBranchFirst_ST_eopBranch = test + @Test def eopBranchFirst_ST_fixedBranch = test + @Test def eopBranchFirst_CT_eopBranch = test + @Test def eopBranchFirst_CT_fixedBranch = test + @Test def eopFollowedByIVC_ST = test + @Test def eopFollowedByIVC_CT = test @Test def TestEndOfParentSimpleTypes14 = test @Test def TestEndOfParentSimpleTypes16 = test @Test def TestEndOfParentSimpleTypes17 = test @@ -98,6 +104,8 @@ class TestLengthKindEndOfParent extends TdmlTests { @Test def text_bool_txt_bytes = test @Test def text_bool_txt_bits = test @Test def text_bool_txt_chars = test + @Test def check_eop_simple_elem_01 = test + @Test def check_eop_simple_elem_02 = test @Test def bin_int_bin_bytes_packed = test @Test def bin_int_bin_bytes_bcd = test @Test def bin_int_bin_bytes_ibm4690 = test @@ -117,4 +125,11 @@ class TestLengthKindEndOfParent extends TdmlTests { @Test def nested_04 = test @Test def checks_01 = test @Test def checks_02 = test + @Test def checks_03 = test + + @Test def eop_not_byte_aligned_01 = test + @Test def eop_not_byte_aligned_02 = test + + @Test def eop_unparse_fill_exact = test + @Test def eop_unparse_fill_short = test } From 78a824e96c612ca432bdb2eed659447ff07483b0 Mon Sep 17 00:00:00 2001 From: olabusayoT <50379531+olabusayoT@users.noreply.github.com> Date: Mon, 8 Jun 2026 14:27:53 -0400 Subject: [PATCH 5/9] Refactor packed/binary representation checks in `ElementBaseGrammarMixin` and update equality operator in `SpecifiedLengthParsers`. --- .../grammar/ElementBaseGrammarMixin.scala | 42 ++++++++++--------- .../parsers/SpecifiedLengthParsers.scala | 4 +- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala index 0a27f8db80..1a8fbcfbfc 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala @@ -281,24 +281,25 @@ trait ElementBaseGrammarMixin || (currentElement.representation == Representation.Text) || (currentElement.primType eq PrimType.HexBinary) || (currentElement.representation == Representation.Binary - && ((currentElement.binaryNumberRep match { - case BinaryNumberRep.Packed => true - case BinaryNumberRep.Bcd => true - case BinaryNumberRep.Ibm4690Packed => true + && (( + currentElement.binaryNumberRep, + currentElement.optionBinaryCalendarRep + ) match { + case ( + BinaryNumberRep.Packed | BinaryNumberRep.Bcd | BinaryNumberRep.Ibm4690Packed, + _ + ) => + true + case ( + _, + Some( + BinaryCalendarRep.Packed | BinaryCalendarRep.Bcd | + BinaryCalendarRep.Ibm4690Packed + ) + ) => + true case _ => false - }) || ( - currentElement.optionBinaryCalendarRep match { - case Some(bcr) => - bcr match { - case BinaryCalendarRep.Packed => true - case BinaryCalendarRep.Bcd => true - case BinaryCalendarRep.Ibm4690Packed => true - case _ => false - } - case None => - false // if no binary calendar rep, then we dont care about what it returns - } - ))) + })) schemaDefinitionWhen( !meetsSimpleTypeRestrictions, "element is a simple type specified as dfdl:lengthKind=\"endOfParent\", but isn't a string type, doesn't have text representation, isn't a hexbinary type, or doesn't have binary representation with packed decimal representation." @@ -629,6 +630,7 @@ trait ElementBaseGrammarMixin ) } + private lazy val unsupportedLengthKindEOPMessage = "lengthKind='endOfParent' only supported for packed binary formats." /** * Property consistency check for called for all binary numbers * @@ -661,17 +663,17 @@ trait ElementBaseGrammarMixin case LengthKind.EndOfParent if optionBinaryCalendarRep.isDefined => binaryCalendarRep match { case BinaryCalendarRep.BinaryMilliseconds | BinaryCalendarRep.BinarySeconds => - SDE("lengthKind='endOfParent' only supported for packed binary formats.") + SDE(unsupportedLengthKindEOPMessage) case _ => -1 } case LengthKind.EndOfParent if optionBinaryNumberRep.isDefined => binaryNumberRep match { case BinaryNumberRep.Binary => - SDE("lengthKind='endOfParent' only supported for packed binary formats.") + SDE(unsupportedLengthKindEOPMessage) case _ => -1 } case LengthKind.EndOfParent => - SDE("lengthKind='endOfParent' only supported for packed binary formats.") + SDE(unsupportedLengthKindEOPMessage) } private def explicitBinaryLengthInBits() = { diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala index 3f72372c9c..ec6baece91 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala @@ -93,7 +93,7 @@ sealed abstract class SpecifiedLengthParserBase(eParser: Parser, erd: RuntimeDat ): Option[(Long, InputSourceDataInputStream, Long)] = { val maybeNBits = getBitLength(pState) - if (pState.processorStatus._ne_(Success)) return None + if (pState.processorStatus ne Success) return None val nBits = maybeNBits.get val dis = pState.dataInputStream @@ -213,7 +213,7 @@ class SpecifiedLengthEndOfParentParser( val maybeNBits = getBitLength(pState) - if (pState.processorStatus._ne_(Success)) return + if (pState.processorStatus ne Success) return val nBits = maybeNBits.get if (!dis.isDefinedForLength(nBits)) { From 2d40ca0c5f0091cc9ebf60ea816deed79ad22ef7 Mon Sep 17 00:00:00 2001 From: olabusayoT <50379531+olabusayoT@users.noreply.github.com> Date: Tue, 9 Jun 2026 11:00:17 -0400 Subject: [PATCH 6/9] fixup! Refactor packed/binary representation checks in `ElementBaseGrammarMixin` and update equality operator in `SpecifiedLengthParsers`. - format and remove unsued inputs --- .../daffodil/core/grammar/ElementBaseGrammarMixin.scala | 4 +++- .../runtime1/processors/parsers/SpecifiedLengthParsers.scala | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala index 1a8fbcfbfc..1762f38f7e 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala @@ -630,7 +630,9 @@ trait ElementBaseGrammarMixin ) } - private lazy val unsupportedLengthKindEOPMessage = "lengthKind='endOfParent' only supported for packed binary formats." + private lazy val unsupportedLengthKindEOPMessage = + "lengthKind='endOfParent' only supported for packed binary formats." + /** * Property consistency check for called for all binary numbers * diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala index ec6baece91..9da3209a69 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala @@ -21,7 +21,6 @@ import java.lang.Long as JLong import java.util.regex.Matcher import org.apache.daffodil.io.InputSourceDataInputStream -import org.apache.daffodil.lib.equality.* import org.apache.daffodil.lib.exceptions.Assert import org.apache.daffodil.lib.schema.annotation.props.gen.LengthUnits import org.apache.daffodil.lib.util.MaybeULong From 72fdd8ee3a7e4d99a2f2b61ab41cd7efaa2147b1 Mon Sep 17 00:00:00 2001 From: olabusayoT <50379531+olabusayoT@users.noreply.github.com> Date: Thu, 11 Jun 2026 09:06:46 -0400 Subject: [PATCH 7/9] Implement EOP unparse padding to enclosing explicit-length box - Add EndOfParent to the `isMinLengthKind` condition so EOP text elements are padded on unparse; rename the predicate from `isDelimitedPrefixedPattern` to `isMinLengthKind` (the old name was misleading once EOP was included). - Add `eopElementInfoMap`, which carries a `(suppressesPadding: Boolean, Option[UnparseTargetLengthInBitsEv])` tuple per EOP element. The Option holds a target-length ev when the EOP element is inside a constant-length explicit element or an explicit-length choice, so the element pads to the full box rather than only to `minLength`. - In `maybeUnparseMinOrTargetLengthInBitsEv`, return `One(boxEv)` when the info map supplies a box ev for an EOP element. - Fix `ChoiceUnusedUnparserSuspendableOperation.continuation()`: The old formula read the wrong DOS when `AlignmentFillUnparserSuspendableOperation` introduced an extra intermediate DOS split inside the branch. - Fix length units for delimited/pattern padding: `minLenUnits` now always returns `LengthUnits.Characters` for delimited and pattern length kinds, matching spec 13.2. Previously `dfdl:lengthUnits` (bytes/bits) was used, producing under-padded output with multi-byte encodings. - Tests: unparse/round-trip for EOP-in-choice-box padding; parse/unparse for 12.3.2.1 delimiter scanning suspension; round-trips for 9.3.2 zero-length EOP at parent boundary (string and hexBinary). Deprecation/Compatibility - schemas using dfdl:textPadKind="padChar" with lengthKind "delimited" or "pattern" and a multi-byte encoding (ex UTF-16) will now produce correctly character-counted padding instead of byte-counted padding. DAFFODIL-238 --- .../core/dsom/RuntimePropertyMixins.scala | 24 +- .../grammar/ElementBaseGrammarMixin.scala | 44 +- .../core/grammar/TermGrammarMixin.scala | 69 +++ .../unparsers/runtime1/SpecifiedLength2.scala | 19 +- .../section12/lengthKind/DelimitedTests.tdml | 218 +++++++ .../lengthKind/EndOfParentTests.tdml | 556 ++++++++++++++++++ .../section12/lengthKind/PatternTests.tdml | 175 ++++++ .../lengthKind/TestLengthKindDelimited.scala | 9 + .../TestLengthKindEndOfParent.scala | 19 + .../lengthKind/TestLengthKindPattern.scala | 8 + 10 files changed, 1122 insertions(+), 19 deletions(-) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/RuntimePropertyMixins.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/RuntimePropertyMixins.scala index 4145d8fed5..f409ec5b01 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/RuntimePropertyMixins.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/RuntimePropertyMixins.scala @@ -449,10 +449,12 @@ trait ElementRuntimeValuedPropertiesMixin case (Explicit, Text, AnySimpleType) => (lengthUnits, 0L) case (Prefixed, _, String) => (lengthUnits, minLengthLong) case (Prefixed, Text, _) => (lengthUnits, textOutputMinLength) - case (Pattern, _, String) => (lengthUnits, minLengthLong) - case (Pattern, Text, _) => (lengthUnits, textOutputMinLength) - case (Delimited, _, String) => (lengthUnits, minLengthLong) - case (Delimited, Text, _) => (lengthUnits, textOutputMinLength) + case (Pattern, _, String) => (LengthUnits.Characters, minLengthLong) + case (Pattern, Text, _) => (LengthUnits.Characters, textOutputMinLength) + case (Delimited, _, String) => (LengthUnits.Characters, minLengthLong) + case (Delimited, Text, _) => (LengthUnits.Characters, textOutputMinLength) + case (EndOfParent, _, String) => (LengthUnits.Characters, minLengthLong) + case (EndOfParent, Text, _) => (LengthUnits.Characters, textOutputMinLength) case _ => (LengthUnits.Bits, 0L) // anything else. This shuts off checking a min. } res @@ -521,10 +523,18 @@ trait ElementRuntimeValuedPropertiesMixin ((repElement.lengthKind._eq_(LengthKind.Implicit)) && repElement.isSimpleType) ) { repElement.maybeUnparseTargetLengthInBitsEv - } else if (this.isDelimitedPrefixedPatternWithPadding) { + } else if (this.lengthKind == LengthKind.EndOfParent) { + // Per spec 12.3.6/13.2: EOP elements pad to the full "available length" of the + // enclosing box (constant-length element or explicit-length choice) when one is known. + // Fall back to minLength only when no constant box is found. + schemaSet.root.eopElementInfoMap.get(this).flatMap(_._2) match { + case Some(boxEv) => One(boxEv) + case None => if (isMinLengthKindWithPadding) One(minLengthInBitsEv) else Nope + } + } else if (this.isMinLengthKindWithPadding) { // - // if delimited but there is a min length, we just need the min - // length. There is no target length other than it. + // if delimited/pattern/prefixed but there is a min length, + // we just need the min length. There is no target length other than it. // val ev = minLengthInBitsEv One(ev) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala index 1762f38f7e..cf0dd2e81f 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala @@ -72,13 +72,19 @@ trait ElementBaseGrammarMixin lazy val isPrefixed: Boolean = lengthKind == LengthKind.Prefixed - protected lazy val isDelimitedPrefixedPattern: Boolean = { + /** + * True for the length kinds where padding uses a minimum target length + * (textOutputMinLength or xs:minLength) rather than a fixed target length. + * These are: Delimited, Pattern, Prefixed, and EndOfParent. + */ + protected lazy val isMinLengthKind: Boolean = { import LengthKind.* lengthKind match { case Delimited => true // don't test for hasDelimiters because it might not be our delimiter, but a surrounding group's separator, or it's terminator, etc. case Pattern => true case Prefixed => true + case EndOfParent => true case _ => false } } @@ -97,13 +103,37 @@ trait ElementBaseGrammarMixin } /** - * true if padding will be inserted for this delimited element when unparsing. + * True when this element is explicit-length with a runtime-evaluated dfdl:length + * expression. Per DFDL spec 12.3.6, EOP children of such a parent must not receive + * padding grammar pieces on unparse; padding only applies when the parent has a + * constant dfdl:length. + */ + lazy val suppressesEOPChildPadding: Boolean = + lengthKind == LengthKind.Explicit && !lengthEv.isConstant + + /** + * True when this EOP element's nearest enclosing explicit-length parent uses a + * runtime expression for dfdl:length. Queries the root-level eopElementInfoMap + * so the rule is established top-down from the root. + * Per spec 12.3.6, padding is only valid when the parent has constant dfdl:length. + */ + private lazy val isEOPInsideExpressionLengthParent: Boolean = + (lengthKind eq LengthKind.EndOfParent) && + schemaSet.root.eopElementInfoMap.get(this).exists(_._1) + + /** + * True when this element requires min length padding to be inserted on + * unparse i.e the length kind is in the min length group, representation is + * text, justification is not none, and the min length is positive. + * Excludes EOP elements whose parent uses a runtime-expression dfdl:length + * (spec 12.3.6: padding is only valid when parent has constant dfdl:length). */ - protected lazy val isDelimitedPrefixedPatternWithPadding = { - (isDelimitedPrefixedPattern && + protected lazy val isMinLengthKindWithPadding = { + (isMinLengthKind && (impliedRepresentation eq Representation.Text) && (justificationPad ne TextJustificationType.None) && - minLen > 0) + minLen > 0 && + !isEOPInsideExpressionLengthParent) } private lazy val prefixLengthTypeGSTD = LV(Symbol("prefixLengthTypeGSTD")) { @@ -323,7 +353,7 @@ trait ElementBaseGrammarMixin (lengthUnits eq LengthUnits.Characters) ) maybeUnparseTargetLengthInBitsEv.isDefined // "pad" complex types. - else if (isDelimitedPrefixedPatternWithPadding) + else if (isMinLengthKindWithPadding) true // simple type for unparse that needs to be padded. else // simple type, specified length. @@ -364,7 +394,7 @@ trait ElementBaseGrammarMixin // * This is about whether the box we're fitting it into is fixed or varying size. // */ // final protected lazy val isVariableLengthRep: Boolean = { - // isDelimitedPrefixedPattern || + // isMinLengthKind || // ((lengthKind eq LengthKind.Explicit) && // !lengthEv.optConstant.isDefined) // // diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/TermGrammarMixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/TermGrammarMixin.scala index f489adf7a8..c74b6178de 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/TermGrammarMixin.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/TermGrammarMixin.scala @@ -31,6 +31,11 @@ import org.apache.daffodil.lib.schema.annotation.props.gen.LengthUnits import org.apache.daffodil.lib.schema.annotation.props.gen.SeparatorPosition import org.apache.daffodil.lib.schema.annotation.props.gen.SequenceKind import org.apache.daffodil.lib.schema.annotation.props.gen.YesNo +import org.apache.daffodil.lib.util.Maybe.Nope +import org.apache.daffodil.runtime1.processors.ImplicitLengthEv +import org.apache.daffodil.runtime1.processors.LengthInBitsEv +import org.apache.daffodil.runtime1.processors.MinLengthInBitsEv +import org.apache.daffodil.runtime1.processors.UnparseTargetLengthInBitsEv ///////////////////////////////////////////////////////////////// // Common to all Terms (Elements and ModelGroups) @@ -113,6 +118,70 @@ trait TermGrammarMixin extends AlignedMixin with BitOrderMixin with TermRuntime1 } } + /** + * Top-down map from each EOP element to a tuple of: + * _1: Boolean - whether padding is suppressed (parent has expression-based dfdl:length) + * _2: Option[UnparseTargetLengthInBitsEv] - the box target ev when the EOP is inside a + * constant-length element or explicit-length choice; None otherwise. + * + * Built in one pass from this term's descendant tree. Call on the schema root + * to get the complete picture for all EOP elements in the schema. + */ + final lazy val eopElementInfoMap + : Map[ElementBase, (Boolean, Option[UnparseTargetLengthInBitsEv])] = { + type Info = (Boolean, Option[UnparseTargetLengthInBitsEv]) + + def elementInfo(e: ElementBase): (Boolean, Option[UnparseTargetLengthInBitsEv]) = { + val suppresses = e.suppressesEOPChildPadding + val optBoxEv: Option[UnparseTargetLengthInBitsEv] = + if (e.lengthKind == LengthKind.Explicit && e.lengthEv.isConstant) + Some(e.unparseTargetLengthInBitsEv) + else + None + (suppresses, optBoxEv) + } + + def choiceBoxEv(c: ChoiceTermBase, eop: ElementBase): UnparseTargetLengthInBitsEv = { + // we use ImplicitLengthEv because it exists for lengths that are + // schema-compile-time constants with no expression to evaluate. + val constLenEv = new ImplicitLengthEv(c.choiceLength.toLong, eop.eci) + constLenEv.compile(eop.tunable) + // dfdl:choiceLength is specified by the DFDL spec to always be in bytes + val lenInBitsEv = + new LengthInBitsEv(LengthUnits.Bytes, LengthKind.Explicit, Nope, constLenEv, eop.eci) + lenInBitsEv.compile(eop.tunable) + val zeroMinEv = + new MinLengthInBitsEv(LengthUnits.Bytes, LengthKind.Explicit, Nope, 0L, eop.eci) + zeroMinEv.compile(eop.tunable) + val boxEv = new UnparseTargetLengthInBitsEv(lenInBitsEv, zeroMinEv, eop.eci) + boxEv.compile(eop.tunable) + boxEv + } + + def collectFromTerms(terms: Seq[Term]): Seq[(ElementBase, Info)] = + terms.flatMap { + case e: ElementBase if e.lengthKind == LengthKind.EndOfParent => + Nil // EOP child; its enclosing parent/choice already added the entry + case e: ElementBase => + val info = elementInfo(e) + e.childrenEndOfParent.map(_ -> info) ++ collectFromTerms(e.termChildren) + case c: ChoiceTermBase if c.choiceLengthKind == ChoiceLengthKind.Explicit => + val eops = c.childrenEndOfParent + val entries = eops.map(eop => eop -> (false, Some(choiceBoxEv(c, eop)))) + entries ++ collectFromTerms(c.termChildren) + case other => + collectFromTerms(other.termChildren) + } + + this match { + case e: ElementBase => + val info = elementInfo(e) + (e.childrenEndOfParent.map(_ -> info) ++ collectFromTerms(e.termChildren)).toMap + case other => + collectFromTerms(other.termChildren).toMap + } + } + final lazy val childrenEndOfParent: Seq[ElementBase] = LV(Symbol("childrenEndOfParent")) { val gms = termChildren val chls = gms.flatMap { diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/SpecifiedLength2.scala b/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/SpecifiedLength2.scala index 6974e3f3f7..bec7480478 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/SpecifiedLength2.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/unparsers/runtime1/SpecifiedLength2.scala @@ -556,14 +556,23 @@ class ChoiceUnusedUnparserSuspendableOperation( } /** - * determine delta between value length and target length + * Fills the unused tail of the choice box with skip bits. * - * and skip that many bits. + * [[dosToCheck_]] is the DOS chain bracketing the branch content, from the + * RegionSplitUnparser capture at entry to the one at exit. Each DOS in the + * chain either died (was collapsed into its successor via setFinished / + * deliverBufferContent), freezing its relBitPos0b at the bits it had absorbed + * at death, or is still live (dosForEnd), accumulating any remaining writes. + * + * Because deliverBufferContent only ever adds to the absorbing DOS, the DOS + * that was made direct most recently always holds the largest relBitPos0b, + * which equals the true total branch length in bits. Taking the max across + * the chain is therefore correct regardless of how many intermediate DOS + * splits were introduced by suspensions inside the branch (e.g. + * AlignmentFillUnparserSuspendableOperation, OnlyPaddingUnparserSuspendableOperation). */ override def continuation(ustate: UState): Unit = { - val startPos0b = dosToCheck_(0).relBitPos0b - val endPos0b = dosToCheck_.last.relBitPos0b + startPos0b - val vl = (endPos0b - startPos0b).toLong + val vl = dosToCheck_.map(_.relBitPos0b.toLong).max val skipInBits = targetLengthInBits - vl if (skipInBits < 0) UE( diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/DelimitedTests.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/DelimitedTests.tdml index c2f55a0d04..77e998562a 100644 --- a/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/DelimitedTests.tdml +++ b/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/DelimitedTests.tdml @@ -1135,4 +1135,222 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + hi + after + + + + + + + + + + + + abc + rest + + + + + + + + + + + + abcde + rest + + + + + + + + + + + + 42 + rest + + + + + + + + + 00 5F 00 5F 00 5F 00 68 00 69 00 7C + + + + + hi + + + + + + + + + 00 5F 00 5F 00 5F 00 68 00 69 00 7C + + + + + hi + + + + + + + + + 00 61 00 62 00 63 00 64 00 65 00 7C + + + + + abcde + + + + + diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/EndOfParentTests.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/EndOfParentTests.tdml index 1cbec0c1fa..39e1a99b17 100644 --- a/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/EndOfParentTests.tdml +++ b/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/EndOfParentTests.tdml @@ -3512,4 +3512,560 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ___hi + + + + + + hi + + + + + + + + + + __abc + + + + + + abc + + + + + + + + + + abc + + + + + + abc + + + + + + + + + + 42 + + + + + + 42 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 02 34 32 + + + + + 2 + + 42 + + + + + + + + + + 02 34 32 + + + + + 2 + + 42 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ab + + + + + ab + + + + + + + + + ab + + + + + ab + + + + + + + + + ab + + + + + ab + + + + + + + + + ab + + + + + ab + + + + + + + + + + + + + + + + + + + + + + + + + + abcde|fg| + + + + + abc + de|fg + + + + + + + + + abcde|fg| + + + + + abc + de|fg + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + hello + + + + + hello + + + + + + + + + + DE AD BE EF + + + + + DEADBEEF + + + + + + diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/PatternTests.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/PatternTests.tdml index d79649ac30..654e3beedc 100644 --- a/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/PatternTests.tdml +++ b/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/PatternTests.tdml @@ -999,6 +999,181 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + hi + after + + + + + + + + + + + abc + rest + + + + + + + + + + + + abcde + rest + + + + + + + + + 00 5F 00 5F 00 5F 00 68 00 69 00 7C + + + + + hi + + + + + + + + + 00 5F 00 5F 00 5F 00 68 00 69 00 7C + + + + + hi + + + + + + + + + 00 61 00 62 00 63 00 64 00 65 00 7C + + + + + abcde + + + + diff --git a/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindDelimited.scala b/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindDelimited.scala index 7f1b4ef80e..bc695344ec 100644 --- a/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindDelimited.scala +++ b/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindDelimited.scala @@ -78,6 +78,15 @@ class TestLengthKindDelimited extends TdmlTests { @Test def Lesson1_lengthKind_delimited = test @Test def Lesson4_delimited_fixed_length = test @Test def delimited_construct = test + + @Test def delim_pad_str_right_unparse = test + @Test def delim_pad_str_right_roundtrip = test + @Test def delim_pad_str_no_padding_needed = test + @Test def delim_pad_num_right_roundtrip = test + + @Test def delim_pad_utf16_str_right_unparse = test + @Test def delim_pad_utf16_str_right_roundtrip = test + @Test def delim_pad_utf16_str_no_padding_needed = test } class TestLengthKindDelimitedAB extends TdmlTests { diff --git a/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindEndOfParent.scala b/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindEndOfParent.scala index cd72b674b9..4cf4d9c712 100644 --- a/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindEndOfParent.scala +++ b/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindEndOfParent.scala @@ -132,4 +132,23 @@ class TestLengthKindEndOfParent extends TdmlTests { @Test def eop_unparse_fill_exact = test @Test def eop_unparse_fill_short = test + + @Test def eop_text_pad_str_right_unparse = test + @Test def eop_text_pad_str_right_roundtrip = test + @Test def eop_text_pad_str_left_roundtrip = test + @Test def eop_text_pad_num_right_roundtrip = test + + @Test def eop_no_pad_when_expr_parent_unparse = test + @Test def eop_no_pad_when_expr_parent_roundtrip = test + + @Test def eop_pad_to_parent_box_01 = test + @Test def eop_pad_to_parent_box_02 = test + @Test def eop_pad_to_choice_box_01 = test + @Test def eop_pad_to_choice_box_02 = test + + @Test def eop_delim_scan_suspended_01 = test + @Test def eop_delim_scan_suspended_02 = test + + @Test def eop_zero_length_st_01 = test + @Test def eop_zero_length_bin_01 = test } diff --git a/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindPattern.scala b/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindPattern.scala index abd9de9d7e..c0e08a7ddc 100644 --- a/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindPattern.scala +++ b/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindPattern.scala @@ -77,6 +77,14 @@ class TestLengthKindPattern extends TdmlTests { @Test def lengthPatternEncodingErrorReplace = test @Test def lengthPatternBinaryPatternLimit = test + + @Test def pattern_pad_str_right_unparse = test + @Test def pattern_pad_str_right_roundtrip = test + @Test def pattern_pad_str_no_padding_needed = test + + @Test def pattern_pad_utf16_str_right_unparse = test + @Test def pattern_pad_utf16_str_right_roundtrip = test + @Test def pattern_pad_utf16_str_no_padding_needed = test } class TestLengthKindPatternAI extends TdmlTests { From 98f8f4987dff0a01c5be96c2eb6d5e9304cb7c7a Mon Sep 17 00:00:00 2001 From: olabusayoT <50379531+olabusayoT@users.noreply.github.com> Date: Fri, 12 Jun 2026 13:11:29 -0400 Subject: [PATCH 8/9] EndOfParent Performance Fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replaced the recursive collectFromTerms in TermGrammarMixin.eopElementInfoMap with an iterative DFS using mutable.ArrayDeque — stack depth is now O(1) regardless of schema nesting depth. - Schema compilation for schemas with deep nesting no longer consumes JVM stack frames proportional to nesting depth. collectFromTerms uses a heap-allocated work queue, so stack usage is O(1) regardless of how many levels of sequences, choices, and elements the schema contains. - O(1) optEndOfDataPosition for file-backed streams: FileChannel.size() is now captured once in InputSourceDataInputStream factory methods and passed as optSizeHint to BucketingInputSource, skipping the 256 MiB scan entirely. Detection logic is consolidated into private helpers (sizeHintForStream / sizeHintForChannel) so there is a single place to maintain it. BucketingInputSource no longer special-cases FileInputStream internally. - ArrayDeque for bucket spine: switched BucketingInputSource.buckets from ArrayBuffer to ArrayDeque. Periodic spine compaction (every ~2 MiB of freed data) now runs in O(removed buckets) rather than O(live buckets), because ArrayDeque front-removal advances a head pointer instead of shifting all remaining elements. - Fixed Integer overflow in compact(): oldestBucketIndex * bucketSize was evaluated as Int×Int and would overflow for inputs larger than ~2.1 GB on schemas with no choice/optional elements (where releasePosition() is never called). Changed to oldestBucketIndex.toLong * bucketSize, matching the existing .toLong in the releasePosition() compaction path. DAFFODIL-238 --- .../core/grammar/TermGrammarMixin.scala | 45 +++++++--- .../org/apache/daffodil/io/InputSource.scala | 58 ++++++++---- .../io/InputSourceDataInputStream.scala | 42 ++++++++- .../runtime1/processors/parsers/PState.scala | 3 +- .../io/TestDaffodilDataInputSource.scala | 90 +++++++++++++++++++ 5 files changed, 205 insertions(+), 33 deletions(-) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/TermGrammarMixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/TermGrammarMixin.scala index c74b6178de..c33bbadd57 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/TermGrammarMixin.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/TermGrammarMixin.scala @@ -17,6 +17,8 @@ package org.apache.daffodil.core.grammar +import scala.collection.mutable + import org.apache.daffodil.core.dsom.ChoiceTermBase import org.apache.daffodil.core.dsom.ElementBase import org.apache.daffodil.core.dsom.ModelGroup @@ -158,20 +160,37 @@ trait TermGrammarMixin extends AlignedMixin with BitOrderMixin with TermRuntime1 boxEv } - def collectFromTerms(terms: Seq[Term]): Seq[(ElementBase, Info)] = - terms.flatMap { - case e: ElementBase if e.lengthKind == LengthKind.EndOfParent => - Nil // EOP child; its enclosing parent/choice already added the entry - case e: ElementBase => - val info = elementInfo(e) - e.childrenEndOfParent.map(_ -> info) ++ collectFromTerms(e.termChildren) - case c: ChoiceTermBase if c.choiceLengthKind == ChoiceLengthKind.Explicit => - val eops = c.childrenEndOfParent - val entries = eops.map(eop => eop -> (false, Some(choiceBoxEv(c, eop)))) - entries ++ collectFromTerms(c.termChildren) - case other => - collectFromTerms(other.termChildren) + // Iterative DFS over the schema term tree so that deeply-nested schemas + // (hundreds of levels) do not overflow the JVM stack the way a recursive + // implementation would. + def collectFromTerms(initialTerms: Seq[Term]): Seq[(ElementBase, Info)] = { + val result = new mutable.ArrayBuffer[(ElementBase, Info)]() + val work = new mutable.ArrayDeque[Term]() + work.appendAll(initialTerms) + + while (work.nonEmpty) { + val term = work.removeHead() + term match { + case e: ElementBase if e.lengthKind == LengthKind.EndOfParent => + // EOP child; its enclosing parent/choice already added the entry. + + case e: ElementBase => + val info = elementInfo(e) + e.childrenEndOfParent.foreach(eop => result += (eop -> info)) + // Prepend in reverse so leftmost child is processed next (DFS order). + e.termChildren.reverseIterator.foreach(work.prepend) + + case c: ChoiceTermBase if c.choiceLengthKind == ChoiceLengthKind.Explicit => + val eops = c.childrenEndOfParent + eops.foreach(eop => result += (eop -> (false, Some(choiceBoxEv(c, eop))))) + c.termChildren.reverseIterator.foreach(work.prepend) + + case other => + other.termChildren.reverseIterator.foreach(work.prepend) + } } + result.toSeq + } this match { case e: ElementBase => diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSource.scala b/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSource.scala index f393f337cf..ec6f6ea198 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSource.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSource.scala @@ -19,7 +19,7 @@ package org.apache.daffodil.io import java.io.InputStream import java.nio.ByteBuffer -import scala.collection.mutable.ArrayBuffer +import scala.collection.mutable.ArrayDeque import org.apache.daffodil.lib.exceptions.Assert import org.apache.daffodil.lib.exceptions.ThinException @@ -220,11 +220,19 @@ abstract class InputSource { * @param inputStream the java.io.Inputstream to read data from * @param bucketSizeExponent the exponent used to calculate the size of a single bucket, defined as 2^buckeSizeExponent bytes * @param maxCacheSizeInBytes the max memory allowed to be used for bucket storage (num buckets * bucketSize) + * @param optSizeHint if the total number of bytes available from this stream is known in advance, + * supply it here so that [[optEndOfDataPosition]] can short-circuit the + * 256 MiB scan. Must reflect the bytes remaining from the stream's current + * position at the time this BucketingInputSource is constructed. Use the + * factory methods on [[InputSourceDataInputStream]] rather than constructing + * this class directly — they populate this hint automatically for file-backed + * streams. */ class BucketingInputSource( inputStream: java.io.InputStream, bucketSizeExponent: Int = 13, - maxCacheSizeInBytes: Int = 256 * (1 << 20) + maxCacheSizeInBytes: Int = 256 * (1 << 20), + optSizeHint: Option[Long] = None ) extends InputSource { private final val bucketSize: Int = 1 << bucketSizeExponent @@ -251,11 +259,17 @@ class BucketingInputSource( */ private val maxNumberOfNonNullBuckets = Math.max(maxCacheSizeInBytes / bucketSize, 2) + // Compact the null-entry spine of the buckets ArrayBuffer after this many + // buckets have been freed, so that the ArrayBuffer length stays proportional + // to live buckets rather than total bytes ever read. 256 × 8 KiB = 2 MiB + // freed between compactions; the ArrayBuffer.remove call is O(live buckets). + private val compactThresholdBuckets = 256 + /** * Array of buckets */ private val buckets = { - val b = new ArrayBuffer[Bucket]() + val b = new ArrayDeque[Bucket]() // initialize with an empty bucket b += new Bucket() b @@ -545,6 +559,15 @@ class BucketingInputSource( // buckets as possible. Note that this might still not release anything // if the oldest bucket contains the current byte position. releaseBuckets() + // Periodically collapse the null-reference spine so that the ArrayBuffer + // length stays bounded by the live bucket count, not the total number of + // bytes ever read. Only reachable when oldestBucketIndex has actually + // advanced (i.e. refCount was 0), so no live mark covers any removed entry. + if (oldestBucketIndex >= compactThresholdBuckets) { + buckets.remove(0, oldestBucketIndex) + headBucketBytePosition0b += oldestBucketIndex.toLong * bucketSize + oldestBucketIndex = 0 + } } } @@ -576,32 +599,33 @@ class BucketingInputSource( def compact(): Unit = { releaseBuckets() buckets.remove(0, oldestBucketIndex) - val bytesRemoved = oldestBucketIndex * bucketSize + val bytesRemoved = oldestBucketIndex.toLong * bucketSize headBucketBytePosition0b += bytesRemoved oldestBucketIndex = 0 } /** - * Returns the 0-based exclusive byte position of the end of the data stream - * by reading and bucketing data until EOF is reached. Does not affect the - * current read position. + * Returns the 0-based exclusive byte position of the end of the data stream. + * + * When the total stream size was known at construction time (supplied via + * [[optSizeHint]]) this returns immediately without reading any additional + * data from the stream. * + * Otherwise it falls back to the scan-based approach: reads and buckets data + * until EOF is found or the [[maxCacheSizeInBytes]] scan window is exhausted. * The scan window extends [[maxCacheSizeInBytes]] bytes beyond the last * already-filled bucket (not from the current read position), maximising * coverage without invalidating backtracking state. * - * Returns None if EOF is not reached within that window. + * Returns None if EOF is not reachable within the scan window. */ - override lazy val optEndOfDataPosition: Option[Long] = { - val lastFilledBucketIndex = buckets.length - 1 - val goalBucketIndex = lastFilledBucketIndex + maxNumberOfNonNullBuckets - 1 - val endOfDataNotReached = fillBucketsToIndex(goalBucketIndex, bucketSize) - if (endOfDataNotReached) { - None - } else { - Some(totalBytesBucketed) + override lazy val optEndOfDataPosition: Option[Long] = + optSizeHint.orElse { + val lastFilledBucketIndex = buckets.length - 1 + val goalBucketIndex = lastFilledBucketIndex + maxNumberOfNonNullBuckets - 1 + val endOfDataNotReached = fillBucketsToIndex(goalBucketIndex, bucketSize) + if (endOfDataNotReached) None else Some(totalBytesBucketed) } - } } /** diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSourceDataInputStream.scala b/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSourceDataInputStream.scala index d73f854301..b7edfa6e22 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSourceDataInputStream.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSourceDataInputStream.scala @@ -17,11 +17,16 @@ package org.apache.daffodil.io +import java.io.FileInputStream import java.io.InputStream import java.math.BigInteger as JBigInt import java.nio.ByteBuffer import java.nio.CharBuffer import java.nio.LongBuffer +import java.nio.channels.Channels +import java.nio.channels.FileChannel +import java.nio.channels.ReadableByteChannel +import scala.util.Try import org.apache.daffodil.api import org.apache.daffodil.lib.exceptions.Assert @@ -54,7 +59,42 @@ object InputSourceDataInputStream { } def apply(in: InputStream): InputSourceDataInputStream = { - new InputSourceDataInputStream(new BucketingInputSource(in)) + new InputSourceDataInputStream( + new BucketingInputSource(in, optSizeHint = sizeHintForStream(in)) + ) + } + + /** + * Create an InputSourceDataInputStream from a ReadableByteChannel. + * When the channel is a FileChannel its size is captured up front so that + * dfdl:lengthKind='endOfParent' parsers can skip the 256 MiB scan + * normally required to locate end-of-data. + */ + def apply(channel: ReadableByteChannel): InputSourceDataInputStream = { + new InputSourceDataInputStream( + new BucketingInputSource( + Channels.newInputStream(channel), + optSizeHint = sizeHintForChannel(channel) + ) + ) + } + + /** + * Returns the number of bytes remaining in a FileInputStream at the moment + * of the call, or None for any other stream type or on I/O error. + */ + private def sizeHintForStream(in: InputStream): Option[Long] = in match { + case fis: FileInputStream => sizeHintForChannel(fis.getChannel) + case _ => None + } + + /** + * Returns the number of bytes remaining in a FileChannel at the moment of + * the call, or None for any other channel type or on I/O error. + */ + private def sizeHintForChannel(channel: ReadableByteChannel): Option[Long] = channel match { + case fc: FileChannel => Try(fc.size() - fc.position()).toOption + case _ => None } } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/PState.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/PState.scala index fa64d628bb..25c744b622 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/PState.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/PState.scala @@ -17,7 +17,6 @@ package org.apache.daffodil.runtime1.processors.parsers -import java.nio.channels.Channels import java.nio.charset.StandardCharsets import java.nio.file.Files import java.nio.file.Path @@ -801,7 +800,7 @@ object PState { dataProc: DFDL.DataProcessor, areDebugging: Boolean ): PState = { - val dis = InputSourceDataInputStream(Channels.newInputStream(input)) + val dis = InputSourceDataInputStream(input) createInitialPState(root, dis, output, dataProc, areDebugging) } } diff --git a/daffodil-core/src/test/scala/org/apache/daffodil/io/TestDaffodilDataInputSource.scala b/daffodil-core/src/test/scala/org/apache/daffodil/io/TestDaffodilDataInputSource.scala index 3b8caf80a9..196df04bc0 100644 --- a/daffodil-core/src/test/scala/org/apache/daffodil/io/TestDaffodilDataInputSource.scala +++ b/daffodil-core/src/test/scala/org/apache/daffodil/io/TestDaffodilDataInputSource.scala @@ -20,6 +20,7 @@ package org.apache.daffodil.io import java.nio.ByteBuffer import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue import org.junit.Test /** @@ -421,3 +422,92 @@ class TestByteBufferInputSource { assertEquals(-1, bis.get()) } } + +class TestBucketingInputSourceFileChannel { + + /** + * Regression for fix 1: optEndOfDataPosition must short-circuit via the + * file size supplied by InputSourceDataInputStream.apply(InputStream) for a + * FileInputStream, rather than reading up to maxCacheSizeInBytes (256 MiB) + * of lookahead data to scan for EOF. + * + * Verified by asserting knownBytesAvailable() == 0 both before and after the + * call: if the scan path had run it would have loaded data into buckets, + * making knownBytesAvailable() > 0. + */ + @Test def testOptEndOfDataPositionUsesFileSizeForFileInputStream(): Unit = { + val content = Array[Byte](1, 2, 3, 4, 5) + val tmpFile = java.nio.file.Files.createTempFile("daffodil-eop-test", ".bin") + try { + java.nio.file.Files.write(tmpFile, content) + val fis = new java.io.FileInputStream(tmpFile.toFile) + try { + // Size hint is supplied by the factory, not by BucketingInputSource directly. + val src = InputSourceDataInputStream(fis).inputSource + assertEquals(0L, src.knownBytesAvailable()) + assertEquals(Some(5L), src.optEndOfDataPosition) + // Scan path would have populated buckets; verify it was not taken. + assertEquals(0L, src.knownBytesAvailable()) + } finally { + fis.close() + } + } finally { + java.nio.file.Files.deleteIfExists(tmpFile) + } + } + + /** + * Verify that BucketingInputSource constructed directly (no optSizeHint) falls + * back to the scan path: optEndOfDataPosition reads the entire stream into + * buckets, so knownBytesAvailable() is positive after the call. + */ + @Test def testBucketingInputSourceWithoutHintUsesScanner(): Unit = { + val content = Array[Byte](1, 2, 3, 4, 5) + val tmpFile = java.nio.file.Files.createTempFile("daffodil-eop-test", ".bin") + try { + java.nio.file.Files.write(tmpFile, content) + val fis = new java.io.FileInputStream(tmpFile.toFile) + try { + val bis = new BucketingInputSource(fis) // no optSizeHint + assertEquals(0L, bis.knownBytesAvailable()) + assertEquals(Some(5L), bis.optEndOfDataPosition) + // Scan path was taken - data has been buffered. + assertTrue(bis.knownBytesAvailable() > 0) + } finally { + fis.close() + } + } finally { + java.nio.file.Files.deleteIfExists(tmpFile) + } + } + + /** + * Regression for compact() Int overflow: compact() must correctly update + * headBucketBytePosition0b when oldestBucketIndex is large. Uses tiny 2-byte + * buckets so the scenario exercises multiple compaction cycles without + * needing gigabytes of test data. Verifies that position() and subsequent + * reads are correct after compact() runs with a non-trivial oldestBucketIndex. + */ + @Test def testCompactCorrectlyUpdatesPositionOffset(): Unit = { + // 2-byte buckets, 4-byte max cache (2 live buckets). Buckets are evicted + // quickly, driving oldestBucketIndex to a non-zero value before compact(). + val tis = new TestInputStream + tis.setEOF(30) + val bis = new BucketingInputSource(tis, bucketSizeExponent = 1, maxCacheSizeInBytes = 4) + + // Read 20 bytes; with 2-byte buckets the eviction loop inside + // fillBucketsToIndex sets oldestBucketIndex > 0. + var i = 0 + while (i < 20) { assertEquals(i % 10, bis.get()); i += 1 } + + // compact() advances headBucketBytePosition0b. Before the Int-overflow fix + // the multiplication oldestBucketIndex * bucketSize was Int×Int; with large + // oldestBucketIndex that would produce a wrong (potentially negative) offset. + bis.compact() + assertEquals(20L, bis.position()) + + // Reads after compact() must still produce the correct sequence. + while (i < 30) { assertEquals(i % 10, bis.get()); i += 1 } + assertEquals(30L, bis.position()) + } +} From 4552abb3ec7d1c71212c801cdbc2cf5fc7d6d8c2 Mon Sep 17 00:00:00 2001 From: olabusayoT <50379531+olabusayoT@users.noreply.github.com> Date: Mon, 15 Jun 2026 10:34:08 -0400 Subject: [PATCH 9/9] fixup! - Renamed `realElementChildren` to `realEOPElementChildren` and updated references - Simplified body length requirement checks in `ElementBaseGrammarMixin` using `LengthKind` match expression. - Updated error messaging in `ParserTraits` to SDEs for length overflows and data position errors. - Streamlined handling of bit limits and EOP logic for complex types. DAFFODIL-238 --- .../org/apache/daffodil/core/dsom/Term.scala | 4 +- .../grammar/ElementBaseGrammarMixin.scala | 31 ++-- .../core/grammar/TermGrammarMixin.scala | 5 +- .../processors/parsers/ParserTraits.scala | 28 ++-- .../parsers/SpecifiedLengthParsers.scala | 138 +++++++----------- 5 files changed, 73 insertions(+), 133 deletions(-) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/Term.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/Term.scala index e093bbf223..626caf9d8f 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/Term.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/Term.scala @@ -560,11 +560,11 @@ trait Term } } - final lazy val realElementChildren: Seq[ElementBase] = { + final lazy val realEOPElementChildren: Seq[ElementBase] = { termChildren.flatMap { case eb: ElementBase => Seq(eb) case c: Choice => Nil - case mg: ModelGroup => mg.realElementChildren + case mg: ModelGroup => mg.realEOPElementChildren } } } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala index cf0dd2e81f..e677918611 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala @@ -23,7 +23,6 @@ import org.apache.daffodil.core.dsom.ElementBase import org.apache.daffodil.core.dsom.ExpressionCompilers import org.apache.daffodil.core.dsom.InitiatedTerminatedMixin import org.apache.daffodil.core.dsom.PrefixLengthQuasiElementDecl -import org.apache.daffodil.core.dsom.Root import org.apache.daffodil.core.grammar.primitives.* import org.apache.daffodil.core.runtime1.ElementBaseRuntime1Mixin import org.apache.daffodil.lib.exceptions.Assert @@ -1480,30 +1479,24 @@ trait ElementBaseGrammarMixin // so it can do any internal checks for example blobValue's check for // non-explicit lengthKind val body = bodyArg - - val eopSimpleTypeElementThatNeedsBitLimit = - (isSimpleType && lengthKind == LengthKind.EndOfParent) - && !this.isInstanceOf[Root] - && impliedRepresentation != Representation.Text - && !isNillable - && primType != PrimType.HexBinary // there are essentially two categories of processors that read/write data input/output // stream: those that calculate lengths themselves (ex: binary numeric parsers) and those // that expect another processor to calculate the length and set the bit limit which // this processor will use as the length (such as text parsers). The following determines // if this element requires another processor to calculate and set the bit limit, and if so // adds the appropriate grammar to do that - val bodyRequiresSpecifiedLengthBitLimit = lengthKind != LengthKind.Delimited - // Note for non-root EndOfParent simple types, we don't wish to duplicate the length - // calculation efforts unless we know it needs the bit limit set by a parent - && !eopSimpleTypeElementThatNeedsBitLimit - && ( - isSimpleType && impliedRepresentation == Representation.Text || - isSimpleType && isNillable || - isComplexType && lengthKind != LengthKind.Implicit || - lengthKind == LengthKind.Prefixed || - isSimpleType && primType == PrimType.HexBinary && lengthKind == LengthKind.Pattern - ) + val bodyRequiresSpecifiedLengthBitLimit = { + lengthKind match { + case LengthKind.Delimited => false + case LengthKind.Implicit if isComplexType => false + case LengthKind.Prefixed => true + case LengthKind.Pattern if isSimpleType && primType == PrimType.HexBinary => true + case _ if isComplexType => true + case _ if isSimpleType => + impliedRepresentation == Representation.Text || isNillable + case _ => false + } + } if (!bodyRequiresSpecifiedLengthBitLimit) { body } else { diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/TermGrammarMixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/TermGrammarMixin.scala index c33bbadd57..06f120a4b5 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/TermGrammarMixin.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/TermGrammarMixin.scala @@ -173,18 +173,15 @@ trait TermGrammarMixin extends AlignedMixin with BitOrderMixin with TermRuntime1 term match { case e: ElementBase if e.lengthKind == LengthKind.EndOfParent => // EOP child; its enclosing parent/choice already added the entry. - case e: ElementBase => val info = elementInfo(e) e.childrenEndOfParent.foreach(eop => result += (eop -> info)) // Prepend in reverse so leftmost child is processed next (DFS order). e.termChildren.reverseIterator.foreach(work.prepend) - case c: ChoiceTermBase if c.choiceLengthKind == ChoiceLengthKind.Explicit => val eops = c.childrenEndOfParent eops.foreach(eop => result += (eop -> (false, Some(choiceBoxEv(c, eop))))) c.termChildren.reverseIterator.foreach(work.prepend) - case other => other.termChildren.reverseIterator.foreach(work.prepend) } @@ -248,7 +245,7 @@ trait TermGrammarMixin extends AlignedMixin with BitOrderMixin with TermRuntime1 "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with a dfdl:terminator." ) child.schemaDefinitionWhen( - s.realElementChildren.exists(e => e.floating == YesNo.Yes), + s.realEOPElementChildren.exists(e => e.floating == YesNo.Yes), "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with elements defining dfdl:floating='yes'." ) child.schemaDefinitionWhen( diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/ParserTraits.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/ParserTraits.scala index f37e3efc50..62a95c081b 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/ParserTraits.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/ParserTraits.scala @@ -21,7 +21,6 @@ import java.lang.Long as JLong import org.apache.daffodil.lib.exceptions.Assert import org.apache.daffodil.lib.schema.annotation.props.gen.LengthUnits -import org.apache.daffodil.lib.util.Maybe.One import org.apache.daffodil.lib.util.MaybeULong import org.apache.daffodil.lib.util.Numbers import org.apache.daffodil.runtime1.infoset.DIComplex @@ -183,22 +182,17 @@ trait BitLengthFromBitLimitMixin { /** * getLengthInBits converted to Int with a parse error on overflow. * - * The default maxCacheSizeInBytes (256 MiB) yields a maximum bit length of - * exactly Int.MaxValue + 1, so a bare .toInt without this guard can overflow - * by one bit on a full-cache EOP element. + * A bare .toInt without this guard can overflow + * for a bit length over 4GB. */ def getBitLengthAsInt(pstate: PState): Int = { val len = getLengthInBits(pstate) if (pstate.processorStatus ne Success) return 0 if (len > Int.MaxValue) { - pstate.setFailed( - new ParseError( - One(pstate.schemaFileLocation), - One(pstate.currentLocation), - "Bit length %d exceeds maximum (%d) for this parser.", - len, - Int.MaxValue - ) + pstate.SDE( + "Length exceeds maximum of %s bits: %s", + Int.MaxValue, + len ) 0 } else { @@ -237,13 +231,9 @@ trait EndOfParentBitLengthMixin extends BitLengthFromBitLimitMixin { case Some(endOfDataPosition) => (endOfDataPosition * 8) - dis.bitPos0b case None => - pstate.setFailed( - new ParseError( - One(pstate.schemaFileLocation), - One(pstate.currentLocation), - "Cannot determine end-of-data position for dfdl:lengthKind='endOfParent': " + - "data stream exceeds the maximum cache size. Consider increasing maxCacheSizeInBytes." - ) + pstate.SDE( + "Cannot determine end-of-data position for dfdl:lengthKind='endOfParent': %s bytes read", + dis.knownBytesAvailable ) 0L // callers check pstate.processorStatus before using this value } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala index 9da3209a69..f77ca32849 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala @@ -53,17 +53,27 @@ sealed abstract class SpecifiedLengthParserBase(eParser: Parser, erd: RuntimeDat protected def getBitLength(s: PState): MaybeULong def parse(pState: PState): Unit = { - lazy val shouldCheckDefinedForLength = erd match { + val maybeNBits = getBitLength(pState) + + if (pState.processorStatus ne Success) return + val nBits = maybeNBits.get + val dis = pState.dataInputStream + + val shouldCheckDefinedForLength = erd match { case erd: ElementRuntimeData => !erd.isComplexType case _: ChoiceRuntimeData => false case _ => true } - val (nBits: Long, dis: InputSourceDataInputStream, startingBitPos0b: Long) = - checkLengthAndParseWithinBitLimits(pState, shouldCheckDefinedForLength) match { - case Some(result) => result - case None => return - } + if (shouldCheckDefinedForLength && !dis.isDefinedForLength(nBits)) { + PENotEnoughBits(pState, nBits, dis) + return + } + + val startingBitPos0b = dis.bitPos0b + dis.withBitLengthLimit(nBits) { + eParser.parse1(pState) + } // at this point the recursive parse of the children is finished // so if we're still successful we need to advance the position @@ -83,36 +93,15 @@ sealed abstract class SpecifiedLengthParserBase(eParser: Parser, erd: RuntimeDat Assert.invariant(dis eq pState.dataInputStream) val bitsToSkip = finalEndPos0b - dis.bitPos0b - skipBits(pState, bitsToSkip, dis) + skipBits(pState, bitsToSkip) } - protected def checkLengthAndParseWithinBitLimits( - pState: PState, - shouldCheckDefinedForLength: Boolean - ): Option[(Long, InputSourceDataInputStream, Long)] = { - val maybeNBits = getBitLength(pState) - - if (pState.processorStatus ne Success) return None - val nBits = maybeNBits.get - val dis = pState.dataInputStream - - if (shouldCheckDefinedForLength && !dis.isDefinedForLength(nBits)) { - PENotEnoughBits(pState, nBits, dis) - return None - } - - val startingBitPos0b = dis.bitPos0b - dis.withBitLengthLimit(nBits) { - eParser.parse1(pState) - } - Some((nBits, dis, startingBitPos0b)) - } - - def skipBits(pState: PState, bitsToSkip: Long, dis: InputSourceDataInputStream): Unit = { + def skipBits(pState: PState, bitsToSkip: Long): Unit = { Assert.invariant( bitsToSkip >= 0 ) // if this is < 0, then the parsing of children went past the limit, which it isn't supposed to. if (bitsToSkip > 0) { + val dis = pState.dataInputStream // skip left over bits val skipSuccess = dis.skip(bitsToSkip, pState) if (!skipSuccess) { @@ -162,65 +151,36 @@ class SpecifiedLengthEndOfParentParser( override def parse(pState: PState): Unit = { val dis = pState.dataInputStream - if (erd.isComplexType) { - if (dis.bitLimit0b.isDefined) { - val (nBits: Long, dis: InputSourceDataInputStream, startingBitPos0b: Long) = - checkLengthAndParseWithinBitLimits( - pState, - shouldCheckDefinedForLength = false - ) match { - case Some(result) => result - case None => return - } - - // at this point the recursive parse of the children is finished - // so if we're still successful we need to advance the position - // to skip past any bits that the recursive child parse did not - // consume at the end. That is, the specified length can be an - // outer constraint, but the children may not use it all up, leaving - // a section at the end. - if (pState.processorStatus ne Success) return - val finalEndPos0b = startingBitPos0b + nBits - - // we want to capture the length before we do any skipping - // value length of simple types is captured by the eParser - if (pState.infoset.isComplex) - captureValueLength(pState, ULong(startingBitPos0b), ULong(dis.bitPos0b)) - - Assert.invariant(dis eq pState.dataInputStream) - val bitsToSkip = finalEndPos0b - dis.bitPos0b - // if this is < 0, then the parsing of children went past the limit, which it isn't supposed to. - skipBits(pState, bitsToSkip, dis) - } else { - val startingBitPos0b = dis.bitPos0b - - eParser.parse1(pState) - - // at this point the recursive parse of the children is finished - // so if we're still successful we need to advance the position - // to skip past any bits that the recursive child parse did not - // consume at the end. That is, the specified length can be an - // outer constraint, but the children may not use it all up, leaving - // a section at the end. - if (pState.processorStatus ne Success) return - - // we want to capture the length before we do any skipping - // value length of simple types is captured by the eParser - if (pState.infoset.isComplex) { - captureValueLength(pState, ULong(startingBitPos0b), ULong(dis.bitPos0b)) - } - - val maybeNBits = getBitLength(pState) - - if (pState.processorStatus ne Success) return - val nBits = maybeNBits.get - - if (!dis.isDefinedForLength(nBits)) { - PENotEnoughBits(pState, nBits, dis) - return - } - skipBits(pState, nBits, dis) - } + if (erd.isComplexType && dis.bitLimit0b.isEmpty) { + // when bitLimit is not defined, getBitLength will read to EOD, + // which we don't want to do for complex types because EOD could + // be very far away. So we just let the children parse as much as + // they want, and after that we figure out how much is left over to + // EOD and skip it + val startingBitPos0b = dis.bitPos0b + + eParser.parse1(pState) + + // at this point the recursive parse of the children is finished + // so if we're still successful we need to advance the position + // to skip past any bits that the recursive child parse did not + // consume at the end. That is, the specified length can be an + // outer constraint, but the children may not use it all up, leaving + // a section at the end. + if (pState.processorStatus ne Success) return + + captureValueLength(pState, ULong(startingBitPos0b), ULong(dis.bitPos0b)) + + // this will find remaining bits to EndOfData + // Since this is called after the children parse, this value indicates + // the number of bits until EndOfData, rather than the length of + // the complex type + val maybeNBits = getBitLength(pState) + + if (pState.processorStatus ne Success) return + val nBits = maybeNBits.get + + skipBits(pState, nBits) } else { super.parse(pState) }