diff --git a/src/Atn/ATNConfig.php b/src/Atn/ATNConfig.php index 7e9c9b9..a0eba41 100644 --- a/src/Atn/ATNConfig.php +++ b/src/Atn/ATNConfig.php @@ -8,7 +8,7 @@ use Antlr\Antlr4\Runtime\Atn\States\ATNState; use Antlr\Antlr4\Runtime\Comparison\Equality; use Antlr\Antlr4\Runtime\Comparison\Hashable; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\PredictionContexts\PredictionContext; /** @@ -127,22 +127,25 @@ public function equals(object $other): bool return true; } + // Field order follows Java's, cheapest discriminator first. Comparing the + // state by number rather than through `Equality::equals()` also avoids a + // full object comparison on the hottest path in the simulator. return $other instanceof self + && $this->state->stateNumber === $other->state->stateNumber && $this->alt === $other->alt - && $this->isPrecedenceFilterSuppressed() === $other->isPrecedenceFilterSuppressed() + && Equality::equals($this->context, $other->context) && $this->semanticContext->equals($other->semanticContext) - && Equality::equals($this->state, $other->state) - && Equality::equals($this->context, $other->context); + && $this->isPrecedenceFilterSuppressed() === $other->isPrecedenceFilterSuppressed(); } public function hashCode(): int { - return Hasher::hash( + return MurmurHash::hash([ $this->state->stateNumber, $this->alt, $this->context, $this->semanticContext, - ); + ], 7); } public function toString(bool $showAlt): string @@ -172,15 +175,10 @@ public function toString(bool $showAlt): string public function __toString(): string { - return \sprintf( - '(%s,%d%s%s%s)', - $this->state, - $this->alt, - $this->context !== null ? ',[' . $this->context . ']' : '', - $this->semanticContext->equals(SemanticContext::none()) - ? '' - : ',' . $this->semanticContext, - $this->reachesIntoOuterContext > 0 ? ',up=' . $this->reachesIntoOuterContext : '', - ); + // Java's no-arg `toString()` is `toString(null, true)`. Duplicating the + // formatting here is what let the two drift: this copy printed the raw + // `reachesIntoOuterContext` field, which carries the + // SUPPRESS_PRECEDENCE_FILTER bit, instead of `getOuterContextDepth()`. + return $this->toString(true); } } diff --git a/src/Atn/ATNConfigSet.php b/src/Atn/ATNConfigSet.php index 319cd6b..805e61e 100644 --- a/src/Atn/ATNConfigSet.php +++ b/src/Atn/ATNConfigSet.php @@ -5,10 +5,9 @@ namespace Antlr\Antlr4\Runtime\Atn; use Antlr\Antlr4\Runtime\Atn\SemanticContexts\SemanticContext; +use Antlr\Antlr4\Runtime\Atn\States\ATNState; use Antlr\Antlr4\Runtime\Comparison\Equality; -use Antlr\Antlr4\Runtime\Comparison\Equivalence; use Antlr\Antlr4\Runtime\Comparison\Hashable; -use Antlr\Antlr4\Runtime\Comparison\Hasher; use Antlr\Antlr4\Runtime\PredictionContexts\PredictionContext; use Antlr\Antlr4\Runtime\Utils\BitSet; use Antlr\Antlr4\Runtime\Utils\DoubleKeyMap; @@ -33,6 +32,8 @@ class ATNConfigSet implements Hashable /** * All configs but hashed by (s, i, _, pi) not including context. Wiped out * when we go readonly as this set becomes a DFA state. + * + * @var Set|null */ public ?Set $configLookup = null; @@ -82,32 +83,7 @@ public function __construct(bool $fullCtx = true) * not including context. Wiped out when we go readonly as this se * becomes a DFA state. */ - $this->configLookup = new Set(new class implements Equivalence { - public function equivalent(Hashable $left, Hashable $right): bool - { - if ($left === $right) { - return true; - } - - if (!$left instanceof ATNConfig || !$right instanceof ATNConfig) { - return false; - } - - return $left->alt === $right->alt - && $left->semanticContext->equals($right->semanticContext) - && Equality::equals($left->state, $right->state); - } - - public function hash(Hashable $value): int - { - return $value->hashCode(); - } - - public function equals(object $other): bool - { - return $other instanceof self; - } - }); + $this->configLookup = new Set(new ConfigEquivalence()); $this->fullCtx = $fullCtx; } @@ -121,6 +97,8 @@ public function equals(object $other): bool * This method updates {@see ATNConfigSet::$dipsIntoOuterContext} and * {@see ATNConfigSet::$hasSemanticContext} when necessary. * + * @param DoubleKeyMap|null $mergeCache + * * @throws \InvalidArgumentException */ public function add(ATNConfig $config, ?DoubleKeyMap $mergeCache = null): bool @@ -140,7 +118,10 @@ public function add(ATNConfig $config, ?DoubleKeyMap $mergeCache = null): bool /** @var ATNConfig $existing */ $existing = $this->configLookup->getOrAdd($config); - if ($existing->equals($config)) { + // Identity, not equality: `getOrAdd` returns the argument only when it was + // genuinely new. Comparing with `equals()` would also take this branch for + // a distinct-but-equal configuration and append a duplicate to `$configs`. + if ($existing === $config) { $this->cachedHashCode = null; $this->configs[] = $config; // track order here @@ -186,8 +167,12 @@ public function elements(): array return $this->configs; } + /** + * @return Set + */ public function getStates(): Set { + /** @var Set $states */ $states = new Set(); foreach ($this->configs as $config) { $states->add($config->state); @@ -268,25 +253,43 @@ public function equals(object $other): bool return false; } - return $this->fullCtx === $other->fullCtx + // Field order and comparison kinds follow Java. In particular + // `conflictingAlts` is compared by **reference** there + // (`this.conflictingAlts == other.conflictingAlts`), not by value: + // comparing it by value merged config sets that Java keeps distinct. + return Equality::equals($this->configs, $other->configs) + && $this->fullCtx === $other->fullCtx && $this->uniqueAlt === $other->uniqueAlt + && $this->conflictingAlts === $other->conflictingAlts && $this->hasSemanticContext === $other->hasSemanticContext - && $this->dipsIntoOuterContext === $other->dipsIntoOuterContext - && Equality::equals($this->configs, $other->configs) - && Equality::equals($this->conflictingAlts, $other->conflictingAlts); + && $this->dipsIntoOuterContext === $other->dipsIntoOuterContext; } public function hashCode(): int { + // Only a read-only set may cache: while the set is still mutable its + // configurations keep having their contexts merged underneath it. if (!$this->isReadOnly()) { - return Hasher::hash($this->configs); + return $this->computeHashCode(); } - if ($this->cachedHashCode === null) { - $this->cachedHashCode = Hasher::hash($this->configs); + return $this->cachedHashCode ??= $this->computeHashCode(); + } + + /** + * Java's `ATNConfigSet.hashCode()` is `configs.hashCode()` — that is, + * `AbstractList.hashCode()`: a 31-based accumulation over the elements, + * wrapping at 32 bits. + */ + private function computeHashCode(): int + { + $hash = 1; + + foreach ($this->configs as $config) { + $hash = (31 * $hash + $config->hashCode()) & 0xFFFFFFFF; } - return $this->cachedHashCode; + return $hash >= 0x80000000 ? $hash - 0x100000000 : $hash; } public function getLength(): int @@ -313,6 +316,9 @@ public function containsFast(ATNConfig $item): bool return $this->contains($item); } + /** + * @return \Iterator + */ public function getIterator(): \Iterator { return new \ArrayIterator($this->configs); @@ -325,8 +331,14 @@ public function clear(): void } $this->configs = []; - $this->cachedHashCode = -1; - $this->configLookup = new Set(); + // `null` is the "not computed" sentinel; `-1` was a valid cached hash and + // pinned every cleared set to the same value. (Java's sentinel *is* -1, + // which is why the port picked it up.) + $this->cachedHashCode = null; + // Clear in place rather than replacing the set: a fresh `Set` would fall + // back to the default equivalence and silently drop `ConfigEquivalence` + // (or, in `OrderedATNConfigSet`, its own), disabling context merging. + $this->configLookup?->clear(); } public function isReadOnly(): bool @@ -358,7 +370,9 @@ public function __toString(): string return \sprintf( '[%s]%s%s%s%s', \implode(', ', $this->configs), - $this->hasSemanticContext ? ',hasSemanticContext=' . $this->hasSemanticContext : '', + // Java appends the boolean itself, which prints `true`; interpolating + // a PHP bool printed `1`. + $this->hasSemanticContext ? ',hasSemanticContext=true' : '', $this->uniqueAlt !== ATN::INVALID_ALT_NUMBER ? ',uniqueAlt=' . $this->uniqueAlt : '', $this->conflictingAlts !== null ? ',conflictingAlts=' . $this->conflictingAlts : '', $this->dipsIntoOuterContext ? ',dipsIntoOuterContext' : '', diff --git a/src/Atn/ATNDeserializationOptions.php b/src/Atn/ATNDeserializationOptions.php index d04724b..ac5e3fd 100644 --- a/src/Atn/ATNDeserializationOptions.php +++ b/src/Atn/ATNDeserializationOptions.php @@ -12,16 +12,16 @@ final class ATNDeserializationOptions private bool $generateRuleBypassTransitions; + private static ?self $defaultOptions = null; + public static function defaultOptions(): ATNDeserializationOptions { - static $defaultOptions; - - if ($defaultOptions === null) { - $defaultOptions = new ATNDeserializationOptions(); - $defaultOptions->readOnly = true; + if (self::$defaultOptions === null) { + self::$defaultOptions = new ATNDeserializationOptions(); + self::$defaultOptions->readOnly = true; } - return $defaultOptions; + return self::$defaultOptions; } public function __construct(?ATNDeserializationOptions $options = null) diff --git a/src/Atn/ATNSimulator.php b/src/Atn/ATNSimulator.php index 5ecce88..c23e359 100644 --- a/src/Atn/ATNSimulator.php +++ b/src/Atn/ATNSimulator.php @@ -62,11 +62,11 @@ public function __construct(ATN $atn, PredictionContextCache $sharedContextCache $this->sharedContextCache = $sharedContextCache; } + private static ?DFAState $error = null; + public static function error(): DFAState { - static $error; - - return $error ?? ($error = new DFAState(new ATNConfigSet(), 0x7FFFFFFF)); + return self::$error ??= new DFAState(new ATNConfigSet(), 0x7FFFFFFF); } abstract public function reset(): void; @@ -92,7 +92,8 @@ public function getSharedContextCache(): PredictionContextCache public function getCachedContext(PredictionContext $context): PredictionContext { - $visited = []; + /** @var \SplObjectStorage $visited */ + $visited = new \SplObjectStorage(); return PredictionContext::getCachedPredictionContext( $context, diff --git a/src/Atn/Actions/LexerAction.php b/src/Atn/Actions/LexerAction.php index a8ab492..0b4da8e 100644 --- a/src/Atn/Actions/LexerAction.php +++ b/src/Atn/Actions/LexerAction.php @@ -14,7 +14,7 @@ * * @author Sam Harwell */ -interface LexerAction extends Hashable +interface LexerAction extends Hashable, \Stringable { /** * Gets the serialization type of the lexer action. diff --git a/src/Atn/Actions/LexerChannelAction.php b/src/Atn/Actions/LexerChannelAction.php index fe05d3c..57f8b9a 100644 --- a/src/Atn/Actions/LexerChannelAction.php +++ b/src/Atn/Actions/LexerChannelAction.php @@ -4,7 +4,7 @@ namespace Antlr\Antlr4\Runtime\Atn\Actions; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\Lexer; /** @@ -70,7 +70,7 @@ public function execute(Lexer $lexer): void public function hashCode(): int { - return Hasher::hash($this->getActionType(), $this->channel); + return MurmurHash::hash([$this->getActionType(), $this->channel]); } public function equals(object $other): bool diff --git a/src/Atn/Actions/LexerCustomAction.php b/src/Atn/Actions/LexerCustomAction.php index 93a2f7b..c364cd1 100644 --- a/src/Atn/Actions/LexerCustomAction.php +++ b/src/Atn/Actions/LexerCustomAction.php @@ -4,7 +4,7 @@ namespace Antlr\Antlr4\Runtime\Atn\Actions; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\Lexer; /** @@ -97,9 +97,14 @@ public function execute(Lexer $lexer): void $lexer->action(null, $this->ruleIndex, $this->actionIndex); } + public function __toString(): string + { + return \sprintf('custom(%d:%d)', $this->ruleIndex, $this->actionIndex); + } + public function hashCode(): int { - return Hasher::hash($this->getActionType(), $this->ruleIndex, $this->actionIndex); + return MurmurHash::hash([$this->getActionType(), $this->ruleIndex, $this->actionIndex]); } public function equals(object $other): bool diff --git a/src/Atn/Actions/LexerIndexedCustomAction.php b/src/Atn/Actions/LexerIndexedCustomAction.php index 36c8f97..249306e 100644 --- a/src/Atn/Actions/LexerIndexedCustomAction.php +++ b/src/Atn/Actions/LexerIndexedCustomAction.php @@ -4,7 +4,7 @@ namespace Antlr\Antlr4\Runtime\Atn\Actions; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\Lexer; /** @@ -102,9 +102,14 @@ public function execute(Lexer $lexer): void $this->action->execute($lexer); } + public function __toString(): string + { + return \sprintf('%s@%d', $this->action, $this->offset); + } + public function hashCode(): int { - return Hasher::hash($this->getActionType(), $this->offset, $this->action); + return MurmurHash::hash([$this->offset, $this->action]); } public function equals(object $other): bool diff --git a/src/Atn/Actions/LexerModeAction.php b/src/Atn/Actions/LexerModeAction.php index 7377ebd..2f1da9b 100644 --- a/src/Atn/Actions/LexerModeAction.php +++ b/src/Atn/Actions/LexerModeAction.php @@ -4,7 +4,7 @@ namespace Antlr\Antlr4\Runtime\Atn\Actions; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\Lexer; final class LexerModeAction implements LexerAction @@ -64,7 +64,7 @@ public function execute(Lexer $lexer): void public function hashCode(): int { - return Hasher::hash($this->getActionType(), $this->mode); + return MurmurHash::hash([$this->getActionType(), $this->mode]); } public function equals(object $other): bool diff --git a/src/Atn/Actions/LexerMoreAction.php b/src/Atn/Actions/LexerMoreAction.php index e1c5a03..86c623c 100644 --- a/src/Atn/Actions/LexerMoreAction.php +++ b/src/Atn/Actions/LexerMoreAction.php @@ -4,7 +4,7 @@ namespace Antlr\Antlr4\Runtime\Atn\Actions; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\Lexer; /** @@ -20,11 +20,11 @@ final class LexerMoreAction implements LexerAction /** * Provides a singleton instance of this parameterless lexer action. */ + private static ?self $instance = null; + public static function instance(): self { - static $instance; - - return $instance ??= new self(); + return self::$instance ??= new self(); } /** @@ -59,7 +59,7 @@ public function execute(Lexer $lexer): void public function hashCode(): int { - return Hasher::hash($this->getActionType()); + return MurmurHash::hash([$this->getActionType()]); } public function equals(object $other): bool diff --git a/src/Atn/Actions/LexerPopModeAction.php b/src/Atn/Actions/LexerPopModeAction.php index b5de30b..45d1b03 100644 --- a/src/Atn/Actions/LexerPopModeAction.php +++ b/src/Atn/Actions/LexerPopModeAction.php @@ -4,7 +4,7 @@ namespace Antlr\Antlr4\Runtime\Atn\Actions; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\Lexer; /** @@ -20,11 +20,11 @@ final class LexerPopModeAction implements LexerAction /** * Provides a singleton instance of this parameterless lexer action. */ + private static ?self $instance = null; + public static function instance(): self { - static $instance; - - return $instance ??= new self(); + return self::$instance ??= new self(); } /** @@ -59,7 +59,7 @@ public function execute(Lexer $lexer): void public function hashCode(): int { - return Hasher::hash($this->getActionType()); + return MurmurHash::hash([$this->getActionType()]); } public function equals(object $other): bool diff --git a/src/Atn/Actions/LexerPushModeAction.php b/src/Atn/Actions/LexerPushModeAction.php index 5b45318..1eb1104 100644 --- a/src/Atn/Actions/LexerPushModeAction.php +++ b/src/Atn/Actions/LexerPushModeAction.php @@ -4,7 +4,7 @@ namespace Antlr\Antlr4\Runtime\Atn\Actions; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\Lexer; /** @@ -65,7 +65,7 @@ public function execute(Lexer $lexer): void public function hashCode(): int { - return Hasher::hash($this->getActionType(), $this->mode); + return MurmurHash::hash([$this->getActionType(), $this->mode]); } public function equals(object $other): bool diff --git a/src/Atn/Actions/LexerSkipAction.php b/src/Atn/Actions/LexerSkipAction.php index fbb9a8a..7e276b5 100644 --- a/src/Atn/Actions/LexerSkipAction.php +++ b/src/Atn/Actions/LexerSkipAction.php @@ -4,7 +4,7 @@ namespace Antlr\Antlr4\Runtime\Atn\Actions; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\Lexer; /** @@ -20,11 +20,11 @@ final class LexerSkipAction implements LexerAction /** * Provides a singleton instance of this parameterless lexer action. */ + private static ?self $instance = null; + public static function instance(): self { - static $instance; - - return $instance ??= new self(); + return self::$instance ??= new self(); } /** @@ -59,7 +59,7 @@ public function execute(Lexer $lexer): void public function hashCode(): int { - return Hasher::hash($this->getActionType()); + return MurmurHash::hash([$this->getActionType()]); } public function equals(object $other): bool diff --git a/src/Atn/Actions/LexerTypeAction.php b/src/Atn/Actions/LexerTypeAction.php index ab046d9..aaa6ba4 100644 --- a/src/Atn/Actions/LexerTypeAction.php +++ b/src/Atn/Actions/LexerTypeAction.php @@ -4,7 +4,7 @@ namespace Antlr\Antlr4\Runtime\Atn\Actions; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\Lexer; /** @@ -70,7 +70,7 @@ public function execute(Lexer $lexer): void public function hashCode(): int { - return Hasher::hash($this->getActionType(), $this->type); + return MurmurHash::hash([$this->getActionType(), $this->type]); } public function equals(object $other): bool diff --git a/src/Atn/AltAndContextEquivalence.php b/src/Atn/AltAndContextEquivalence.php new file mode 100644 index 0000000..f303c90 --- /dev/null +++ b/src/Atn/AltAndContextEquivalence.php @@ -0,0 +1,59 @@ +state->stateNumber === $right->state->stateNumber + && Equality::equals($left->context, $right->context); + } + + public function hash(Hashable $value): int + { + if (!$value instanceof ATNConfig) { + throw new \InvalidArgumentException('Unsupported value.'); + } + + // The hash is a function of the state number and the context only. + return MurmurHash::hash([$value->state->stateNumber, $value->context], 7); + } + + public function equals(object $other): bool + { + return $other instanceof self; + } +} diff --git a/src/Atn/ConfigEquivalence.php b/src/Atn/ConfigEquivalence.php new file mode 100644 index 0000000..c023ffa --- /dev/null +++ b/src/Atn/ConfigEquivalence.php @@ -0,0 +1,61 @@ +state->stateNumber === $right->state->stateNumber + && $left->alt === $right->alt + && $left->semanticContext->equals($right->semanticContext); + } + + public function hash(Hashable $value): int + { + if (!$value instanceof ATNConfig) { + return 0; + } + + $hash = 7; + $hash = 31 * $hash + $value->state->stateNumber; + $hash = 31 * $hash + $value->alt; + + return 31 * $hash + $value->semanticContext->hashCode(); + } + + public function equals(object $other): bool + { + return $other instanceof self; + } +} diff --git a/src/Atn/LexerATNConfig.php b/src/Atn/LexerATNConfig.php index 0b353c1..39a0bac 100644 --- a/src/Atn/LexerATNConfig.php +++ b/src/Atn/LexerATNConfig.php @@ -7,7 +7,7 @@ use Antlr\Antlr4\Runtime\Atn\States\ATNState; use Antlr\Antlr4\Runtime\Atn\States\DecisionState; use Antlr\Antlr4\Runtime\Comparison\Equality; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\PredictionContexts\PredictionContext; final class LexerATNConfig extends ATNConfig @@ -43,14 +43,14 @@ public function isPassedThroughNonGreedyDecision(): bool public function hashCode(): int { - return Hasher::hash( + return MurmurHash::hash([ $this->state->stateNumber, $this->alt, $this->context, $this->semanticContext, - $this->passedThroughNonGreedyDecision, + $this->passedThroughNonGreedyDecision ? 1 : 0, $this->lexerActionExecutor, - ); + ], 7); } public function equals(object $other): bool @@ -63,15 +63,18 @@ public function equals(object $other): bool return false; } - if (!parent::equals($other)) { + // Java checks the two cheap lexer-specific fields *before* delegating to + // the parent, which is the expensive comparison (context and semantic + // context). Doing it the other way round paid that cost on every miss. + if ($this->passedThroughNonGreedyDecision !== $other->passedThroughNonGreedyDecision) { return false; } - if ($this->passedThroughNonGreedyDecision !== $other->passedThroughNonGreedyDecision) { + if (!Equality::equals($this->lexerActionExecutor, $other->lexerActionExecutor)) { return false; } - return Equality::equals($this->lexerActionExecutor, $other->lexerActionExecutor); + return parent::equals($other); } private static function checkNonGreedyDecision(LexerATNConfig $source, ATNState $target): bool diff --git a/src/Atn/LexerATNSimulator.php b/src/Atn/LexerATNSimulator.php index 0a64035..38857cc 100644 --- a/src/Atn/LexerATNSimulator.php +++ b/src/Atn/LexerATNSimulator.php @@ -51,6 +51,12 @@ class LexerATNSimulator extends ATNSimulator /** @var array */ public array $decisionToDFA = []; + /** + * Counts `match()` calls. Java keeps the same counter for debugging; it is + * not read by the algorithm. + */ + private static int $matchCalls = 0; + protected int $mode = Lexer::DEFAULT_MODE; /** @@ -107,13 +113,7 @@ public function setCharPositionInLine(int $charPositionInLine): void */ public function match(CharStream $input, int $mode): int { - static $match_calls; - - if ($match_calls === null) { - $match_calls = 0; - } - - $match_calls++; + self::$matchCalls++; $this->mode = $mode; $mark = $input->mark(); @@ -175,6 +175,7 @@ protected function execATN(CharStream $input, DFAState $ds0): int $t = $input->LA(1); $s = $ds0; // s is current/from DFA state + $error = ATNSimulator::error(); while (true) { // As we move src->trg, src->trg, we keep track of the previous trg to @@ -195,13 +196,21 @@ protected function execATN(CharStream $input, DFAState $ds0): int // A character will take us back to an existing DFA state // that already has lots of edges out of it. e.g., .* in comments. // print("Target for:" + str(s) + " and:" + str(t)) - $target = $this->getExistingTargetState($s, $t); + // `getExistingTargetState()` and `consume()` are inlined here, and + // only here. This loop runs once per input character — it was the + // hottest path in the runtime — and each call PHP does not make is + // worth more than the readability. Both methods remain for callers + // and subclasses; keep them and this copy in step. + $edges = $s->edges; + $target = $edges !== null && $t >= self::MIN_DFA_EDGE && $t <= self::MAX_DFA_EDGE + ? $edges[$t - self::MIN_DFA_EDGE] ?? null + : null; if ($target === null) { $target = $this->computeTargetState($input, $s, $t); } - if ($target === ATNSimulator::error()) { + if ($target === $error) { break; } @@ -210,11 +219,21 @@ protected function execATN(CharStream $input, DFAState $ds0): int // position accurately reflect the state of the interpreter at the // end of the token. if ($t !== Token::EOF) { - $this->consume($input); + // Inline of `consume()`: the character it would re-read with + // `LA(1)` is already in `$t`. + if ($t === self::NEW_LINE_CODE) { + $this->line++; + $this->charPositionInLine = 0; + } else { + $this->charPositionInLine++; + } + + $input->consume(); } if ($target->isAcceptState) { $this->captureSimState($this->prevAccept, $input, $target); + if ($t === Token::EOF) { break; } @@ -506,7 +525,7 @@ protected function getEpsilonTarget( ): ?LexerATNConfig { $cfg = null; - switch ($t->getSerializationType()) { + switch ($t->serializationType) { case Transition::RULE: if (!$t instanceof RuleTransition) { throw new \LogicException('Unexpected transition type.'); diff --git a/src/Atn/LexerActionExecutor.php b/src/Atn/LexerActionExecutor.php index e6e4e57..86b799f 100644 --- a/src/Atn/LexerActionExecutor.php +++ b/src/Atn/LexerActionExecutor.php @@ -8,8 +8,8 @@ use Antlr\Antlr4\Runtime\Atn\Actions\LexerIndexedCustomAction; use Antlr\Antlr4\Runtime\CharStream; use Antlr\Antlr4\Runtime\Comparison\Equality; -use Antlr\Antlr4\Runtime\Comparison\Equatable; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\Hashable; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\Lexer; /** @@ -22,7 +22,7 @@ * * @author Sam Harwell */ -final class LexerActionExecutor implements Equatable +final class LexerActionExecutor implements Hashable { /** @var array */ private array $lexerActions; @@ -187,11 +187,9 @@ public function execute(Lexer $lexer, CharStream $input, int $startIndex): void public function hashCode(): int { - if ($this->cachedHashCode === null) { - $this->cachedHashCode = Hasher::hash($this->lexerActions); - } - - return $this->cachedHashCode; + // Java computes this once in the constructor: + // `initialize()`, one `update()` per action, `finish(hash, actions.length)`. + return $this->cachedHashCode ??= MurmurHash::hash($this->lexerActions); } public function equals(object $other): bool @@ -207,9 +205,11 @@ public function equals(object $other): bool public function __toString(): string { - return \sprintf( - 'LexerActionExecutor[%s]', - \implode(', ', \array_map('\strval', $this->lexerActions)), + $actions = \array_map( + static fn (LexerAction $action): string => (string) $action, + $this->lexerActions, ); + + return \sprintf('LexerActionExecutor[%s]', \implode(', ', $actions)); } } diff --git a/src/Atn/ParserATNSimulator.php b/src/Atn/ParserATNSimulator.php index 57fb55a..10d6a76 100644 --- a/src/Atn/ParserATNSimulator.php +++ b/src/Atn/ParserATNSimulator.php @@ -250,6 +250,8 @@ final class ParserATNSimulator extends ATNSimulator * This maps graphs a and b to merged result c. (a,b)→c. We can avoid * the merge if we ever see a and b again. Note that (b,a)→c should * also be examined during cache lookup. + * + * @var DoubleKeyMap|null */ protected ?DoubleKeyMap $mergeCache = null; @@ -372,9 +374,7 @@ public function adaptivePredict(TokenStream $input, int $decision, ParserRuleCon } } - $alt = $this->execATN($dfa, $s0, $input, $index, $outerContext); - - return $alt ?? 0; + return $this->execATN($dfa, $s0, $input, $index, $outerContext); } finally { $this->mergeCache = null; // wack cache after each prediction $this->dfa = null; @@ -422,7 +422,7 @@ public function execATN( TokenStream $input, int $startIndex, ParserRuleContext $outerContext, - ): ?int { + ): int { if (self::$traceAtnSimulation) { $this->logger->debug( 'execATN decision {decision}, DFA state {state}, LA(1)=={token} line {line}:{pos}', @@ -889,6 +889,7 @@ protected function computeReachSet(ATNConfigSet $closure, int $t, bool $fullCtx) // operation on the intermediate set to compute its initial value. if ($reach === null) { $reach = new ATNConfigSet($fullCtx); + /** @var Set $closureBusy */ $closureBusy = new Set(); $treatEofAsEpsilon = $t === Token::EOF; @@ -1014,6 +1015,7 @@ protected function computeStartState(ATNState $p, RuleContext $ctx, bool $fullCt foreach ($p->getTransitions() as $i => $t) { $c = new ATNConfig(null, $t->target, $initialContext, null, $i + 1); + /** @var Set $closureBusy */ $closureBusy = new Set(); $this->closure($c, $configs, $closureBusy, true, $fullCtx, false); @@ -1478,6 +1480,9 @@ protected function evalSemanticContextOne( * waste to pursue the closure. Might have to advance when we do * ambig detection thought :( */ + /** + * @param Set $closureBusy + */ protected function closure( ATNConfig $config, ATNConfigSet $configs, @@ -1503,6 +1508,9 @@ protected function closure( } } + /** + * @param Set $closureBusy + */ protected function closureCheckingStopState( ATNConfig $config, ATNConfigSet $configs, @@ -1585,6 +1593,9 @@ protected function closureCheckingStopState( /** * Do the actual work of walking epsilon edges. */ + /** + * @param Set $closureBusy + */ protected function closure_( ATNConfig $config, ATNConfigSet $configs, @@ -1597,25 +1608,30 @@ protected function closure_( $p = $config->state; // optimization - if (!$p->onlyHasEpsilonTransitions()) { + // The accessors are read directly here: this runs hundreds of thousands + // of times per parse and each getter is a call PHP cannot inline. + if (!$p->epsilonOnlyTransitions) { // make sure to not return here, because EOF transitions can act as // both epsilon transitions and non-epsilon transitions. $configs->add($config, $this->mergeCache); } - foreach ($p->getTransitions() as $i => $t) { + $isRuleStop = $p instanceof RuleStopState; + $atDepthZero = $depth === 0; + + foreach ($p->transitions as $i => $t) { if ($i === 0 && $this->canDropLoopEntryEdgeInLeftRecursiveRule($config)) { continue; } $continueCollecting = $collectPredicates && !$t instanceof ActionTransition; - $c = $this->getEpsilonTarget($config, $t, $continueCollecting, $depth === 0, $fullCtx, $treatEofAsEpsilon); + $c = $this->getEpsilonTarget($config, $t, $continueCollecting, $atDepthZero, $fullCtx, $treatEofAsEpsilon); if ($c !== null) { $newDepth = $depth; - if ($config->state instanceof RuleStopState) { + if ($isRuleStop) { if ($fullCtx) { throw new \LogicException('Unexpected error.'); } @@ -1660,15 +1676,32 @@ protected function closure_( } } - $this->closureCheckingStopState( - $c, - $configs, - $closureBusy, - $continueCollecting, - $fullCtx, - $newDepth, - $treatEofAsEpsilon, - ); + // `closureCheckingStopState()` forwards straight to `closure_()` + // unless the state is a rule stop, so for the common case the + // frame is skipped. It also emits the ATN trace line, hence the + // guard: with tracing on, the original path is always taken and + // the trace stays byte-identical. + if (self::$traceAtnSimulation || $c->state instanceof RuleStopState) { + $this->closureCheckingStopState( + $c, + $configs, + $closureBusy, + $continueCollecting, + $fullCtx, + $newDepth, + $treatEofAsEpsilon, + ); + } else { + $this->closure_( + $c, + $configs, + $closureBusy, + $continueCollecting, + $fullCtx, + $newDepth, + $treatEofAsEpsilon, + ); + } } } } @@ -1773,12 +1806,18 @@ protected function canDropLoopEntryEdgeInLeftRecursiveRule(ATNConfig $config): b * Are we the special loop entry/exit state? or SLL wildcard */ + // Cheapest discriminator first: this is called for the first transition + // out of every state, and almost none of them are loop entries. + if (!$p instanceof StarLoopEntryState) { + return false; + } + if ($config->context === null) { throw new \LogicException('Prediction context cannot be null.'); } if ($p->getStateType() !== ATNState::STAR_LOOP_ENTRY - || ($p instanceof StarLoopEntryState && !$p->isPrecedenceDecision) + || !$p->isPrecedenceDecision || $config->context->isEmpty() || $config->context->hasEmptyPath()) { return false; @@ -1875,7 +1914,7 @@ protected function getEpsilonTarget( bool $fullCtx, bool $treatEofAsEpsilon, ): ?ATNConfig { - switch ($t->getSerializationType()) { + switch ($t->serializationType) { case Transition::RULE: if (!$t instanceof RuleTransition) { throw new \LogicException('Unexpected transition type.'); diff --git a/src/Atn/PredictionMode.php b/src/Atn/PredictionMode.php index b08a3de..a69c364 100644 --- a/src/Atn/PredictionMode.php +++ b/src/Atn/PredictionMode.php @@ -7,10 +7,6 @@ use Antlr\Antlr4\Runtime\Atn\SemanticContexts\SemanticContext; use Antlr\Antlr4\Runtime\Atn\States\ATNState; use Antlr\Antlr4\Runtime\Atn\States\RuleStopState; -use Antlr\Antlr4\Runtime\Comparison\Equality; -use Antlr\Antlr4\Runtime\Comparison\Equivalence; -use Antlr\Antlr4\Runtime\Comparison\Hashable; -use Antlr\Antlr4\Runtime\Comparison\Hasher; use Antlr\Antlr4\Runtime\Utils\BitSet; use Antlr\Antlr4\Runtime\Utils\Map; @@ -495,31 +491,7 @@ public static function getAlts(array $altSets): BitSet public static function getConflictingAltSubsets(ATNConfigSet $configs): array { /** @var Map $configToAlts */ - $configToAlts = new Map( - new class implements Equivalence { - public function equals(object $other): bool - { - return $other instanceof self; - } - - public function equivalent(Hashable $left, Hashable $right): bool - { - return $left instanceof ATNConfig - && $right instanceof ATNConfig - && $left->state->stateNumber === $right->state->stateNumber - && Equality::equals($left->context, $right->context); - } - - public function hash(Hashable $value): int - { - if (!$value instanceof ATNConfig) { - throw new \InvalidArgumentException('Unsupported value.'); - } - - return Hasher::hash($value->state->stateNumber, $value->context); - } - }, - ); + $configToAlts = new Map(AltAndContextEquivalence::instance()); foreach ($configs->elements() as $config) { $alts = $configToAlts->get($config); @@ -541,6 +513,9 @@ public function hash(Hashable $value): int * * map[c.{@see ATNConfig::$state}] U= c.{@see ATNConfig::$alt} */ + /** + * @return Map<\Antlr\Antlr4\Runtime\Atn\States\ATNState, \Antlr\Antlr4\Runtime\Utils\BitSet> + */ public static function getStateToAltMap(ATNConfigSet $configs): Map { /** @var Map $map */ @@ -563,7 +538,7 @@ public static function getStateToAltMap(ATNConfigSet $configs): Map public static function hasStateAssociatedWithOneAlt(ATNConfigSet $configs): bool { foreach (self::getStateToAltMap($configs)->getValues() as $value) { - if ($value instanceof BitSet && $value->length() === 1) { + if ($value->length() === 1) { return true; } } diff --git a/src/Atn/SemanticContexts/AndOperator.php b/src/Atn/SemanticContexts/AndOperator.php index 31efb17..7a58d78 100644 --- a/src/Atn/SemanticContexts/AndOperator.php +++ b/src/Atn/SemanticContexts/AndOperator.php @@ -5,7 +5,7 @@ namespace Antlr\Antlr4\Runtime\Atn\SemanticContexts; use Antlr\Antlr4\Runtime\Comparison\Equality; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\Recognizer; use Antlr\Antlr4\Runtime\RuleContext; use Antlr\Antlr4\Runtime\Utils\Set; @@ -16,6 +16,12 @@ */ final class AndOperator extends Operator { + /** + * Java seeds this with `AND.class.hashCode()`, a JVM identity hash that + * differs between runs, so it cannot be mirrored exactly. A fixed seed + * keeps the value deterministic, which is what the collections need. + */ + private const HASH_SEED = 41; /** @var array */ public array $operands; @@ -127,9 +133,14 @@ public function equals(object $other): bool return Equality::equals($this->operands, $other->operands); } + /** + * Memoised: every field feeding it is set once in the constructor. + */ + private ?int $cachedHashCode = null; + public function hashCode(): int { - return Hasher::hash(41, $this->operands); + return $this->cachedHashCode ??= MurmurHash::hash($this->operands, self::HASH_SEED); } public function __toString(): string diff --git a/src/Atn/SemanticContexts/OrOperator.php b/src/Atn/SemanticContexts/OrOperator.php index ff6f2bd..bf43fbc 100644 --- a/src/Atn/SemanticContexts/OrOperator.php +++ b/src/Atn/SemanticContexts/OrOperator.php @@ -5,7 +5,7 @@ namespace Antlr\Antlr4\Runtime\Atn\SemanticContexts; use Antlr\Antlr4\Runtime\Comparison\Equality; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\Recognizer; use Antlr\Antlr4\Runtime\RuleContext; use Antlr\Antlr4\Runtime\Utils\Set; @@ -16,6 +16,12 @@ */ final class OrOperator extends Operator { + /** + * Java seeds this with `OR.class.hashCode()`, a JVM identity hash that + * differs between runs, so it cannot be mirrored exactly. A fixed seed + * keeps the value deterministic, which is what the collections need. + */ + private const HASH_SEED = 37; /** @var array */ public array $operand; @@ -129,9 +135,14 @@ public function equals(object $other): bool return Equality::equals($this->operand, $other->operand); } + /** + * Memoised: every field feeding it is set once in the constructor. + */ + private ?int $cachedHashCode = null; + public function hashCode(): int { - return Hasher::hash(37, $this->operand); + return $this->cachedHashCode ??= MurmurHash::hash($this->operand, self::HASH_SEED); } public function __toString(): string diff --git a/src/Atn/SemanticContexts/PrecedencePredicate.php b/src/Atn/SemanticContexts/PrecedencePredicate.php index 122d305..2af4f11 100644 --- a/src/Atn/SemanticContexts/PrecedencePredicate.php +++ b/src/Atn/SemanticContexts/PrecedencePredicate.php @@ -4,7 +4,6 @@ namespace Antlr\Antlr4\Runtime\Atn\SemanticContexts; -use Antlr\Antlr4\Runtime\Comparison\Hasher; use Antlr\Antlr4\Runtime\Recognizer; use Antlr\Antlr4\Runtime\RuleContext; @@ -33,7 +32,7 @@ public function evalPrecedence(Recognizer $parser, RuleContext $parserCallStack) public function hashCode(): int { - return Hasher::hash(31, $this->precedence); + return 31 + $this->precedence; } public function compareTo(PrecedencePredicate $other): int diff --git a/src/Atn/SemanticContexts/Predicate.php b/src/Atn/SemanticContexts/Predicate.php index 0574386..c68d227 100644 --- a/src/Atn/SemanticContexts/Predicate.php +++ b/src/Atn/SemanticContexts/Predicate.php @@ -4,7 +4,7 @@ namespace Antlr\Antlr4\Runtime\Atn\SemanticContexts; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\Recognizer; use Antlr\Antlr4\Runtime\RuleContext; @@ -30,9 +30,16 @@ public function eval(Recognizer $parser, RuleContext $parserCallStack): bool return $parser->sempred($localctx, $this->ruleIndex, $this->predIndex); } + /** + * Memoised: every field feeding it is set once in the constructor. + */ + private ?int $cachedHashCode = null; + public function hashCode(): int { - return Hasher::hash($this->ruleIndex, $this->predIndex, $this->isCtxDependent); + return $this->cachedHashCode ??= MurmurHash::hash( + [$this->ruleIndex, $this->predIndex, $this->isCtxDependent ? 1 : 0], + ); } public function equals(object $other): bool diff --git a/src/Atn/SemanticContexts/SemanticContext.php b/src/Atn/SemanticContexts/SemanticContext.php index 3431f6f..0511ae4 100644 --- a/src/Atn/SemanticContexts/SemanticContext.php +++ b/src/Atn/SemanticContexts/SemanticContext.php @@ -24,11 +24,11 @@ abstract class SemanticContext implements Hashable * The default {@see SemanticContext}, which is semantically equivalent to * a predicate of the form `{true}?`. */ + private static ?Predicate $none = null; + public static function none(): Predicate { - static $none; - - return $none ??= new Predicate(); + return self::$none ??= new Predicate(); } public static function andContext(?self $a, ?self $b): ?self @@ -103,14 +103,23 @@ public function evalPrecedence(Recognizer $parser, RuleContext $parserCallStack) } /** + * @param Set $set + * * @return array */ public static function filterPrecedencePredicates(Set $set): array { $result = []; + foreach ($set->getValues() as $context) { if ($context instanceof PrecedencePredicate) { $result[] = $context; + + // Java's version is a *mutating* filter — `iterator.remove()`. + // Only collecting them left every precedence predicate in the + // operand set, so `AND`/`OR` never reduced them to the single + // strictest/loosest bound and both survived into evaluation. + $set->remove($context); } } diff --git a/src/Atn/States/ATNState.php b/src/Atn/States/ATNState.php index 8a9d51e..2bcf6c1 100644 --- a/src/Atn/States/ATNState.php +++ b/src/Atn/States/ATNState.php @@ -60,9 +60,12 @@ abstract class ATNState implements Hashable /** * Track the transitions emanating from this ATN state. * + * Public, as in the reference runtime, so the simulator's inner loops can + * read it without a getter call. + * * @var array */ - protected array $transitions = []; + public array $transitions = []; /** * Used to cache lookahead during parsing, not used during construction. @@ -75,7 +78,9 @@ public function equals(object $other): bool return true; } - return $other instanceof static + // Java compares against `ATNState`, not the runtime class, so two states + // of different concrete types are still comparable. + return $other instanceof self && $this->stateNumber === $other->stateNumber; } @@ -152,7 +157,7 @@ public function __toString(): string public function hashCode(): int { - return $this->getStateType(); + return $this->stateNumber; } abstract public function getStateType(): int; diff --git a/src/Atn/Transitions/AtomTransition.php b/src/Atn/Transitions/AtomTransition.php index ef67b7d..b5035f9 100644 --- a/src/Atn/Transitions/AtomTransition.php +++ b/src/Atn/Transitions/AtomTransition.php @@ -18,7 +18,7 @@ public function __construct(ATNState $target, int $label) $this->label = $label; } - public function label(): ?IntervalSet + public function label(): IntervalSet { return IntervalSet::fromInt($this->label); } diff --git a/src/Atn/Transitions/RangeTransition.php b/src/Atn/Transitions/RangeTransition.php index d4470c4..b67fc12 100644 --- a/src/Atn/Transitions/RangeTransition.php +++ b/src/Atn/Transitions/RangeTransition.php @@ -22,7 +22,7 @@ public function __construct(ATNState $target, int $from, int $to) $this->to = $to; } - public function label(): ?IntervalSet + public function label(): IntervalSet { return IntervalSet::fromRange($this->from, $this->to); } diff --git a/src/Atn/Transitions/Transition.php b/src/Atn/Transitions/Transition.php index b6d5d3a..38c9a73 100644 --- a/src/Atn/Transitions/Transition.php +++ b/src/Atn/Transitions/Transition.php @@ -39,9 +39,20 @@ abstract class Transition implements Equatable */ public ATNState $target; + /** + * The serialization type, resolved once at construction. + * + * `getSerializationType()` is called for every transition the closure walks, + * hundreds of thousands of times per parse, and each call returns the same + * constant. Reading a property instead removes that call from the + * simulator's inner loop. + */ + public int $serializationType; + public function __construct(ATNState $target) { $this->target = $target; + $this->serializationType = $this->getSerializationType(); } /** diff --git a/src/BufferedTokenStream.php b/src/BufferedTokenStream.php index f213170..7ea23a1 100644 --- a/src/BufferedTokenStream.php +++ b/src/BufferedTokenStream.php @@ -4,8 +4,6 @@ namespace Antlr\Antlr4\Runtime; -use Antlr\Antlr4\Runtime\Utils\Set; - /** * This implementation of {@see TokenStream} loads tokens from a * {@see TokenSource} on-demand, and places the tokens in a buffer to provide @@ -286,9 +284,11 @@ public function getAllTokens(): array /** * Get all tokens from start..stop inclusively * + * @param array|null $types token types to keep, or null for all + * * @return array|null */ - public function getTokens(int $start, int $stop, ?Set $types = null): ?array + public function getTokens(int $start, int $stop, ?array $types = null): ?array { if ($start < 0 || $stop < 0) { return null; @@ -308,7 +308,7 @@ public function getTokens(int $start, int $stop, ?Set $types = null): ?array break; } - if ($types === null || $types->contains($t->getType())) { + if ($types === null || \in_array($t->getType(), $types, true)) { $subset[] = $t; } } diff --git a/src/CommonToken.php b/src/CommonToken.php index 4571bc0..9b6fd5c 100644 --- a/src/CommonToken.php +++ b/src/CommonToken.php @@ -104,11 +104,11 @@ public function __construct( * An empty {@see Pair}, which is used as the default value of * {@see CommonToken::source()} for tokens that do not have a source. */ + private static ?Pair $emptySource = null; + public static function emptySource(): Pair { - static $source; - - return $source ??= new Pair(null, null); + return self::$emptySource ??= new Pair(null, null); } /** diff --git a/src/CommonTokenFactory.php b/src/CommonTokenFactory.php index ee3df1b..28705f8 100644 --- a/src/CommonTokenFactory.php +++ b/src/CommonTokenFactory.php @@ -48,11 +48,11 @@ public function __construct(bool $copyText = false) * This token factory does not explicitly copy token text when constructing * tokens. */ + private static ?self $default = null; + public static function default(): self { - static $default; - - return $default ??= new CommonTokenFactory(); + return self::$default ??= new CommonTokenFactory(); } public function createEx( diff --git a/src/Comparison/Equality.php b/src/Comparison/Equality.php index a83de26..9dc0ea1 100644 --- a/src/Comparison/Equality.php +++ b/src/Comparison/Equality.php @@ -39,7 +39,10 @@ private static function deeplyEquals(array $left, array $right): bool } foreach ($left as $key => $value) { - if (!isset($right[$key])) { + // `array_key_exists`, not `isset`: a legitimately `null` element — + // prediction-context parent arrays are full of them — would otherwise + // read as a missing key and make two identical arrays compare unequal. + if (!\array_key_exists($key, $right)) { return false; } diff --git a/src/Comparison/MurmurHash.php b/src/Comparison/MurmurHash.php new file mode 100644 index 0000000..953ed6c --- /dev/null +++ b/src/Comparison/MurmurHash.php @@ -0,0 +1,208 @@ + update... -> finish` for the same list. + * + * `null` contributes 0 and a {@see Hashable} contributes its own + * `hashCode()`, exactly as Java's `update(int, Object)` overload does. + * + * @param array $values + */ + public static function hash(array $values, int $seed = self::DEFAULT_SEED): int + { + $words = []; + + foreach ($values as $value) { + // `hashCode()` is the hottest call in the simulator, and the vast + // majority of words are already integers, so the common cases are + // inlined rather than dispatched through `valueOf()`. `pack('V')` + // truncates to 32 bits on its own, so no masking is needed here. + if (\is_int($value)) { + $words[] = $value; + } elseif ($value instanceof Hashable) { + $words[] = $value->hashCode(); + } else { + $words[] = self::valueOf($value); + } + } + + self::$native ??= \in_array('murmur3a', \hash_algos(), true); + + return self::$native + ? self::nativeHash($words, $seed) + : self::fallbackHash($words, $seed); + } + + /** + * Java's `String.hashCode()`: `s[0]*31^(n-1) + ... + s[n-1]`, over UTF-16 + * code units, wrapping at 32 bits. + */ + public static function hashString(string $value): int + { + $hash = 0; + + // Java hashes UTF-16 code units, so anything outside the BMP has to + // contribute its surrogate pair rather than its code point. + $utf16 = \mb_convert_encoding($value, 'UTF-16BE', 'UTF-8'); + $length = \strlen($utf16); + + for ($i = 0; $i < $length; $i += 2) { + $unit = (\ord($utf16[$i]) << 8) | \ord($utf16[$i + 1]); + $hash = self::multiply($hash, 31) + $unit & self::MASK; + } + + return self::toSigned($hash); + } + + /** + * @param array $words + */ + private static function nativeHash(array $words, int $seed): int + { + $digest = \hash( + 'murmur3a', + \pack('V*', ...$words), + true, + ['seed' => $seed & self::MASK], + ); + + /** @var array{1: int} $unpacked */ + $unpacked = \unpack('N', $digest); + + return self::toSigned($unpacked[1]); + } + + /** + * The reference fold, kept verbatim so the native path above has something + * to be checked against. + * + * @param array $words + */ + private static function fallbackHash(array $words, int $seed): int + { + $hash = $seed & self::MASK; + + foreach ($words as $word) { + $k = self::multiply($word, 0xCC9E2D51); + $k = self::rotateLeft($k, 15); + $k = self::multiply($k, 0x1B873593); + + $hash = self::rotateLeft(($hash ^ $k) & self::MASK, 13); + $hash = self::multiply($hash, 5) + 0xE6546B64 & self::MASK; + } + + $hash = ($hash ^ \count($words) * 4) & self::MASK; + $hash ^= $hash >> 16; + $hash = self::multiply($hash, 0x85EBCA6B); + $hash ^= $hash >> 13; + $hash = self::multiply($hash, 0xC2B2AE35); + $hash ^= $hash >> 16; + + return self::toSigned($hash & self::MASK); + } + + private static function valueOf(mixed $value): int + { + if (\is_int($value)) { + return $value; + } + + if ($value === null) { + return 0; + } + + if ($value instanceof Hashable) { + return $value->hashCode(); + } + + if (\is_bool($value)) { + return $value ? 1 : 0; + } + + if (\is_string($value)) { + return self::hashString($value); + } + + if (\is_object($value)) { + // Java folds in `Object.hashCode()`, which is identity-based for a + // type that does not override it. `Pair` holds exactly such values. + return \spl_object_id($value); + } + + throw new \InvalidArgumentException(\sprintf( + 'Cannot hash a value of type "%s".', + \get_debug_type($value), + )); + } + + /** + * Multiplies two 32-bit values with Java's wrap-around semantics. + * + * The operands are split into 16-bit halves because a full 32x32 product + * overflows PHP's signed 64-bit integer and would silently become a float. + */ + private static function multiply(int $a, int $b): int + { + $a &= self::MASK; + $b &= self::MASK; + + $high = ($a >> 16) & 0xFFFF; + $low = $a & 0xFFFF; + + return (($high * $b & 0xFFFF) << 16) + ($low * $b) & self::MASK; + } + + private static function rotateLeft(int $value, int $bits): int + { + $value &= self::MASK; + + return ($value << $bits | $value >> 32 - $bits) & self::MASK; + } + + private static function toSigned(int $value): int + { + return $value >= 0x80000000 ? $value - 0x100000000 : $value; + } +} diff --git a/src/Dfa/DFAState.php b/src/Dfa/DFAState.php index 0ba4686..bd4a524 100644 --- a/src/Dfa/DFAState.php +++ b/src/Dfa/DFAState.php @@ -8,7 +8,7 @@ use Antlr\Antlr4\Runtime\Atn\LexerActionExecutor; use Antlr\Antlr4\Runtime\Comparison\Equality; use Antlr\Antlr4\Runtime\Comparison\Hashable; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; /** * A DFA state represents a set of possible ATN configurations. @@ -137,6 +137,10 @@ public function __toString(): string public function hashCode(): int { - return Hasher::hash($this->configs); + // Hashing through `ATNConfigSet::hashCode()` matters: that value is cached + // once the set goes read-only, and a DFA state is looked up on every + // transition. Hashing the configurations directly here re-walked the whole + // set each time. + return MurmurHash::hash([$this->configs], 7); } } diff --git a/src/InputStream.php b/src/InputStream.php index d645ca3..d7fedb0 100644 --- a/src/InputStream.php +++ b/src/InputStream.php @@ -4,10 +4,19 @@ namespace Antlr\Antlr4\Runtime; -use Antlr\Antlr4\Runtime\Utils\StringUtils; - /** * Vacuum all input from a string and then treat it like a buffer. + * + * Indexing is by Unicode code point, matching Java's `CodePointCharStream`, so + * an astral-plane character occupies one position rather than the two a UTF-16 + * stream would use. + * + * The decoding is done **once**, up front, into an array of code points. The + * lexer reads `LA()` several times per input character — it was the single + * hottest call in the runtime — and decoding on each read made every one of + * those a `mb_ord()` call. Text extraction needs the original bytes, so a byte + * offset per code point is kept alongside, letting `getText()` be a plain + * `substr()` instead of a scan. */ final class InputStream implements CharStream { @@ -19,24 +28,81 @@ final class InputStream implements CharStream public string $input; - /** @var array */ - public array $characters = []; + /** + * The decoded input, one Unicode code point per element. + * + * @var array + */ + private array $codePoints = []; /** - * @param array $characters + * Byte offset of each code point within {@see InputStream::$input}, with one + * extra entry holding the total length so a slice's end is always known. + * Empty when the input is pure ASCII, where the offset is the index itself. + * + * @var array */ - private function __construct(string $input, array $characters) + private array $byteOffsets = []; + + /** + * @param array $codePoints + * @param array $byteOffsets + */ + private function __construct(string $input, array $codePoints, array $byteOffsets) { $this->input = $input; - $this->characters = $characters; - $this->size = \count($this->characters); + $this->codePoints = $codePoints; + $this->byteOffsets = $byteOffsets; + $this->size = \count($codePoints); } public static function fromString(string $input): InputStream { - $chars = \preg_split('//u', $input, -1, \PREG_SPLIT_NO_EMPTY); + if ($input === '') { + return new self($input, [], []); + } + + $length = \strlen($input); + + // Pure ASCII is the overwhelmingly common case and needs no decoding at + // all: byte value is code point, and byte offset is index. + if (\preg_match('/[\x80-\xFF]/', $input) === 0) { + /** @var array $bytes */ + $bytes = \unpack('C*', $input); + + return new self($input, \array_values($bytes), []); + } + + // Converting the whole string at once keeps the decoding inside mbstring + // rather than paying a PHP call per character. + $utf32 = @\mb_convert_encoding($input, 'UTF-32BE', 'UTF-8'); + + if ($utf32 === '') { + return new self($input, [], []); + } + + /** @var array $unpacked */ + $unpacked = \unpack('N*', $utf32); + $codePoints = \array_values($unpacked); + + // A code point's UTF-8 width is a function of its value, so the offsets + // follow from the decoded points without re-walking the bytes. + $byteOffsets = []; + $offset = 0; + + foreach ($codePoints as $codePoint) { + $byteOffsets[] = $offset; + $offset += match (true) { + $codePoint < 0x80 => 1, + $codePoint < 0x800 => 2, + $codePoint < 0x10000 => 3, + default => 4, + }; + } + + $byteOffsets[] = $length; - return new self($input, $chars === false ? [] : $chars); + return new self($input, $codePoints, $byteOffsets); } public static function fromPath(string $path): InputStream @@ -47,7 +113,25 @@ public static function fromPath(string $path): InputStream throw new \InvalidArgumentException(\sprintf('File not found at %s.', $path)); } - return self::fromString($content); + // `CharStreams.fromPath()` records the path as the stream's source name; + // it surfaces through `Lexer::getSourceName()` and `Parser::getSourceName()`. + $stream = self::fromString($content); + $stream->name = $path; + + return $stream; + } + + /** + * The decoded code points backing this stream. + * + * Exposed so the lexer's inner loop can read characters without a method + * call per access; treat it as read-only. + * + * @return array + */ + public function getCodePoints(): array + { + return $this->codePoints; } public function getIndex(): int @@ -88,7 +172,7 @@ public function LA(int $offset): int return Token::EOF; } - return StringUtils::codePoint($this->characters[$pos]); + return $this->codePoints[$pos]; } public function LT(int $offset): int @@ -132,16 +216,27 @@ public function getText(int $start, int $stop): string $stop = $this->size - 1; } - if ($start >= $this->size) { + if ($start >= $this->size || $start > $stop) { return ''; } - return \implode(\array_slice($this->characters, $start, $stop - $start + 1)); + if ($this->byteOffsets === []) { + // ASCII: index and byte offset coincide. + return \substr($this->input, $start, $stop - $start + 1); + } + + $from = $this->byteOffsets[$start]; + + return \substr($this->input, $from, $this->byteOffsets[$stop + 1] - $from); } public function getSourceName(): string { - return ''; + // Java falls back to `IntStream.UNKNOWN_SOURCE_NAME` rather than an empty + // string, and the constant already existed here unused. + return $this->name === '' || $this->name === '' + ? IntStream::UNKNOWN_SOURCE_NAME + : $this->name; } public function __toString(): string diff --git a/src/Interval.php b/src/Interval.php index 1cbdea9..2a42ee0 100644 --- a/src/Interval.php +++ b/src/Interval.php @@ -21,11 +21,11 @@ public function __construct(int $start, int $stop) $this->stop = $stop; } + private static ?self $invalid = null; + public static function invalid(): self { - static $invalid; - - return $invalid ?? $invalid = new Interval(-1, -2); + return self::$invalid ??= new Interval(-1, -2); } public function contains(int $item): bool @@ -35,6 +35,13 @@ public function contains(int $item): bool public function getLength(): int { + // An inverted interval is empty, not negatively long. `IntervalSet` + // sums these to report its cardinality, so a negative would silently + // corrupt the total. + if ($this->stop < $this->start) { + return 0; + } + return $this->stop - $this->start + 1; } @@ -124,10 +131,8 @@ public function intersection(Interval $other): self public function __toString(): string { - if ($this->start === $this->stop) { - return (string) $this->start; - } - + // Java renders `a..b` unconditionally; collapsing a single-element + // interval to just `a` was a PHP-only shorthand. return $this->start . '..' . $this->stop; } } diff --git a/src/IntervalSet.php b/src/IntervalSet.php index 357d73f..9198a73 100644 --- a/src/IntervalSet.php +++ b/src/IntervalSet.php @@ -5,7 +5,8 @@ namespace Antlr\Antlr4\Runtime; use Antlr\Antlr4\Runtime\Comparison\Equality; -use Antlr\Antlr4\Runtime\Comparison\Equatable; +use Antlr\Antlr4\Runtime\Comparison\Hashable; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; use Antlr\Antlr4\Runtime\Utils\StringUtils; /** @@ -19,7 +20,7 @@ * the range {@see Integer::MIN_VALUE} to {@see Integer::MAX_VALUE} * (inclusive). */ -final class IntervalSet implements Equatable +final class IntervalSet implements Hashable { /** @var array */ protected array $intervals = []; @@ -323,19 +324,21 @@ public function removeOne(int $v): void } // check for upper boundary - if ($v === $i->stop - 1) { + // Java compares against `b`, not `b - 1`; testing `stop - 1` meant + // the last element of an interval could never be removed. + if ($v === $i->stop) { $this->intervals[$k] = new Interval($i->start, $i->stop - 1); return; } // split existing range - if ($v < $i->stop - 1) { - $x = new Interval($i->start, $v); - - $i->start = $v + 1; - - \array_splice($this->intervals, $k, 0, [$x]); + if ($v > $i->start && $v < $i->stop) { + // The left half ends *before* the removed element. Using `$v` as + // the stop left the element in the set. + $stop = $i->stop; + $this->intervals[$k] = new Interval($i->start, $v - 1); + $this->addRange($v + 1, $stop); return; } @@ -487,6 +490,30 @@ protected function elementName(Vocabulary $vocabulary, int $a): string return $vocabulary->getDisplayName($a); } + /** + * Java's `IntervalSet` defines `hashCode()`; the port implemented only + * `equals()`, leaving the two out of contract for any hash-based lookup. + */ + public function hashCode(): int + { + $words = []; + + foreach ($this->intervals as $interval) { + $words[] = $interval->start; + $words[] = $interval->stop; + } + + return MurmurHash::hash($words); + } + + /** + * @return array + */ + public function getIntervals(): array + { + return $this->intervals; + } + public function equals(object $other): bool { if ($this === $other) { diff --git a/src/LL1Analyzer.php b/src/LL1Analyzer.php index 57b4bbf..02a6347 100644 --- a/src/LL1Analyzer.php +++ b/src/LL1Analyzer.php @@ -54,6 +54,7 @@ public function getDecisionLookahead(?ATNState $s): ?array $look = []; for ($alt = 0; $alt < $s->getNumberOfTransitions(); $alt++) { $interval = new IntervalSet(); + /** @var Set $lookBusy */ $lookBusy = new Set(); $seeThruPreds = false; // fail to get lookahead upon pred @@ -108,12 +109,15 @@ public function look(ATNState $s, ?ATNState $stopState, ?RuleContext $context): PredictionContext::fromRuleContext($s->atn, $context) : null; + /** @var Set $lookBusy */ + $lookBusy = new Set(); + $this->lookRecursively( $s, $stopState, $lookContext, $r, - new Set(), + $lookBusy, new BitSet(), $seeThruPreds, true, @@ -142,7 +146,7 @@ public function look(ATNState $s, ?ATNState $stopState, ?RuleContext $context): * if the outer context should * not be used. * @param IntervalSet $look The result lookahead set. - * @param Set $lookBusy A set used for preventing + * @param Set $lookBusy A set used for preventing * epsilon closures in the ATN * from causing a stack overflow. * Outside code should pass @@ -170,6 +174,9 @@ public function look(ATNState $s, ?ATNState $stopState, ?RuleContext $context): * has no effect if `context` * is `null`. */ + /** + * @param Set $lookBusy + */ protected function lookRecursively( ATNState $s, ?ATNState $stopState, diff --git a/src/Lexer.php b/src/Lexer.php index f596fe5..86f196b 100644 --- a/src/Lexer.php +++ b/src/Lexer.php @@ -247,7 +247,11 @@ public function popMode(): int public function getSourceName(): string { - return $this->input === null ? '' : $this->input->getSourceName(); + // With no input Java would NPE; reporting the documented unknown-source + // sentinel is the closest honest answer. + return $this->input === null + ? IntStream::UNKNOWN_SOURCE_NAME + : $this->input->getSourceName(); } public function getInputStream(): ?IntStream diff --git a/src/Parser.php b/src/Parser.php index 2be0e4b..9157169 100644 --- a/src/Parser.php +++ b/src/Parser.php @@ -499,7 +499,7 @@ public function consume(): Token $this->tokenStream()->consume(); } - if ($this->buildParseTree || \count($this->parseListeners) > 0) { + if ($this->buildParseTree || $this->parseListeners !== []) { if ($this->errorHandler->inErrorRecoveryMode($this)) { $node = $this->context()->addErrorNode($this->createErrorNode($this->context(), $o)); @@ -564,30 +564,35 @@ public function enterRule(ParserRuleContext $localctx, int $state, int $ruleInde { $this->setState($state); $this->ctx = $localctx; - $this->context()->start = $this->tokenStream()->LT(1); + $localctx->start = $this->tokenStream()->LT(1); if ($this->buildParseTree) { $this->addContextToParseTree(); } - $this->triggerEnterRuleEvent(); + // Skipped rather than called into when there are no listeners: this runs + // once per rule invocation. + if ($this->parseListeners !== []) { + $this->triggerEnterRuleEvent(); + } } public function exitRule(): void { - if ($this->matchedEOF) { - // if we have matched EOF, it cannot consume past EOF so we use LT(1) here - $this->context()->stop = $this->tokenStream()->LT(1); // LT(1) will be end of file - } else { - $this->context()->stop = $this->tokenStream()->LT(-1); // stop node is what we just matched - } + // Resolved once: this runs on every rule exit and each accessor is a call. + $context = $this->context(); + + // if we have matched EOF, it cannot consume past EOF so we use LT(1) here + $context->stop = $this->tokenStream()->LT($this->matchedEOF ? 1 : -1); // trigger event on _ctx, before it reverts to parent - $this->triggerExitRuleEvent(); + if ($this->parseListeners !== []) { + $this->triggerExitRuleEvent(); + } - $this->setState($this->context()->invokingState); + $this->setState($context->invokingState); - $parent = $this->context()->getParent(); + $parent = $context->getParent(); if ($parent === null || $parent instanceof ParserRuleContext) { $this->ctx = $parent; @@ -905,14 +910,29 @@ public function getSourceName(): string */ public function setTrace(bool $trace): void { - if ($this->tracer !== null) { - $this->removeParseListener($this->tracer); + if (!$trace) { + if ($this->tracer !== null) { + $this->removeParseListener($this->tracer); + + // Discarded, not just detached: `isTrace()` reports on this + // reference, so keeping it would leave tracing switched off yet + // reported as on. + $this->tracer = null; + } + + return; } - if ($trace) { - $this->tracer = new ParserTraceListener($this); - $this->addParseListener($this->tracer); + $tracer = $this->tracer; + + if ($tracer === null) { + $tracer = new ParserTraceListener($this); + $this->tracer = $tracer; + } else { + $this->removeParseListener($tracer); } + + $this->addParseListener($tracer); } /** diff --git a/src/ParserTraceListener.php b/src/ParserTraceListener.php index d194b11..9caaff9 100644 --- a/src/ParserTraceListener.php +++ b/src/ParserTraceListener.php @@ -22,17 +22,19 @@ public function enterEveryRule(ParserRuleContext $context): void $stream = $this->parser->getTokenStream(); $token = $stream?->LT(1); + // Java's TraceListener uses `println`. Without the newline every trace + // line ran together on one unreadable line — issue #33. echo \sprintf( - 'enter %s, LT(1)=%s', + 'enter %s, LT(1)=%s' . \PHP_EOL, $this->parser->getRuleNames()[$context->getRuleIndex()], - $token === null? '' : $token->getText() ?? '', + $token === null ? '' : $token->getText() ?? '', ); } public function visitTerminal(TerminalNode $node): void { echo \sprintf( - 'consume %s rule %s', + 'consume %s rule %s' . \PHP_EOL, $node->getSymbol(), $this->parser->getCurrentRuleName(), ); @@ -44,9 +46,9 @@ public function exitEveryRule(ParserRuleContext $context): void $token = $stream?->LT(1); echo \sprintf( - 'exit %s, LT(1)=%s', + 'exit %s, LT(1)=%s' . \PHP_EOL, $this->parser->getRuleNames()[$context->getRuleIndex()], - $token === null? '' : $token->getText() ?? '', + $token === null ? '' : $token->getText() ?? '', ); } diff --git a/src/PredictionContexts/ArrayPredictionContext.php b/src/PredictionContexts/ArrayPredictionContext.php index a174752..7501854 100644 --- a/src/PredictionContexts/ArrayPredictionContext.php +++ b/src/PredictionContexts/ArrayPredictionContext.php @@ -5,7 +5,7 @@ namespace Antlr\Antlr4\Runtime\PredictionContexts; use Antlr\Antlr4\Runtime\Comparison\Equality; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; final class ArrayPredictionContext extends PredictionContext { @@ -86,11 +86,18 @@ public function equals(object $other): bool return false; } - if ($this->returnStates === $other->returnStates) { + // Cheap rejection first, as Java does: contexts that hash differently + // cannot be equal, and this comparison runs constantly during merging. + if (($this->cachedHashCode ?? $this->hashCode()) !== ($other->cachedHashCode ?? $other->hashCode())) { return false; } - return Equality::equals($this->parents, $other->parents); + // This condition used to be inverted — `if ($returnStates === $other->returnStates) return false;` + // — which declared identical contexts unequal and judged differing ones + // on their parents alone. It stayed hidden for as long as + // `ATNConfigSet::add()` was failing to merge contexts at all. + return $this->returnStates === $other->returnStates + && Equality::equals($this->parents, $other->parents); } public function __toString(): string @@ -125,6 +132,12 @@ public function __toString(): string protected function computeHashCode(): int { - return Hasher::hash($this->parents, $this->returnStates); + // `PredictionContext.calculateHashCode(parents, returnStates)`: every + // parent, then every return state, folded into one accumulation seeded + // with INITIAL_HASH and finished with `2 * count`. + return MurmurHash::hash( + [...$this->parents, ...$this->returnStates], + PredictionContext::INITIAL_HASH, + ); } } diff --git a/src/PredictionContexts/EmptyPredictionContext.php b/src/PredictionContexts/EmptyPredictionContext.php index 6a26f4b..3b98bc2 100644 --- a/src/PredictionContexts/EmptyPredictionContext.php +++ b/src/PredictionContexts/EmptyPredictionContext.php @@ -28,7 +28,12 @@ public function getParent(int $index): ?PredictionContext public function equals(object $other): bool { - return $other instanceof self; + // Java's is `this == o`, which is safe there because the constructor is + // private and `Instance` is the only instance. `PredictionContext::empty()` + // is the only construction site here, so identity is equally safe — and + // `instanceof self` would wrongly equate two distinct empty contexts if + // one were ever created. + return $this === $other; } public function __toString(): string diff --git a/src/PredictionContexts/PredictionContext.php b/src/PredictionContexts/PredictionContext.php index 5e2c97c..2652b8d 100644 --- a/src/PredictionContexts/PredictionContext.php +++ b/src/PredictionContexts/PredictionContext.php @@ -11,6 +11,7 @@ use Antlr\Antlr4\Runtime\LoggerProvider; use Antlr\Antlr4\Runtime\RuleContext; use Antlr\Antlr4\Runtime\Utils\DoubleKeyMap; +use Antlr\Antlr4\Runtime\Utils\Map; abstract class PredictionContext implements Hashable { @@ -26,6 +27,12 @@ abstract class PredictionContext implements Hashable */ public const EMPTY_RETURN_STATE = 0x7FFFFFFF; + /** + * The seed every prediction-context hash starts from, matching + * `PredictionContext.INITIAL_HASH` in the reference runtime. + */ + public const INITIAL_HASH = 1; + /** * Stores the computed hash code of this {@see PredictionContext}. The hash * code is computed in parts to match the following reference algorithm. @@ -53,17 +60,17 @@ public function __construct() $this->id = self::$globalNodeCount++; } + private static ?EmptyPredictionContext $empty = null; + public static function empty(): EmptyPredictionContext { - static $empty; - - if ($empty === null) { + if (self::$empty === null) { self::$globalNodeCount--; - $empty = new EmptyPredictionContext(); - $empty->id = 0; + self::$empty = new EmptyPredictionContext(); + self::$empty->id = 0; } - return $empty; + return self::$empty; } /** @@ -255,7 +262,10 @@ public static function mergeSingletons( // see if we can collapse parents due to $+x parents if local ctx $singleParent = null; - if ($a === $b || ($a->parent !== null && $a->parent === $b->parent)) { + // Java collapses on `a.parent.equals(b.parent)`; comparing by identity + // missed every equal-but-distinct parent, so `ax + bx = [a,b]x` never + // fired and the context graph grew where Java's stayed flat. + if ($a === $b || ($a->parent !== null && $b->parent !== null && $a->parent->equals($b->parent))) { // ax + // bx = // [a,b]x @@ -511,9 +521,12 @@ public static function mergeArrays( $M = new ArrayPredictionContext($mergedParents, $mergedReturnStates); - // if we created same array as a or b, return that instead + // If we created the same array as a or b, return that instead. + // Java compares with `equals()`; identity can never hold here because `M` + // was just constructed, so the fast path never fired and every merge + // allocated a fresh context. // TODO: track whether this is possible above during merge sort for speed - if ($M === $a) { + if ($M->equals($a)) { if ($mergeCache !== null) { $mergeCache->set($a, $b, $a); } @@ -529,7 +542,7 @@ public static function mergeArrays( return $a; } - if ($M === $b) { + if ($M->equals($b)) { if ($mergeCache !== null) { $mergeCache->set($a, $b, $b); } @@ -551,7 +564,9 @@ public static function mergeArrays( if (ParserATNSimulator::$traceAtnSimulation) { LoggerProvider::getLogger() - ->debug('mergeArrays a={a},b={b} -> M', [ + // `{M}` — the placeholder was missing its braces, so the trace + // printed a literal "M" instead of the merged context. + ->debug('mergeArrays a={a},b={b} -> {M}', [ 'a' => $a->__toString(), 'b' => $b->__toString(), 'M' => $M->__toString(), @@ -562,36 +577,52 @@ public static function mergeArrays( } /** - * @param array $parents + * Makes a pass over all `M` parents and merges any that are `equals()`. + * + * @param array $parents */ protected static function combineCommonParents(array &$parents): void { - $uniqueParents = new \SplObjectStorage(); + // `SplObjectStorage` keys on object *identity*, so the previous version + // could only ever collapse a parent onto itself — the canonicalisation + // never happened and equal-but-distinct parents accumulated. Java uses a + // `HashMap`, whose whole purpose here is `equals()`-based lookup. + /** @var Map $uniqueParents */ + $uniqueParents = new Map(); foreach ($parents as $parent) { - if (!$uniqueParents->contains($parent)) { - $uniqueParents[$parent] = $parent; + // Java's `HashMap` tolerates a null key; a null parent simply has + // nothing to canonicalise against. + if ($parent !== null && !$uniqueParents->contains($parent)) { + $uniqueParents->put($parent, $parent); // don't replace } } foreach ($parents as $i => $parent) { - $parents[$i] = $uniqueParents[$parent]; + if ($parent !== null) { + $parents[$i] = $uniqueParents->get($parent); + } } } /** - * @param array $visited + * @param \SplObjectStorage $visited + * an identity map, mirroring Java's `IdentityHashMap` */ public static function getCachedPredictionContext( PredictionContext $context, PredictionContextCache $contextCache, - array &$visited, + \SplObjectStorage $visited, ): self { if ($context->isEmpty()) { return $context; } - $existing = $visited[\spl_object_id($context)] ?? null; + // Identity, deliberately: this map memoises "which object did I already + // rewrite?" during one traversal. Keying it on `spl_object_id()` in a + // plain array — as this used to — is unsafe, because ids are reused once + // an object is collected, so an id could alias two different contexts. + $existing = $visited[$context] ?? null; if ($existing !== null) { return $existing; @@ -600,7 +631,7 @@ public static function getCachedPredictionContext( $existing = $contextCache->get($context); if ($existing !== null) { - $visited[\spl_object_id($context)] = $existing; + $visited[$context] = $existing; return $existing; } @@ -616,7 +647,11 @@ public static function getCachedPredictionContext( $parent = self::getCachedPredictionContext($parentContext, $contextCache, $visited); - if ($changed || !$parent->equals($parentContext)) { + // Identity, as Java has it: the cache exists to canonicalise + // instances, so "did this parent get replaced?" is an identity + // question. `equals()` would report no change for an equal-but- + // distinct instance and leave the un-canonicalised one in place. + if ($changed || $parent !== $parentContext) { if (!$changed) { $parents = []; @@ -634,7 +669,7 @@ public static function getCachedPredictionContext( if (!$changed) { $contextCache->add($context); - $visited[\spl_object_id($context)] = $context; + $visited[$context] = $context; return $context; } @@ -654,8 +689,8 @@ public static function getCachedPredictionContext( } $contextCache->add($updated); - $visited[\spl_object_id($updated)] = $updated; - $visited[\spl_object_id($context)] = $updated; + $visited[$updated] = $updated; + $visited[$context] = $updated; return $updated; } diff --git a/src/PredictionContexts/SingletonPredictionContext.php b/src/PredictionContexts/SingletonPredictionContext.php index c90aa82..6798469 100644 --- a/src/PredictionContexts/SingletonPredictionContext.php +++ b/src/PredictionContexts/SingletonPredictionContext.php @@ -4,8 +4,7 @@ namespace Antlr\Antlr4\Runtime\PredictionContexts; -use Antlr\Antlr4\Runtime\Comparison\Equality; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; /** * Used to cache {@see PredictionContext} objects. Its used for @@ -65,15 +64,34 @@ public function equals(object $other): bool return true; } - if (!$other instanceof static) { + // `self`, not `static`: Java checks `instanceof SingletonPredictionContext`, + // so a singleton and an `EmptyPredictionContext` remain comparable in both + // directions. With `static` the comparison was asymmetric. + if (!$other instanceof self) { return false; } + // Java is `returnState == s.returnState && (parent != null && parent.equals(s.parent))`, + // guarded by a hash comparison. Two null-parent singletons are + // deliberately *not* equal there, and `parent.equals(null)` is false — + // hence both null checks. + // + // The return state is compared before the hash guard: both are + // necessary conditions so the outcome is identical, but in PHP the hash + // is two method calls where the return state is an int comparison. if ($this->returnState !== $other->returnState) { return false; } - return Equality::equals($this->parent, $other->parent); + if ($this->parent === null || $other->parent === null) { + return false; + } + + if (($this->cachedHashCode ?? $this->hashCode()) !== ($other->cachedHashCode ?? $other->hashCode())) { + return false; + } + + return $this->parent->equals($other->parent); } public function __toString(): string @@ -93,10 +111,9 @@ public function __toString(): string protected function computeHashCode(): int { - if ($this->parent === null) { - return Hasher::hash(0); - } - - return Hasher::hash($this->parent, $this->returnState); + // `PredictionContext.calculateEmptyHashCode()` / `calculateHashCode()`. + return $this->parent === null + ? MurmurHash::hash([], PredictionContext::INITIAL_HASH) + : MurmurHash::hash([$this->parent, $this->returnState], PredictionContext::INITIAL_HASH); } } diff --git a/src/Recognizer.php b/src/Recognizer.php index 55d04df..cd604aa 100644 --- a/src/Recognizer.php +++ b/src/Recognizer.php @@ -17,8 +17,15 @@ abstract class Recognizer /** @var array */ public array $log = []; - /** @var array> */ - private static array $tokenTypeMapCache = []; + /** + * Java keys this on the vocabulary itself in a `WeakHashMap`, so a cached + * map dies with the vocabulary it describes. `WeakMap` is the exact + * equivalent: keying on identity like `SplObjectStorage`, but without + * pinning every vocabulary the process has ever seen in memory. + * + * @var \WeakMap>|null + */ + private static ?\WeakMap $tokenTypeMapCache = null; /** @var array */ private array $listeners; @@ -50,32 +57,41 @@ abstract public function getVocabulary(): Vocabulary; public function getTokenTypeMap(): array { $vocabulary = $this->getVocabulary(); + $cache = self::$tokenTypeMapCache; - $key = \spl_object_hash($vocabulary); - $result = self::$tokenTypeMapCache[$key] ?? null; + if ($cache === null) { + /** @var \WeakMap> $cache */ + $cache = new \WeakMap(); - if ($result === null) { - $result = []; + self::$tokenTypeMapCache = $cache; + } - for ($i = 0; $i <= $this->getATN()->maxTokenType; $i++) { - $literalName = $vocabulary->getLiteralName($i); + $cached = $cache[$vocabulary] ?? null; - if ($literalName !== null) { - $result[$literalName] = $i; - } + if ($cached !== null) { + return $cached; + } - $symbolicName = $vocabulary->getSymbolicName($i); + $result = []; - if ($symbolicName !== null) { - $result[$symbolicName] = $i; - } + for ($i = 0; $i <= $this->getATN()->maxTokenType; $i++) { + $literalName = $vocabulary->getLiteralName($i); + + if ($literalName !== null) { + $result[$literalName] = $i; } - $result['EOF'] = Token::EOF; + $symbolicName = $vocabulary->getSymbolicName($i); - self::$tokenTypeMapCache[$key] = $result; + if ($symbolicName !== null) { + $result[$symbolicName] = $i; + } } + $result['EOF'] = Token::EOF; + + $cache[$vocabulary] = $result; + return $result; } diff --git a/src/RuleContext.php b/src/RuleContext.php index 998d68b..59e2cf1 100644 --- a/src/RuleContext.php +++ b/src/RuleContext.php @@ -76,11 +76,11 @@ public function __construct(?RuleContext $parent, ?int $invokingState = null) $this->invokingState = $invokingState ?? -1; } + private static ?ParserRuleContext $emptyContext = null; + public static function emptyContext(): ParserRuleContext { - static $empty; - - return $empty ?? ($empty = new ParserRuleContext(null)); + return self::$emptyContext ??= new ParserRuleContext(null); } public function depth(): int diff --git a/src/StdoutMessageLogger.php b/src/StdoutMessageLogger.php index 58d2af4..68e3626 100644 --- a/src/StdoutMessageLogger.php +++ b/src/StdoutMessageLogger.php @@ -19,7 +19,7 @@ public function log($level, \Stringable|string $message, array $context = []): v } /** - * @param array $context + * @param array $context */ private static function formatMessage(\Stringable|string $message, array $context): string { diff --git a/src/Tree/ParseTreeWalker.php b/src/Tree/ParseTreeWalker.php index a8374e8..485f7c7 100644 --- a/src/Tree/ParseTreeWalker.php +++ b/src/Tree/ParseTreeWalker.php @@ -8,11 +8,11 @@ class ParseTreeWalker { + private static ?self $default = null; + public static function default(): self { - static $instance; - - return $instance ?? ($instance = new self()); + return self::$default ??= new self(); } public function walk(ParseTreeListener $listener, ParseTree $tree): void diff --git a/src/Utils/BitSet.php b/src/Utils/BitSet.php index c7f11af..b8bbcc4 100644 --- a/src/Utils/BitSet.php +++ b/src/Utils/BitSet.php @@ -4,7 +4,7 @@ namespace Antlr\Antlr4\Runtime\Utils; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; final class BitSet { @@ -32,27 +32,40 @@ public function contains(int $value): bool } /** + * The set bits, in ascending order. + * + * A `BitSet` is ordered by bit position by definition, but the backing array + * is keyed by bit index and so iterates in *insertion* order. Returning that + * raw order made `{2, 1}` and `{1, 2}` distinguishable — visible to users, + * because `DiagnosticErrorListener` formats an alternative set straight into + * the `ambigAlts=` of a parser message. + * * @return array */ public function values(): array { - return \array_keys($this->data); + $values = \array_keys($this->data); + + \sort($values); + + return $values; } public function minValue(): int { - $values = $this->values(); - - if (\count($values) === 0) { + if ($this->data === []) { throw new \LogicException('BitSet is empty'); } - return \min($values); + // `nextSetBit(0)`: the lowest set bit, independent of insertion order. + return \min(\array_keys($this->data)); } public function hashCode(): int { - return Hasher::hash(...$this->values()); + // Hashed over the ordered bits so that equal sets hash alike — the same + // hash/equals contract that `equals()` below now honours. + return MurmurHash::hash($this->values()); } public function equals(object $other): bool @@ -61,10 +74,22 @@ public function equals(object $other): bool return true; } - return $other instanceof self - && $this->data === $other->data; + if (!$other instanceof self) { + return false; + } + + // `===` on the backing arrays compares key *order* as well as content, so + // two identical alternative sets built in different orders compared + // unequal. Java's `BitSet.equals` is positional and order-free. + return $this->values() === $other->values(); } + /** + * The number of set bits — Java's `cardinality()`. + * + * Note this is not Java's `BitSet.length()`, which is the highest set bit + * plus one. Every call site here means cardinality. + */ public function length(): int { return \count($this->data); diff --git a/src/Utils/Map.php b/src/Utils/Map.php index 5bab247..4ec7ecb 100644 --- a/src/Utils/Map.php +++ b/src/Utils/Map.php @@ -12,6 +12,8 @@ /** * @template K of Hashable * @template V + * + * @implements \IteratorAggregate */ final class Map implements Equatable, \Countable, \IteratorAggregate { diff --git a/src/Utils/Pair.php b/src/Utils/Pair.php index 3a3c5a3..b7e800d 100644 --- a/src/Utils/Pair.php +++ b/src/Utils/Pair.php @@ -6,7 +6,7 @@ use Antlr\Antlr4\Runtime\Comparison\Equality; use Antlr\Antlr4\Runtime\Comparison\Equatable; -use Antlr\Antlr4\Runtime\Comparison\Hasher; +use Antlr\Antlr4\Runtime\Comparison\MurmurHash; final class Pair implements Equatable { @@ -33,13 +33,14 @@ public function equals(object $other): bool public function hashCode(): int { - return Hasher::hash($this->a, $this->b); + return MurmurHash::hash([$this->a, $this->b]); } public function __toString(): string { + // Java's `String.format("(%s, %s)", a, b)` includes the parentheses. return \sprintf( - '%s, %s', + '(%s, %s)', $this->a === null ? 'null' : ($this->a instanceof \Stringable ? (string) $this->a : $this->a::class), diff --git a/src/Utils/Set.php b/src/Utils/Set.php index 1072d96..6f198ad 100644 --- a/src/Utils/Set.php +++ b/src/Utils/Set.php @@ -11,6 +11,8 @@ /** * @template T of Hashable + * + * @implements \IteratorAggregate */ final class Set implements Equatable, \IteratorAggregate, \Countable { @@ -140,6 +142,16 @@ public function add(Hashable $value): bool return true; } + /** + * Empties the set while keeping its equivalence, mirroring + * `Array2DHashSet.clear()`. + */ + public function clear(): void + { + $this->table = []; + $this->size = 0; + } + /** * @param T $value */ @@ -152,7 +164,7 @@ public function remove(Hashable $value): void } foreach ($this->table[$hash] as $index => $entry) { - if ($this->equivalence->equivalent($value, $entry)) { + if (!$this->equivalence->equivalent($value, $entry)) { continue; } @@ -176,24 +188,19 @@ public function equals(object $other): bool return true; } - if (!$other instanceof self - || $this->size !== $other->size - || !$this->equivalence->equals($other)) { + if (!$other instanceof self || $this->size !== $other->size) { return false; } - foreach ($this->table as $hash => $bucket) { - if (!isset($other->table[$hash]) || \count($bucket) !== \count($other->table[$hash])) { + // Set equality is membership, not layout: `Array2DHashSet.equals()` is + // `size() == other.size() && containsAll(other)`. Walking buckets pairwise + // by index — as this used to — makes equality depend on insertion order + // and on the hash distribution, so two sets with the same elements could + // compare unequal. + foreach ($other as $value) { + if (!$this->contains($value)) { return false; } - - $otherBucket = $other->table[$hash]; - - foreach ($bucket as $index => $value) { - if (!$value->equals($otherBucket[$index])) { - return false; - } - } } return true; diff --git a/src/VocabularyImpl.php b/src/VocabularyImpl.php index 16a76eb..bb8d25e 100644 --- a/src/VocabularyImpl.php +++ b/src/VocabularyImpl.php @@ -61,11 +61,11 @@ public function __construct(array $literalNames = [], array $symbolicNames = [], * {@see Vocabulary::getDisplayName()} returns the numeric value for * all tokens except {@see Token::EOF}. */ + private static ?self $emptyVocabulary = null; + public static function emptyVocabulary(): self { - static $empty; - - return $empty ?? ($empty = new self()); + return self::$emptyVocabulary ??= new self(); } /**