diff --git a/std/regex/internal/backtracking.d b/std/regex/internal/backtracking.d index 9d3c1034244..dff6589bfab 100644 --- a/std/regex/internal/backtracking.d +++ b/std/regex/internal/backtracking.d @@ -656,12 +656,16 @@ final: } break; case IR.Backref: + case IR.BackrefI: immutable n = re.ir[pc].data; auto g = re.ir[pc].localRef ? matches[n] : backrefed[n]; if (!g) goto L_backtrack; auto referenced = s[g.begin .. g.end]; - while (!atEnd && !referenced.empty && front == referenced.front) + immutable fold = re.ir[pc].code == IR.BackrefI; + while (!atEnd && !referenced.empty + && (fold ? equalCasefold(front, referenced.front) + : front == referenced.front)) { next(); referenced.popFront(); @@ -1219,7 +1223,7 @@ struct CtContext { pc++; } - else if (ir[pc].code == IR.Backref) + else if (ir[pc].code == IR.Backref || ir[pc].code == IR.BackrefI) break; else { @@ -1448,6 +1452,25 @@ struct CtContext else $$`, gStr, bailOut, gStr, gStr, nextInstr, bailOut); break; + case IR.BackrefI: + string gStr = ir[0].localRef + ? ctSub("matches[$$]", ir[0].data) + : ctSub("backrefed[$$]", ir[0].data); + code ~= ctSub( ` + if (!$$) + $$ + auto referenced = s[$$.begin .. $$.end]; + while (!atEnd && !referenced.empty + && equalCasefold(front, referenced.front)) + { + next(); + referenced.popFront(); + } + if (referenced.empty) + $$ + else + $$`, gStr, bailOut, gStr, gStr, nextInstr, bailOut); + break; case IR.Nop: case IR.End: break; diff --git a/std/regex/internal/ir.d b/std/regex/internal/ir.d index 8a745b8b1d0..92d7cc4f5d2 100644 --- a/std/regex/internal/ir.d +++ b/std/regex/internal/ir.d @@ -107,6 +107,12 @@ package(std) string regexOptionsToString()(uint flags) nothrow pure @safe return buffer[0 .. pos].idup; } +/// True if `a` and `b` match under simple Unicode case folding (for `/i` backrefs). +bool equalCasefold(dchar lhs, dchar rhs) @safe pure nothrow @nogc +{ + return simpleCaseCmp(lhs, rhs) == 0; +} + // flags that allow guide execution of engine enum RegexInfo : uint { oneShot = 0x80 } @@ -141,6 +147,7 @@ enum IR:uint { Wordboundary = 0b1_01001_00, //boundary of a word Notwordboundary = 0b1_01010_00, //not a word boundary Backref = 0b1_01011_00, //backreference to a group (that has to be pinned, i.e. locally unique) (group index) + BackrefI = 0b1_10010_00, //case-insensitive backreference (casefold at parse position) GroupStart = 0b1_01100_00, //start of a group (x) (groupIndex+groupPinning(1bit)) GroupEnd = 0b1_01101_00, //end of a group (x) (groupIndex+groupPinning(1bit)) Option = 0b1_01110_00, //start of an option within an alternation x | y (length) @@ -324,14 +331,14 @@ struct Bytecode //mark as local reference (for backrefs in lookarounds) void setLocalRef() { - assert(code == IR.Backref); + assert(code == IR.Backref || code == IR.BackrefI); raw = raw | 1 << 23; } //is a local ref @property bool localRef() const { - assert(code == IR.Backref); + assert(code == IR.Backref || code == IR.BackrefI); return (raw & 1 << 23) != 0; } @@ -468,10 +475,10 @@ debug(std_regex_parser) @trusted string disassemble(in Bytecode[] irb, uint pc, uint start = irb[pc+1].raw, end = irb[pc+2].raw; formattedWrite(output, " pc=>%u [%u..%u]", pc + len + IRL!(IR.LookaheadStart), start, end); break; - case IR.Backref: case IR.CodepointSet: case IR.Trie: + case IR.Backref: case IR.BackrefI: case IR.CodepointSet: case IR.Trie: uint n = irb[pc].data; formattedWrite(output, " %u", n); - if (irb[pc].code == IR.Backref) + if (irb[pc].code == IR.Backref || irb[pc].code == IR.BackrefI) formattedWrite(output, " %s", irb[pc].localRef ? "local" : "global"); break; default://all data-free instructions diff --git a/std/regex/internal/parser.d b/std/regex/internal/parser.d index 0b08f5b2fa6..773d162f119 100644 --- a/std/regex/internal/parser.d +++ b/std/regex/internal/parser.d @@ -981,13 +981,15 @@ if (isForwardRange!R && is(ElementType!R : dchar)) nref /= 10; enforce(!g.isOpenGroup(nref), "Backref to open group"); uint localLimit = maxBackref - g.groupStack.top; + immutable IR backrefOp = (re_flags & RegexOption.casefold) + ? IR.BackrefI : IR.Backref; if (nref >= localLimit) { - g.put(Bytecode(IR.Backref, nref-localLimit)); + g.put(Bytecode(backrefOp, nref-localLimit)); g.ir[$-1].setLocalRef(); } else - g.put(Bytecode(IR.Backref, nref)); + g.put(Bytecode(backrefOp, nref)); g.markBackref(nref); break; default: diff --git a/std/regex/internal/tests.d b/std/regex/internal/tests.d index 6b739050d85..0a174034d31 100644 --- a/std/regex/internal/tests.d +++ b/std/regex/internal/tests.d @@ -276,6 +276,15 @@ debug(std_regex_test) import std.stdio; TestVectors( `[adzУ-Я]{4}`, "DzюЯ", "y", "$&", "DzюЯ", "i"), TestVectors( `\p{L}\p{Lu}{10}`, "абвгдеЖЗИКЛ", "y", "$&", "абвгдеЖЗИКЛ", "i"), TestVectors( `(?:Dåb){3}`, "DåbDÅBdÅb", "y", "$&", "DåbDÅBdÅb", "i"), + // https://github.com/dlang/phobos/issues/11025 backrefs must casefold under /i + TestVectors( `(.)\1`, "Aa", "y", "$&", "Aa", "i"), + TestVectors( `(.)\1`, "aA", "y", "$&", "aA", "i"), + TestVectors( `(.)\1`, "BB", "y", "$&", "BB", "i"), + TestVectors( `(.)\1`, "xy", "n", "-", "-", "i"), + TestVectors( `(ab)\1`, "AbAb", "y", "$&", "AbAb", "i"), + TestVectors( `(?i)(.)\1`, "Aa BB", "y", "$&", "Aa"), + TestVectors( `(.)\1(?i)X`, "Aax", "n", "-", "-"), + TestVectors( `(?i)(.)\1(?-i)X`, "AaX", "y", "$&", "AaX"), //escapes: TestVectors( `\u0041\u005a\U00000065\u0001`, "AZe\u0001", "y", "$&", "AZe\u0001"), TestVectors( `\u`, "", "c", "-", "-"), diff --git a/std/regex/internal/thompson.d b/std/regex/internal/thompson.d index 81d2ea73857..83b2942674e 100644 --- a/std/regex/internal/thompson.d +++ b/std/regex/internal/thompson.d @@ -511,6 +511,48 @@ template ThompsonOps(E, S, bool withInput:true) } } + static bool op(IR code:IR.BackrefI)(E e, S* state) + { + with(e) with(state) + { + uint n = re.ir[t.pc].data; + Group!DataIndex* source = re.ir[t.pc].localRef ? t.matches.ptr : backrefed.ptr; + assert(source); + if (!source[n]) // unmatched group + { + recycle(t); + t = worklist.fetch(); + return t != null; + } + if (source[n].begin == source[n].end) // zero-width match + { + t.pc += IRL!(IR.BackrefI); + return true; + } + else + { + size_t idx = source[n].begin + t.uopCounter; + size_t end = source[n].end; + if (equalCasefold(s[idx .. end].front, front)) + { + import std.utf : stride; + + t.uopCounter += stride(s[idx .. end], 0); + if (t.uopCounter + source[n].begin == source[n].end) + {//last codepoint + t.pc += IRL!(IR.BackrefI); + t.uopCounter = 0; + } + nlist.insertBack(t); + } + else + recycle(t); + t = worklist.fetch(); + return t != null; + } + } + } + static bool op(IR code)(E e, S* state) if (code == IR.LookbehindStart || code == IR.NeglookbehindStart) @@ -707,10 +749,30 @@ template ThompsonOps(E,S, bool withInput:false) } } + static bool op(IR code:IR.BackrefI)(E e, S* state) + { + with(e) with(state) + { + uint n = re.ir[t.pc].data; + Group!DataIndex* source = re.ir[t.pc].localRef ? t.matches.ptr : backrefed.ptr; + assert(source); + if (!source[n]) // unmatched group + return popState(e); + if (source[n].begin == source[n].end) // zero-width match + { + t.pc += IRL!(IR.BackrefI); + return true; + } + else + return popState(e); + } + } + // forward all control flow to normal versions static bool op(IR code)(E e, S* state) if (code != IR.Char && code != IR.OrChar && code != IR.CodepointSet - && code != IR.Trie && code != IR.Char && code != IR.Any && code != IR.Backref) + && code != IR.Trie && code != IR.Char && code != IR.Any + && code != IR.Backref && code != IR.BackrefI) { return ThompsonOps!(E,S,true).op!code(e,state); } diff --git a/std/uni/package.d b/std/uni/package.d index d09b39ccf9c..8305184d46b 100644 --- a/std/uni/package.d +++ b/std/uni/package.d @@ -8016,6 +8016,36 @@ static assert(Grapheme.sizeof == size_t.sizeof*4); int[Grapheme] aa; } +// Per-codepoint simple case compare (trie tables); used by $(LREF sicmp). +package(std) int simpleCaseCmp(dchar lhs, dchar rhs) @safe @nogc pure nothrow +{ + import std.internal.unicode_tables : sTable = simpleCaseTable; + import std.ascii : toLower; + + int diff = lhs - rhs; + if (!diff) + return 0; + if ((lhs | rhs) < 0x80) + return toLower(lhs) - toLower(rhs); + size_t idx = simpleCaseTrie[lhs]; + size_t idx2 = simpleCaseTrie[rhs]; + if (idx != EMPTY_CASE_TRIE) + { + if (idx2 != EMPTY_CASE_TRIE) + { + idx = idx - sTable(idx).n; + idx2 = idx2 - sTable(idx2).n; + if (idx == idx2) + return 0; + return sTable(idx).ch - sTable(idx2).ch; + } + return sTable(idx - sTable(idx).n).ch - rhs; + } + if (idx2 != EMPTY_CASE_TRIE) + return lhs - sTable(idx2 - sTable(idx2).n).ch; + return diff; +} + /++ $(P Does basic case-insensitive comparison of `r1` and `r2`. This function uses simpler comparison rule thus achieving better performance @@ -8043,12 +8073,10 @@ int sicmp(S1, S2)(scope S1 r1, scope S2 r2) if (isInputRange!S1 && isSomeChar!(ElementEncodingType!S1) && isInputRange!S2 && isSomeChar!(ElementEncodingType!S2)) { - import std.internal.unicode_tables : sTable = simpleCaseTable; // generated file import std.range.primitives : isInfinite; import std.utf : decodeFront; import std.traits : isDynamicArray; import std.typecons : Yes; - static import std.ascii; static if ((isDynamicArray!S1 || isRandomAccessRange!S1) && (isDynamicArray!S2 || isRandomAccessRange!S2) @@ -8074,7 +8102,7 @@ if (isInputRange!S1 && isSomeChar!(ElementEncodingType!S1) auto rhs = r2[i]; if ((lhs | rhs) >= 0x80) goto NonAsciiPath; if (lhs == rhs) continue; - auto lowDiff = std.ascii.toLower(lhs) - std.ascii.toLower(rhs); + immutable lowDiff = simpleCaseCmp(lhs, rhs); if (lowDiff) return lowDiff; } static if (isInfinite!S1) @@ -8096,38 +8124,9 @@ if (isInputRange!S1 && isSomeChar!(ElementEncodingType!S1) if (r2.empty) return 1; immutable rhs = decodeFront!(Yes.useReplacementDchar)(r2); - int diff = lhs - rhs; + immutable diff = simpleCaseCmp(lhs, rhs); if (!diff) continue; - if ((lhs | rhs) < 0x80) - { - immutable d = std.ascii.toLower(lhs) - std.ascii.toLower(rhs); - if (!d) continue; - return d; - } - size_t idx = simpleCaseTrie[lhs]; - size_t idx2 = simpleCaseTrie[rhs]; - // simpleCaseTrie is packed index table - if (idx != EMPTY_CASE_TRIE) - { - if (idx2 != EMPTY_CASE_TRIE) - {// both cased chars - // adjust idx --> start of bucket - idx = idx - sTable(idx).n; - idx2 = idx2 - sTable(idx2).n; - if (idx == idx2)// one bucket, equivalent chars - continue; - else// not the same bucket - diff = sTable(idx).ch - sTable(idx2).ch; - } - else - diff = sTable(idx - sTable(idx).n).ch - rhs; - } - else if (idx2 != EMPTY_CASE_TRIE) - { - diff = lhs - sTable(idx2 - sTable(idx2).n).ch; - } - // one of chars is not cased at all return diff; } return int(r2.empty) - 1;