[searchcursor addon] Support multi-line regexp matching

Author: marijnh

addon/search/search.js

@@ -54,7 +54,7 @@
 
   function getSearchCursor(cm, query, pos) {
     // Heuristic: if the query string is all lowercase, do a case insensitive search.
-    return cm.getSearchCursor(query, pos, queryCaseInsensitive(query));
+    return cm.getSearchCursor(query, pos, {caseFold: queryCaseInsensitive(query), multiline: true});
   }
 
   function persistentDialog(cm, text, deflt, onEnter, onKeyDown) {

addon/search/searchcursor.js

@@ -12,8 +12,19 @@
   "use strict"
   var Pos = CodeMirror.Pos
 
+  function regexpFlags(regexp) {
+    var flags = regexp.flags
+    return flags != null ? flags : (regexp.ignoreCase ? "i" : "")
+      + (regexp.global ? "g" : "")
+      + (regexp.multiline ? "m" : "")
+  }
+
   function ensureGlobal(regexp) {
-    return regexp.global ? regexp : new RegExp(regexp.source, regexp.ignoreCase ? "ig" : "g")
+    return regexp.global ? regexp : new RegExp(regexp.source, regexpFlags(regexp) + "g")
+  }
+
+  function maybeMultiline(regexp) {
+    return /\\s|\\n|\n|\\W|\\D|\[\^/.test(regexp.source)
   }
 
   function searchRegexpForward(doc, regexp, start) {
@@ -28,27 +39,82 @@
     }
   }
 
+  function searchRegexpForwardMultiline(doc, regexp, start) {
+    if (!maybeMultiline(regexp)) return searchRegexpForward(doc, regexp, start)
+
+    regexp = ensureGlobal(regexp)
+    var string, chunk = 1
+    for (var line = start.line, last = doc.lastLine(); line <= last;) {
+      // This grows the search buffer in exponentially-sized chunks
+      // between matches, so that nearby matches are fast and don't
+      // require concatenating the whole document (in case we're
+      // searching for something that has tons of matches), but at the
+      // same time, the amount of retries is limited.
+      for (var i = 0; i < chunk; i++) {
+        var curLine = doc.getLine(line++)
+        string = string == null ? curLine : string + "\n" + curLine
+      }
+      chunk = chunk * 2
+      regexp.lastIndex = start.ch
+      var match = regexp.exec(string)
+      if (match && match[0].length) {
+        var before = string.slice(0, match.index).split("\n"), inside = match[0].split("\n")
+        var startLine = start.line + before.length - 1, startCh = before[before.length - 1].length
+        return {from: Pos(startLine, startCh),
+                to: Pos(startLine + inside.length - 1,
+                        inside.length == 1 ? startCh + inside[0].length : inside[inside.length - 1].length),
+                match: match}
+      }
+    }
+  }
+
+  function lastMatchIn(string, regexp) {
+    var cutOff = 0, match
+    for (;;) {
+      regexp.lastIndex = cutOff
+      var newMatch = regexp.exec(string)
+      if (!newMatch) return match
+      match = newMatch
+      cutOff = match.index + (match[0].length || 1)
+      if (cutOff == string.length) return match
+    }
+  }
+
   function searchRegexpBackward(doc, regexp, start) {
     regexp = ensureGlobal(regexp)
     for (var line = start.line, ch = start.ch, first = doc.firstLine(); line >= first; line--, ch = -1) {
-      var string = doc.getLine(line), cutOff = 0, match
+      var string = doc.getLine(line)
       if (ch > -1) string = string.slice(0, ch)
-      for (;;) {
-        regexp.lastIndex = cutOff
-        var newMatch = regexp.exec(string)
-        if (!newMatch) break
-        match = newMatch
-        cutOff = match.index + (match[0].length || 1)
-        if (cutOff == line.length) break
-      }
-
+      var match = lastMatchIn(string, regexp)
       if (match && match[0].length)
         return {from: Pos(line, match.index),
                 to: Pos(line, match.index + match[0].length),
                 match: match}
     }
   }
 
+  function searchRegexpBackwardMultiline(doc, regexp, start) {
+    regexp = ensureGlobal(regexp)
+    var string, chunk = 1
+    for (var line = start.line, first = doc.firstLine(); line >= first;) {
+      for (var i = 0; i < chunk; i++) {
+        var curLine = doc.getLine(line--)
+        string = string == null ? curLine.slice(0, start.ch) : curLine + "\n" + string
+      }
+      chunk *= 2
+
+      var match = lastMatchIn(string, regexp)
+      if (match && match[0].length) {
+        var before = string.slice(0, match.index).split("\n"), inside = match[0].split("\n")
+        var startLine = line + before.length, startCh = before[before.length - 1].length
+        return {from: Pos(startLine, startCh),
+                to: Pos(startLine + inside.length - 1,
+                        inside.length == 1 ? startCh + inside[0].length : inside[inside.length - 1].length),
+                match: match}
+      }
+    }
+  }
+
   function doFold(str) { return str.toLowerCase() }
   function noFold(str) { return str }
 
@@ -119,22 +185,35 @@
     }
   }
 
-  function SearchCursor(doc, query, pos, caseFold) {
+  function SearchCursor(doc, query, pos, options) {
     this.atOccurrence = false
     this.doc = doc
     pos = pos ? doc.clipPos(pos) : Pos(0, 0)
     this.pos = {from: pos, to: pos}
 
+    var caseFold
+    if (typeof options == "object") {
+      caseFold = options.caseFold
+    } else { // Backwards compat for when caseFold was the 4th argument
+      caseFold = options
+      options = null
+    }
+
     if (typeof query == "string") {
       if (caseFold == null) caseFold = false
       this.matches = function(reverse, pos) {
         return (reverse ? searchStringBackward : searchStringForward)(doc, query, pos, caseFold)
       }
     } else {
       query = ensureGlobal(query)
-      this.matches = function(reverse, pos) {
-        return (reverse ? searchRegexpBackward : searchRegexpForward)(doc, query, pos)
-      }
+      if (!options || options.multiline !== false)
+        this.matches = function(reverse, pos) {
+          return (reverse ? searchRegexpBackwardMultiline : searchRegexpForwardMultiline)(doc, query, pos)
+        }
+      else
+        this.matches = function(reverse, pos) {
+          return (reverse ? searchRegexpBackward : searchRegexpForward)(doc, query, pos)
+        }
     }
   }
 

doc/manual.html

@@ -2263,16 +2263,18 @@ <h2 id="addons">Addons</h2>
       <p>Depends on <code>addon/dialog/dialog.css</code>.</p></dd>
 
       <dt id="addon_searchcursor"><a href="../addon/search/searchcursor.js"><code>search/searchcursor.js</code></a></dt>
-      <dd>Adds the <code>getSearchCursor(query, start, caseFold) →
+      <dd>Adds the <code>getSearchCursor(query, start, options) →
       cursor</code> method to CodeMirror instances, which can be used
       to implement search/replace functionality. <code>query</code>
-      can be a regular expression or a string (only strings will match
-      across lines—if they contain newlines). <code>start</code>
+      can be a regular expression or a string. <code>start</code>
       provides the starting position of the search. It can be
       a <code>{line, ch}</code> object, or can be left off to default
-      to the start of the document. <code>caseFold</code> is only
-      relevant when matching a string. It will cause the search to be
-      case-insensitive. A search cursor has the following methods:
+      to the start of the document. <code>options</code> is an
+      optional object, which can contain the property `caseFold:
+      false` to disable case folding when mathing a string, or the
+      property `multiline: disable` to disable multi-line matching for
+      regular expressions (which may help performance). A search
+      cursor has the following methods:
         <dl>
           <dt><code><strong>findNext</strong>() → boolean</code></dt>
           <dt><code><strong>findPrevious</strong>() → boolean</code></dt>

test/search_test.js

@@ -1,16 +1,8 @@
 (function() {
   "use strict";
 
-  function test(name) {
-    var text = Array.prototype.slice.call(arguments, 1, arguments.length - 1).join("\n");
-    var body = arguments[arguments.length - 1];
-    return window.test("search_" + name, function() {
-      body(new CodeMirror.Doc(text));
-    });
-  }
-
-  function run(doc, query, insensitive) {
-    var cursor = doc.getSearchCursor(query, null, insensitive);
+  function run(doc, query, options) {
+    var cursor = doc.getSearchCursor(query, null, options);
     for (var i = 3; i < arguments.length; i += 4) {
       var found = cursor.findNext();
       is(found, "not enough results (forward)");
@@ -27,35 +19,59 @@
     is(!cursor.findPrevious(), "too many matches (backwards)");
   }
 
-  test("simple", "abcdefg", "abcdefg", function(doc) {
+  function test(name, f) { window.test("search_" + name, f) }
+
+  test("simple", function() {
+    var doc = new CodeMirror.Doc("abcdefg\nabcdefg")
     run(doc, "cde", false, 0, 2, 0, 5, 1, 2, 1, 5);
   });
 
-  test("multiline", "hallo", "a", "b", "goodbye", function(doc) {
+  test("multiline", function() {
+    var doc = new CodeMirror.Doc("hallo\na\nb\ngoodbye")
     run(doc, "llo\na\nb\ngoo", false, 0, 2, 3, 3);
     run(doc, "blah\na\nb\nhall", false);
     run(doc, "bye\nx\neye", false);
   });
 
-  test("regexp", "abcde", "abcde", function(doc) {
+  test("regexp", function() {
+    var doc = new CodeMirror.Doc("abcde\nabcde")
     run(doc, /bcd/, false, 0, 1, 0, 4, 1, 1, 1, 4);
     run(doc, /BCD/, false);
     run(doc, /BCD/i, false, 0, 1, 0, 4, 1, 1, 1, 4);
   });
 
-  test("insensitive", "hallo", "HALLO", "oink", "hAllO", function(doc) {
+  test("regexpMultiline", function() {
+    var doc = new CodeMirror.Doc("foo foo\nbar\nbaz")
+    run(doc, /fo[^]*az/, {multiline: true}, 0, 0, 2, 3)
+    run(doc, /[oa][^u]/, {multiline: true}, 0, 1, 0, 3, 0, 5, 0, 7, 1, 1, 1, 3, 2, 1, 2, 3)
+    run(doc, /[a][^u]{2}/, {multiline: true}, 1, 1, 2, 0)
+  })
+
+  test("insensitive", function() {
+    var doc = new CodeMirror.Doc("hallo\nHALLO\noink\nhAllO")
     run(doc, "All", false, 3, 1, 3, 4);
     run(doc, "All", true, 0, 1, 0, 4, 1, 1, 1, 4, 3, 1, 3, 4);
   });
 
-  test("multilineInsensitive", "zie ginds komT", "De Stoomboot", "uit Spanje weer aan", function(doc) {
+  test("multilineInsensitive", function() {
+    var doc = new CodeMirror.Doc("zie ginds komT\nDe Stoomboot\nuit Spanje weer aan")
     run(doc, "komt\nde stoomboot\nuit", false);
-    run(doc, "komt\nde stoomboot\nuit", true, 0, 10, 2, 3);
-    run(doc, "kOMt\ndE stOOmboot\nuiT", true, 0, 10, 2, 3);
+    run(doc, "komt\nde stoomboot\nuit", {caseFold: true}, 0, 10, 2, 3);
+    run(doc, "kOMt\ndE stOOmboot\nuiT", {caseFold: true}, 0, 10, 2, 3);
   });
 
-  test("expandingCaseFold", "<b>İİ İİ</b>", "<b>uu uu</b>", function(doc) {
+  test("multilineInsensitiveSlow", function() {
+    var text = ""
+    for (var i = 0; i < 1000; i++) text += "foo\nbar\n"
+    var doc = new CodeMirror.Doc("find\nme\n" + text + "find\nme\n")
+    var t0 = +new Date
+    run(doc, /find\nme/, {multiline: true}, 0, 0, 1, 2, 2002, 0, 2003, 2)
+    is(+new Date - t0 < 100)
+  })
+
+  test("expandingCaseFold", function() {
     if (phantom) return; // A Phantom bug makes this hang
+    var doc = new CodeMirror.Doc("<b>İİ İİ</b>\n<b>uu uu</b>")
     run(doc, "</b>", true, 0, 8, 0, 12, 1, 8, 1, 12);
     run(doc, "İİ", true, 0, 3, 0, 5, 0, 6, 0, 8);
   });