Merge pull request #90 from tildeio/add-mode

Add codemod mode to support `<pre>\nhi</pre>` stability in codemods.

Author: Robert Jackson

src/evented-tokenizer.ts

@@ -14,7 +14,8 @@ export default class EventedTokenizer {
 
   constructor(
     private delegate: TokenizerDelegate,
-    private entityParser: EntityParser
+    private entityParser: EntityParser,
+    private mode: 'codemod' | 'precompile' = 'precompile'
   ) {
     this.reset();
   }
@@ -131,7 +132,7 @@ export default class EventedTokenizer {
         this.markTagStart();
         this.consume();
       } else {
-        if (char === '\n') {
+        if (this.mode === 'precompile' && char === '\n') {
           let tag = this.tagNameBuffer.toLowerCase();
 
           if (tag === 'pre' || tag === 'textarea') {

src/tokenizer.ts

@@ -21,7 +21,7 @@ export default class Tokenizer implements TokenizerDelegate {
     entityParser: EntityParser,
     private options: TokenizerOptions = {}
   ) {
-    this.tokenizer = new EventedTokenizer(this, entityParser);
+    this.tokenizer = new EventedTokenizer(this, entityParser, options.mode);
     this._currentAttribute = undefined;
   }
 

src/types.ts

@@ -8,6 +8,7 @@ export interface EntityParser {
 
 export interface TokenizerOptions {
   loc?: boolean;
+  mode?: 'precompile' | 'codemod';
 }
 
 export type Attribute = [string, string, boolean];

tests/tokenizer-tests.ts

@@ -222,6 +222,29 @@ QUnit.test('A newline immediately following a <textarea> tag is stripped', funct
   assert.deepEqual(tokens, [startTag('textarea'), chars('hello'), endTag('textarea')]);
 });
 
+// https://html.spec.whatwg.org/multipage/syntax.html#element-restrictions
+QUnit.test('codemod: A newline immediately following a <pre> tag is stripped', function(assert) {
+  let tokens = tokenize("<pre>\nhello</pre>", { mode: 'codemod' });
+  assert.deepEqual(tokens, [startTag('pre'), chars('\nhello'), endTag('pre')]);
+});
+
+QUnit.test('codemod: A newline immediately following a closing </pre> tag is not stripped', function(assert) {
+  let tokens = tokenize("\n<pre>\nhello</pre>\n", { mode: 'codemod' });
+  assert.deepEqual(tokens, [chars('\n'), startTag('pre'), chars('\nhello'), endTag('pre'), chars('\n')]);
+});
+
+// https://html.spec.whatwg.org/multipage/syntax.html#element-restrictions
+QUnit.test('codemod: A newline immediately following a <PRE> tag is stripped', function(assert) {
+  let tokens = tokenize("<PRE>\nhello</PRE>", { mode: 'codemod' });
+  assert.deepEqual(tokens, [startTag('PRE'), chars('\nhello'), endTag('PRE')]);
+});
+
+// https://html.spec.whatwg.org/multipage/syntax.html#element-restrictions
+QUnit.test('codemod: A newline immediately following a <textarea> tag is stripped', function(assert) {
+  let tokens = tokenize("<textarea>\nhello</textarea>", { mode: 'codemod' });
+  assert.deepEqual(tokens, [startTag('textarea'), chars('\nhello'), endTag('textarea')]);
+});
+
 // https://html.spec.whatwg.org/multipage/semantics.html#the-title-element
 QUnit.test('The title element content is always text', function(assert) {
   let tokens = tokenize("<title>&quot;hey <b>there</b><!-- comment --></title>");