Fix missing null checks in uses of consumeIdentOrUrlOrFunctions (#266)

CssTokens code assumed that consumeIdentOrUrlOrFunctions always
returned a token type and consumed characters.

This commit audits all uses of that function and checks that
they make progress.

Author: mikesamuel

src/main/java/org/owasp/html/CssTokens.java

@@ -453,7 +453,7 @@ void lex() {
         char ch = css.charAt(pos);
         int startOfToken = pos;
         int startOfOutputToken = sb.length();
-        final TokenType type;
+        TokenType type;
         switch (ch) {
           case '\t': case '\n': case '\f': case '\r': case ' ': case '\ufeff':
             consumeIgnorable();
@@ -514,6 +514,7 @@ void lex() {
               type = TokenType.UNICODE_RANGE;
             } else {
               type = consumeIdentOrUrlOrFunction();
+              assert type != null;
             }
             break;
           case '0': case '1': case '2': case '3': case '4':
@@ -533,7 +534,14 @@ && isDecimal(css.charAt(pos + 2)))) {
               if (consumeIgnorable()) {  // -->
                 type = TokenType.WHITESPACE;
               } else {
-                type = consumeIdentOrUrlOrFunction();
+                TokenType identType = consumeIdentOrUrlOrFunction();
+                if (identType == null) {
+                  breakOutput();
+                  consumeDelim(ch);
+                  type = TokenType.DELIM;
+                } else {
+                  type = identType;
+                }
               }
             } else if (isIdentPart(lookahead)) {
               // treat ".<IDENT>" as one token.
@@ -589,9 +597,17 @@ && isDecimal(css.charAt(pos + 2)))) {
             }
             break;
           }
-          case '_':
-            type = consumeIdentOrUrlOrFunction();
+          case '_': {
+            TokenType identType = consumeIdentOrUrlOrFunction();
+            if (identType != null) {
+              type = identType;
+            } else {
+              ++pos;  // drop
+              breakOutput();
+              type = TokenType.WHITESPACE;
+            }
             break;
+          }
           case '\\': {
             // Optimistically parse as an ident.
             TokenType identType = consumeIdentOrUrlOrFunction();
@@ -624,7 +640,13 @@ && isDecimal(css.charAt(pos + 2)))) {
               type = TokenType.WHITESPACE;
             }
         }
-        assert pos > startOfToken
+        // Make progress even in the face of errors above.
+        if (type == null && pos == startOfToken) {
+          type = TokenType.WHITESPACE;
+          breakOutput();
+          ++pos;
+        }
+        assert type != null && pos > startOfToken
             : "empty token at " + pos + ", ch0=" + css.charAt(startOfToken)
             + ":U+" + Integer.toHexString(css.charAt(startOfToken));
         int endOfOutputToken = sb.length();

src/test/java/org/owasp/html/HtmlSanitizerTest.java

@@ -447,6 +447,13 @@ public static final void testIssue254SemicolonlessNamedCharactersInUrls() {
     assertEquals(want, sanitize(input));
   }
 
+  @Test
+  public static final void testStylingCornerCase() {
+    String input = "<a style=\\006-\\000038";
+    String want = "";
+    assertEquals(want, sanitize(input));
+  }
+
   private static String sanitize(@Nullable String html) {
     StringBuilder sb = new StringBuilder();
     HtmlStreamRenderer renderer = HtmlStreamRenderer.create(