Lexer parses bit values.

Author: udan11

src/Lexer.php

@@ -559,26 +559,33 @@ public function parseNumber()
             // Below are the states of the machines and the conditions to change
             // the state.
             //
-            //      1 ---------------------[ + or - ]---------------------> 1
-            //      1 --------------------[ 0x or 0X ]--------------------> 2
-            //      1 ---------------------[ 0 to 9 ]---------------------> 3
-            //      1 ------------------------[ . ]-----------------------> 4
+            //      1 --------------------[ + or - ]-------------------> 1
+            //      1 -------------------[ 0x or 0X ]------------------> 2
+            //      1 --------------------[ 0 to 9 ]-------------------> 3
+            //      1 -----------------------[ . ]---------------------> 4
+            //      1 -----------------------[ b ]---------------------> 7
             //
-            //      2 ---------------------[ 0 to F ]---------------------> 2
+            //      2 --------------------[ 0 to F ]-------------------> 2
             //
-            //      3 ---------------------[ 0 to 9 ]---------------------> 3
-            //      3 ------------------------[ . ]-----------------------> 4
-            //      3 ---------------------[ e or E ]---------------------> 5
+            //      3 --------------------[ 0 to 9 ]-------------------> 3
+            //      3 -----------------------[ . ]---------------------> 4
+            //      3 --------------------[ e or E ]-------------------> 5
             //
-            //      4 ---------------------[ 0 to 9 ]---------------------> 4
-            //      4 ---------------------[ e or E ]---------------------> 5
+            //      4 --------------------[ 0 to 9 ]-------------------> 4
+            //      4 --------------------[ e or E ]-------------------> 5
             //
-            //      5 ----------------[ + or - or 0 to 9 ]----------------> 6
+            //      5 ---------------[ + or - or 0 to 9 ]--------------> 6
+            //
+            //      7 -----------------------[ ' ]---------------------> 8
+            //
+            //      8 --------------------[ 0 or 1 ]-------------------> 8
+            //      8 -----------------------[ ' ]---------------------> 9
             //
             // State 1 may be reached by negative numbers.
             // State 2 is reached only by hex numbers.
             // State 4 is reached only by float numbers.
             // State 5 is reached only by numbers in approximate form.
+            // State 7 is reached only by numbers in bit representation.
             //
             // Valid final states are: 2, 3, 4 and 6. Any parsing that finished in a
             // state other than these is invalid.
@@ -590,15 +597,19 @@ public function parseNumber()
                 if ($state === 1) {
                     if ($this->str[$this->last] === '-') {
                         $flags |= Token::FLAG_NUMBER_NEGATIVE;
-                    } elseif (($this->str[$this->last] === '0') && ($this->last + 1 < $this->len)
-                        && (($this->str[$this->last + 1] === 'x') || ($this->str[$this->last + 1] === 'X'))
+                    } elseif (($this->last + 1 < $this->len)
+                        && ($this->str[$this->last] === '0')
+                        && (($this->str[$this->last + 1] === 'x')
+                            || ($this->str[$this->last + 1] === 'X'))
                     ) {
                         $token .= $this->str[$this->last++];
                         $state = 2;
                     } elseif (($this->str[$this->last] >= '0') && ($this->str[$this->last] <= '9')) {
                         $state = 3;
                     } elseif ($this->str[$this->last] === '.') {
                         $state = 4;
+                    } elseif ($this->str[$this->last] === 'b') {
+                        $state = 7;
                     } elseif ($this->str[$this->last] !== '+') {
                         // `+` is a valid character in a number.
                         break;
@@ -642,10 +653,30 @@ public function parseNumber()
                         // Just digits are valid characters.
                         break;
                     }
+                } elseif ($state === 7) {
+                    $flags |= Token::FLAG_NUMBER_BINARY;
+                    if ($this->str[$this->last] === '\'') {
+                        $state = 8;
+                    } else {
+                        break;
+                    }
+                } elseif ($state === 8) {
+                    if ($this->str[$this->last] === '\'') {
+                        $state = 9;
+                    } elseif (($this->str[$this->last] !== '0')
+                        && ($this->str[$this->last] !== '1')
+                    ) {
+                        break;
+                    }
+                } elseif ($state === 9) {
+                    break;
                 }
                 $token .= $this->str[$this->last];
             }
-            if (($state === 2) || ($state === 3) || (($token !== '.') && ($state === 4)) || ($state === 6)) {
+            if (($state === 2) || ($state === 3)
+                || (($token !== '.') && ($state === 4))
+                || ($state === 6) || ($state === 9)
+            ) {
                 --$this->last;
                 return new Token($token, Token::TYPE_NUMBER, $flags);
             }

src/Token.php

@@ -134,6 +134,7 @@ class Token
     const FLAG_NUMBER_FLOAT             =  2;
     const FLAG_NUMBER_APPROXIMATE       =  4;
     const FLAG_NUMBER_NEGATIVE          =  8;
+    const FLAG_NUMBER_BINARY            = 16;
 
     // Strings related flags.
     const FLAG_STRING_SINGLE_QUOTES     =  1;

tests/data/lexer/lexNumber.in

@@ -1,3 +1,3 @@
-SELECT 12, 34, 5.67, 0x89, -10, --11, +12, .15, 0xFFa, 0xfFA, 0XFfA, 1e-10, 1e10, .5e10;
+SELECT 12, 34, 5.67, 0x89, -10, --11, +12, .15, 0xFFa, 0xfFA, 0XFfA, 1e-10, 1e10, .5e10, b'10';
 -- invalid number
-SELECT 12ex10;
+SELECT 12ex10, b'15';