msi: Remove tokens that aren't valid for MSI SQL.

dlls/msi/sql.y

@@ -79,40 +79,17 @@ static struct expr * EXPR_wildcard( void *info );
     int integer;
 }
 
-%token TK_ABORT TK_AFTER TK_AGG_FUNCTION TK_ALL TK_ALTER TK_AND TK_AS TK_ASC
+%token TK_ALTER TK_AND TK_BY TK_CHAR TK_COMMA TK_CREATE TK_DELETE
-%token TK_BEFORE TK_BEGIN TK_BETWEEN TK_BITAND TK_BITNOT TK_BITOR TK_BY
+%token TK_DISTINCT TK_DOT TK_EQ TK_FREE TK_FROM TK_GE TK_GT TK_HOLD
-%token TK_CASCADE TK_CASE TK_CHAR TK_CHECK TK_CLUSTER TK_COLLATE TK_COLUMN
+%token <str> TK_ID
-%token TK_COMMA TK_COMMENT TK_COMMIT TK_CONCAT TK_CONFLICT
+%token TK_ILLEGAL TK_INSERT TK_INT
-%token TK_CONSTRAINT TK_COPY TK_CREATE
-%token TK_DEFAULT TK_DEFERRABLE TK_DEFERRED TK_DELETE TK_DELIMITERS TK_DESC
-%token TK_DISTINCT TK_DOT TK_DROP TK_EACH
-%token TK_ELSE TK_END TK_END_OF_FILE TK_EQ TK_EXCEPT TK_EXPLAIN
-%token TK_FAIL TK_FLOAT TK_FOR TK_FOREIGN TK_FREE TK_FROM TK_FUNCTION
-%token TK_GE TK_GLOB TK_GROUP TK_GT
-%token TK_HAVING TK_HOLD
-%token TK_IGNORE TK_ILLEGAL TK_IMMEDIATE TK_IN TK_INDEX TK_INITIALLY
-%token <str> TK_ID 
-%token TK_INSERT TK_INSTEAD TK_INT 
 %token <str> TK_INTEGER
-%token TK_INTERSECT TK_INTO TK_IS
+%token TK_INTO TK_IS TK_KEY TK_LE TK_LONG TK_LONGCHAR TK_LP TK_LT
-%token TK_ISNULL
+%token TK_LOCALIZABLE TK_MINUS TK_NE TK_NOT TK_NULL
-%token TK_JOIN TK_JOIN_KW
+%token TK_OBJECT TK_OR TK_ORDER TK_PRIMARY TK_RP
-%token TK_KEY
+%token TK_SELECT TK_SET TK_SHORT TK_SPACE TK_STAR
-%token TK_LE TK_LIKE TK_LIMIT TK_LONG TK_LONGCHAR TK_LP TK_LSHIFT TK_LT
-%token TK_LOCALIZABLE
-%token TK_MATCH TK_MINUS
-%token TK_NE TK_NOT TK_NOTNULL TK_NULL
-%token TK_OBJECT TK_OF TK_OFFSET TK_ON TK_OR TK_ORACLE_OUTER_JOIN TK_ORDER
-%token TK_PLUS TK_PRAGMA TK_PRIMARY
-%token TK_RAISE TK_REFERENCES TK_REM TK_REPLACE TK_RESTRICT TK_ROLLBACK
-%token TK_ROW TK_RP TK_RSHIFT
-%token TK_SELECT TK_SEMI TK_SET TK_SHORT TK_SLASH TK_SPACE TK_STAR TK_STATEMENT 
 %token <str> TK_STRING
-%token TK_TABLE TK_TEMPORARY TK_THEN TK_TRANSACTION TK_TRIGGER
+%token TK_TABLE TK_TEMPORARY TK_UPDATE TK_VALUES TK_WHERE TK_WILDCARD
-%token TK_UMINUS TK_UNCLOSED_STRING TK_UNION TK_UNIQUE
-%token TK_UPDATE TK_UPLUS TK_USING
-%token TK_VACUUM TK_VALUES TK_VIEW
-%token TK_WHEN TK_WHERE TK_WILDCARD
 
 /*
  * These are extra tokens used by the lexer but never seen by the
@@ -136,8 +113,7 @@ static struct expr * EXPR_wildcard( void *info );
 %left TK_OR
 %left TK_AND
 %left TK_NOT
-%left TK_EQ TK_NE TK_LT TK_GT TK_LE TK_GE TK_ISNULL TK_LIKE TK_BETWEEN TK_IN
+%left TK_EQ TK_NE TK_LT TK_GT TK_LE TK_GE TK_LIKE
-%left TK_PLUS TK_MINUS TK_CONCAT
 %right TK_NEGATION
 
 %%
@@ -163,10 +139,10 @@ oneinsert:
     TK_INSERT TK_INTO table TK_LP selcollist TK_RP TK_VALUES TK_LP constlist TK_RP
         {
             SQL_input *sql = (SQL_input*) info;
-            MSIVIEW *insert = NULL; 
+            MSIVIEW *insert = NULL;
             UINT r;
 
-            r = INSERT_CreateView( sql->db, &insert, $3, $5, $9, FALSE ); 
+            r = INSERT_CreateView( sql->db, &insert, $3, $5, $9, FALSE );
             if( !insert )
                 YYABORT;
             $$ = insert;
@@ -174,9 +150,9 @@ oneinsert:
   | TK_INSERT TK_INTO table TK_LP selcollist TK_RP TK_VALUES TK_LP constlist TK_RP TK_TEMPORARY
         {
             SQL_input *sql = (SQL_input*) info;
-            MSIVIEW *insert = NULL; 
+            MSIVIEW *insert = NULL;
 
-            INSERT_CreateView( sql->db, &insert, $3, $5, $9, TRUE ); 
+            INSERT_CreateView( sql->db, &insert, $3, $5, $9, TRUE );
             if( !insert )
                 YYABORT;
             $$ = insert;
@@ -187,7 +163,7 @@ onecreate:
     TK_CREATE TK_TABLE table TK_LP table_def TK_RP
         {
             SQL_input* sql = (SQL_input*) info;
-            MSIVIEW *create = NULL; 
+            MSIVIEW *create = NULL;
 
             if( !$5 )
                 YYABORT;
@@ -199,7 +175,7 @@ onecreate:
   | TK_CREATE TK_TABLE table TK_LP table_def TK_RP TK_HOLD
         {
             SQL_input* sql = (SQL_input*) info;
-            MSIVIEW *create = NULL; 
+            MSIVIEW *create = NULL;
 
             if( !$5 )
                 YYABORT;
@@ -237,7 +213,7 @@ onedelete:
     TK_DELETE from
         {
             SQL_input* sql = (SQL_input*) info;
-            MSIVIEW *delete = NULL; 
+            MSIVIEW *delete = NULL;
 
             DELETE_CreateView( sql->db, &delete, $2 );
             if( !delete )
@@ -408,7 +384,7 @@ unorderedsel:
     ;
 
 selectfrom:
-    selcollist multifrom 
+    selcollist multifrom
         {
             SQL_input* sql = (SQL_input*) info;
             UINT r;
@@ -429,9 +405,9 @@ selectfrom:
     ;
 
 selcollist:
-    column 
+    column
   | column TK_COMMA selcollist
-        { 
+        {
             $1->next = $3;
         }
   | TK_STAR
@@ -457,7 +433,7 @@ multifrom:
 from:
     fromtable
   | fromtable TK_WHERE expr
-        { 
+        {
             SQL_input* sql = (SQL_input*) info;
             UINT r;
 
@@ -621,7 +597,7 @@ const_val:
     ;
 
 column_val:
-    column 
+    column
         {
             $$ = EXPR_column( info, $1 );
             if( !$$ )
@@ -717,7 +693,7 @@ static int sql_lex( void *SQL_lval, SQL_input *sql )
     while( token == TK_SPACE );
 
     /* TRACE("token : %d (%s)\n", token, debugstr_wn(&sql->command[sql->n], sql->len)); */
-    
+
     return token;
 }
 
@@ -728,7 +704,7 @@ LPWSTR SQL_getstring( void *info, struct sql_str *strdata )
     LPWSTR str;
 
     /* if there's quotes, remove them */
-    if( ( (p[0]=='`') && (p[len-1]=='`') ) || 
+    if( ( (p[0]=='`') && (p[len-1]=='`') ) ||
         ( (p[0]=='\'') && (p[len-1]=='\'') ) )
     {
         p++;

dlls/msi/tokenize.c

@@ -38,227 +38,89 @@ struct Keyword {
 
 #define MAX_TOKEN_LEN 11
 
-static const WCHAR ABORT_W[] = { 'A','B','O','R','T',0 };
-static const WCHAR AFTER_W[] = { 'A','F','T','E','R',0 };
-static const WCHAR ALL_W[] = { 'A','L','L',0 };
 static const WCHAR ALTER_W[] = { 'A','L','T','E','R',0 };
 static const WCHAR AND_W[] = { 'A','N','D',0 };
-static const WCHAR AS_W[] = { 'A','S',0 };
-static const WCHAR ASC_W[] = { 'A','S','C',0 };
-static const WCHAR BEFORE_W[] = { 'B','E','F','O','R','E',0 };
-static const WCHAR BEGIN_W[] = { 'B','E','G','I','N','W',0 };
-static const WCHAR BETWEEN_W[] = { 'B','E','T','W','E','E','N',0 };
 static const WCHAR BY_W[] = { 'B','Y',0 };
-static const WCHAR CASCADE_W[] = { 'C','A','S','C','A','D','E',0 };
-static const WCHAR CASE_W[] = { 'C','A','S','E',0 };
 static const WCHAR CHAR_W[] = { 'C','H','A','R',0 };
 static const WCHAR CHARACTER_W[] = { 'C','H','A','R','A','C','T','E','R',0 };
-static const WCHAR CHECK_W[] = { 'C','H','E','C','K',0 };
-static const WCHAR CLUSTER_W[] = { 'C','L','U','S','T','E','R',0 };
-static const WCHAR COLLATE_W[] = { 'C','O','L','L','A','T','E',0 };
-static const WCHAR COMMIT_W[] = { 'C','O','M','M','I','T',0 };
-static const WCHAR CONFLICT_W[] = { 'C','O','N','F','L','I','C','T',0 };
-static const WCHAR CONSTRAINT_W[] = { 'C','O','N','S','T','R','A','I','N','T',0 };
-static const WCHAR COPY_W[] = { 'C','O','P','Y',0 };
 static const WCHAR CREATE_W[] = { 'C','R','E','A','T','E',0 };
 static const WCHAR CROSS_W[] = { 'C','R','O','S','S',0 };
-static const WCHAR DEFAULT_W[] = { 'D','E','F','A','U','L','T',0 };
-static const WCHAR DEFERRED_W[] = { 'D','E','F','E','R','R','E','D',0 };
-static const WCHAR DEFERRABLE_W[] = { 'D','E','F','E','R','R','A','B','L','E',0 };
 static const WCHAR DELETE_W[] = { 'D','E','L','E','T','E',0 };
-static const WCHAR DELIMITERS_W[] = { 'D','E','L','I','M','I','T','E','R','S',0 };
-static const WCHAR DESC_W[] = { 'D','E','S','C',0 };
 static const WCHAR DISTINCT_W[] = { 'D','I','S','T','I','N','C','T',0 };
-static const WCHAR DROP_W[] = { 'D','R','O','P',0 };
-static const WCHAR END_W[] = { 'E','N','D',0 };
-static const WCHAR EACH_W[] = { 'E','A','C','H',0 };
-static const WCHAR ELSE_W[] = { 'E','L','S','E',0 };
-static const WCHAR EXCEPT_W[] = { 'E','X','C','E','P','T',0 };
-static const WCHAR EXPLAIN_W[] = { 'E','X','P','L','A','I','N',0 };
-static const WCHAR FAIL_W[] = { 'F','A','I','L',0 };
-static const WCHAR FOR_W[] = { 'F','O','R',0 };
-static const WCHAR FOREIGN_W[] = { 'F','O','R','E','I','G','N',0 };
 static const WCHAR FREE_W[] = { 'F','R','E','E',0 };
 static const WCHAR FROM_W[] = { 'F','R','O','M',0 };
 static const WCHAR FULL_W[] = { 'F','U','L','L',0 };
-static const WCHAR GLOB_W[] = { 'G','L','O','B',0 };
-static const WCHAR GROUP_W[] = { 'G','R','O','U','P',0 };
-static const WCHAR HAVING_W[] = { 'H','A','V','I','N','G',0 };
 static const WCHAR HOLD_W[] = { 'H','O','L','D',0 };
-static const WCHAR IGNORE_W[] = { 'I','G','N','O','R','E',0 };
-static const WCHAR IMMEDIATE_W[] = { 'I','M','M','E','D','I','A','T','E',0 };
-static const WCHAR IN_W[] = { 'I','N',0 };
-static const WCHAR INDEX_W[] = { 'I','N','D','E','X',0 };
-static const WCHAR INITIALLY_W[] = { 'I','N','I','T','I','A','L','L','Y',0 };
 static const WCHAR INNER_W[] = { 'I','N','N','E','R',0 };
 static const WCHAR INSERT_W[] = { 'I','N','S','E','R','T',0 };
-static const WCHAR INSTEAD_W[] = { 'I','N','S','T','E','A','D',0 };
 static const WCHAR INT_W[] = { 'I','N','T',0 };
 static const WCHAR INTEGER_W[] = { 'I','N','T','E','G','E','R',0 };
-static const WCHAR INTERSECT_W[] = { 'I','N','T','E','R','S','E','C','T',0 };
 static const WCHAR INTO_W[] = { 'I','N','T','O',0 };
 static const WCHAR IS_W[] = { 'I','S',0 };
-static const WCHAR ISNULL_W[] = { 'I','S','N','U','L','L',0 };
 static const WCHAR JOIN_W[] = { 'J','O','I','N',0 };
 static const WCHAR KEY_W[] = { 'K','E','Y',0 };
 static const WCHAR LEFT_W[] = { 'L','E','F','T',0 };
 static const WCHAR LIKE_W[] = { 'L','I','K','E',0 };
-static const WCHAR LIMIT_W[] = { 'L','I','M','I','T',0 };
 static const WCHAR LOCALIZABLE_W[] = { 'L','O','C','A','L','I','Z','A','B','L','E',0 };
 static const WCHAR LONG_W[] = { 'L','O','N','G',0 };
 static const WCHAR LONGCHAR_W[] = { 'L','O','N','G','C','H','A','R',0 };
-static const WCHAR MATCH_W[] = { 'M','A','T','C','H',0 };
 static const WCHAR NATURAL_W[] = { 'N','A','T','U','R','A','L',0 };
 static const WCHAR NOT_W[] = { 'N','O','T',0 };
-static const WCHAR NOTNULL_W[] = { 'N','O','T','N','U','L','L',0 };
 static const WCHAR NULL_W[] = { 'N','U','L','L',0 };
 static const WCHAR OBJECT_W[] = { 'O','B','J','E','C','T',0 };
-static const WCHAR OF_W[] = { 'O','F',0 };
-static const WCHAR OFFSET_W[] = { 'O','F','F','S','E','T',0 };
-static const WCHAR ON_W[] = { 'O','N',0 };
 static const WCHAR OR_W[] = { 'O','R',0 };
 static const WCHAR ORDER_W[] = { 'O','R','D','E','R',0 };
 static const WCHAR OUTER_W[] = { 'O','U','T','E','R',0 };
-static const WCHAR PRAGMA_W[] = { 'P','R','A','G','M','A',0 };
 static const WCHAR PRIMARY_W[] = { 'P','R','I','M','A','R','Y',0 };
-static const WCHAR RAISE_W[] = { 'R','A','I','S','E',0 };
-static const WCHAR REFERENCES_W[] = { 'R','E','F','E','R','E','N','C','E','S',0 };
-static const WCHAR REPLACE_W[] = { 'R','E','P','L','A','C','E',0 };
-static const WCHAR RESTRICT_W[] = { 'R','E','S','T','R','I','C','T',0 };
 static const WCHAR RIGHT_W[] = { 'R','I','G','H','T',0 };
-static const WCHAR ROLLBACK_W[] = { 'R','O','L','L','B','A','C','K',0 };
-static const WCHAR ROW_W[] = { 'R','O','W',0 };
 static const WCHAR SELECT_W[] = { 'S','E','L','E','C','T',0 };
 static const WCHAR SET_W[] = { 'S','E','T',0 };
 static const WCHAR SHORT_W[] = { 'S','H','O','R','T',0 };
-static const WCHAR STATEMENT_W[] = { 'S','T','A','T','E','M','E','N','T',0 };
 static const WCHAR TABLE_W[] = { 'T','A','B','L','E',0 };
 static const WCHAR TEMP_W[] = { 'T','E','M','P',0 };
 static const WCHAR TEMPORARY_W[] = { 'T','E','M','P','O','R','A','R','Y',0 };
-static const WCHAR THEN_W[] = { 'T','H','E','N',0 };
-static const WCHAR TRANSACTION_W[] = { 'T','R','A','N','S','A','C','T','I','O','N',0 };
-static const WCHAR TRIGGER_W[] = { 'T','R','I','G','G','E','R',0 };
-static const WCHAR UNION_W[] = { 'U','N','I','O','N',0 };
-static const WCHAR UNIQUE_W[] = { 'U','N','I','Q','U','E',0 };
 static const WCHAR UPDATE_W[] = { 'U','P','D','A','T','E',0 };
-static const WCHAR USING_W[] = { 'U','S','I','N','G',0 };
-static const WCHAR VACUUM_W[] = { 'V','A','C','U','U','M',0 };
 static const WCHAR VALUES_W[] = { 'V','A','L','U','E','S',0 };
-static const WCHAR VIEW_W[] = { 'V','I','E','W',0 };
-static const WCHAR WHEN_W[] = { 'W','H','E','N',0 };
 static const WCHAR WHERE_W[] = { 'W','H','E','R','E',0 };
 
 /*
 ** These are the keywords
 */
 static const Keyword aKeywordTable[] = {
-  { ABORT_W, TK_ABORT },
-  { AFTER_W, TK_AFTER },
-  { ALL_W, TK_ALL },
   { ALTER_W, TK_ALTER },
   { AND_W, TK_AND },
-  { AS_W, TK_AS },
-  { ASC_W, TK_ASC },
-  { BEFORE_W, TK_BEFORE },
-  { BEGIN_W, TK_BEGIN },
-  { BETWEEN_W, TK_BETWEEN },
   { BY_W, TK_BY },
-  { CASCADE_W, TK_CASCADE },
-  { CASE_W, TK_CASE },
   { CHAR_W, TK_CHAR },
   { CHARACTER_W, TK_CHAR },
-  { CHECK_W, TK_CHECK },
-  { CLUSTER_W, TK_CLUSTER },
-  { COLLATE_W, TK_COLLATE },
-  { COMMIT_W, TK_COMMIT },
-  { CONFLICT_W, TK_CONFLICT },
-  { CONSTRAINT_W, TK_CONSTRAINT },
-  { COPY_W, TK_COPY },
   { CREATE_W, TK_CREATE },
-  { CROSS_W, TK_JOIN_KW },
-  { DEFAULT_W, TK_DEFAULT },
-  { DEFERRED_W, TK_DEFERRED },
-  { DEFERRABLE_W, TK_DEFERRABLE },
   { DELETE_W, TK_DELETE },
-  { DELIMITERS_W, TK_DELIMITERS },
-  { DESC_W, TK_DESC },
   { DISTINCT_W, TK_DISTINCT },
-  { DROP_W, TK_DROP },
-  { END_W, TK_END },
-  { EACH_W, TK_EACH },
-  { ELSE_W, TK_ELSE },
-  { EXCEPT_W, TK_EXCEPT },
-  { EXPLAIN_W, TK_EXPLAIN },
-  { FAIL_W, TK_FAIL },
-  { FOR_W, TK_FOR },
-  { FOREIGN_W, TK_FOREIGN },
   { FREE_W, TK_FREE },
   { FROM_W, TK_FROM },
-  { FULL_W, TK_JOIN_KW },
-  { GLOB_W, TK_GLOB },
-  { GROUP_W, TK_GROUP },
-  { HAVING_W, TK_HAVING },
   { HOLD_W, TK_HOLD },
-  { IGNORE_W, TK_IGNORE },
-  { IMMEDIATE_W, TK_IMMEDIATE },
-  { IN_W, TK_IN },
-  { INDEX_W, TK_INDEX },
-  { INITIALLY_W, TK_INITIALLY },
-  { INNER_W, TK_JOIN_KW },
   { INSERT_W, TK_INSERT },
-  { INSTEAD_W, TK_INSTEAD },
   { INT_W, TK_INT },
   { INTEGER_W, TK_INT },
-  { INTERSECT_W, TK_INTERSECT },
   { INTO_W, TK_INTO },
   { IS_W, TK_IS },
-  { ISNULL_W, TK_ISNULL },
-  { JOIN_W, TK_JOIN },
   { KEY_W, TK_KEY },
-  { LEFT_W, TK_JOIN_KW },
   { LIKE_W, TK_LIKE },
-  { LIMIT_W, TK_LIMIT },
   { LOCALIZABLE_W, TK_LOCALIZABLE },
   { LONG_W, TK_LONG },
   { LONGCHAR_W, TK_LONGCHAR },
-  { MATCH_W, TK_MATCH },
-  { NATURAL_W, TK_JOIN_KW },
   { NOT_W, TK_NOT },
-  { NOTNULL_W, TK_NOTNULL },
   { NULL_W, TK_NULL },
   { OBJECT_W, TK_OBJECT },
-  { OF_W, TK_OF },
-  { OFFSET_W, TK_OFFSET },
-  { ON_W, TK_ON },
   { OR_W, TK_OR },
   { ORDER_W, TK_ORDER },
-  { OUTER_W, TK_JOIN_KW },
-  { PRAGMA_W, TK_PRAGMA },
   { PRIMARY_W, TK_PRIMARY },
-  { RAISE_W, TK_RAISE },
-  { REFERENCES_W, TK_REFERENCES },
-  { REPLACE_W, TK_REPLACE },
-  { RESTRICT_W, TK_RESTRICT },
-  { RIGHT_W, TK_JOIN_KW },
-  { ROLLBACK_W, TK_ROLLBACK },
-  { ROW_W, TK_ROW },
   { SELECT_W, TK_SELECT },
   { SET_W, TK_SET },
   { SHORT_W, TK_SHORT },
-  { STATEMENT_W, TK_STATEMENT },
   { TABLE_W, TK_TABLE },
   { TEMPORARY_W, TK_TEMPORARY },
-  { THEN_W, TK_THEN },
-  { TRANSACTION_W, TK_TRANSACTION },
-  { TRIGGER_W, TK_TRIGGER },
-  { UNION_W, TK_UNION },
-  { UNIQUE_W, TK_UNIQUE },
   { UPDATE_W, TK_UPDATE },
-  { USING_W, TK_USING },
-  { VACUUM_W, TK_VACUUM },
   { VALUES_W, TK_VALUES },
-  { VIEW_W, TK_VIEW },
-  { WHEN_W, TK_WHEN },
   { WHERE_W, TK_WHERE },
 };
 
@@ -342,57 +204,21 @@ int sqliteGetToken(const WCHAR *z, int *tokenType){
     }
     case '-': {
       if( z[1]==0 ) return -1;
-      if( z[1]=='-' ){
-        for(i=2; z[i] && z[i]!='\n'; i++){}
-        *tokenType = TK_COMMENT;
-        return i;
-      }
       *tokenType = TK_MINUS;
       return 1;
     }
-    case '(': {
+    case '(':
-      if( z[1]=='+' && z[2]==')' ){
+      *tokenType = TK_LP;
-        *tokenType = TK_ORACLE_OUTER_JOIN;
+      return 1;
-        return 3;
+    case ')':
-      }else{
-        *tokenType = TK_LP;
-        return 1;
-      }
-    }
-    case ')': {
       *tokenType = TK_RP;
       return 1;
-    }
+    case '*':
-    case ';': {
-      *tokenType = TK_SEMI;
-      return 1;
-    }
-    case '+': {
-      *tokenType = TK_PLUS;
-      return 1;
-    }
-    case '*': {
       *tokenType = TK_STAR;
       return 1;
-    }
+    case '=':
-    case '/': {
-      if( z[1]!='*' || z[2]==0 ){
-        *tokenType = TK_SLASH;
-        return 1;
-      }
-      for(i=3; z[i] && (z[i]!='/' || z[i-1]!='*'); i++){}
-      if( z[i] ) i++;
-      *tokenType = TK_COMMENT;
-      return i;
-    }
-    case '%': {
-      *tokenType = TK_REM;
-      return 1;
-    }
-    case '=': {
       *tokenType = TK_EQ;
-      return 1 + (z[1]=='=');
+      return 1;
-    }
     case '<': {
       if( z[1]=='=' ){
         *tokenType = TK_LE;
@@ -400,9 +226,6 @@ int sqliteGetToken(const WCHAR *z, int *tokenType){
       }else if( z[1]=='>' ){
         *tokenType = TK_NE;
         return 2;
-      }else if( z[1]=='<' ){
-        *tokenType = TK_LSHIFT;
-        return 2;
       }else{
         *tokenType = TK_LT;
         return 1;
@@ -412,9 +235,6 @@ int sqliteGetToken(const WCHAR *z, int *tokenType){
       if( z[1]=='=' ){
         *tokenType = TK_GE;
         return 2;
-      }else if( z[1]=='>' ){
-        *tokenType = TK_RSHIFT;
-        return 2;
       }else{
         *tokenType = TK_GT;
         return 1;
@@ -429,31 +249,12 @@ int sqliteGetToken(const WCHAR *z, int *tokenType){
         return 2;
       }
     }
-    case '|': {
+    case '?':
-      if( z[1]!='|' ){
-        *tokenType = TK_BITOR;
-        return 1;
-      }else{
-        *tokenType = TK_CONCAT;
-        return 2;
-      }
-    }
-    case '?': {
       *tokenType = TK_WILDCARD;
       return 1;
-    }
+    case ',':
-    case ',': {
       *tokenType = TK_COMMA;
       return 1;
-    }
-    case '&': {
-      *tokenType = TK_BITAND;
-      return 1;
-    }
-    case '~': {
-      *tokenType = TK_BITNOT;
-      return 1;
-    }
     case '`': case '\'': case '"': {
       int delim = z[0];
       for(i=1; z[i]; i++){
@@ -483,22 +284,6 @@ int sqliteGetToken(const WCHAR *z, int *tokenType){
     case '5': case '6': case '7': case '8': case '9': {
       *tokenType = TK_INTEGER;
       for(i=1; isdigit(z[i]); i++){}
-      if( z[i]=='.' ){
-        i++;
-        while( isdigit(z[i]) ){ i++; }
-        *tokenType = TK_FLOAT;
-      }
-      if( (z[i]=='e' || z[i]=='E') &&
-           ( isdigit(z[i+1]) 
-            || ((z[i+1]=='+' || z[i+1]=='-') && isdigit(z[i+2]))
-           )
-      ){
-        i += 2;
-        while( isdigit(z[i]) ){ i++; }
-        *tokenType = TK_FLOAT;
-      }else if( z[0]=='.' ){
-        *tokenType = TK_FLOAT;
-      }
       return i;
     }
     case '[': {