Avoid a potential buffer overrun if an SQL statement being parsed ends

with an illegal "!" token.  (This problem was detected by fuzzcheck
running under valgrind. The problem was introduced by check-in [9570b6b43df3].)

FossilOrigin-Name: 2a8d97e7c8976df0312e1294e8c1da8b15686654

manifest

@@ -1,5 +1,5 @@
-C Improved\shandling\sof\sthe\s-v\soption\son\sthe\sfuzzcheck\stest\sprogram.
-D 2016-02-18T14:03:15.183
+C Avoid\sa\spotential\sbuffer\soverrun\sif\san\sSQL\sstatement\sbeing\sparsed\sends\nwith\san\sillegal\s"!"\stoken.\s\s(This\sproblem\swas\sdetected\sby\sfuzzcheck\nrunning\sunder\svalgrind.\sThe\sproblem\swas\sintroduced\sby\scheck-in\s[9570b6b43df3].)
+D 2016-02-18T14:49:28.741
 F Makefile.in 4e90dc1521879022aa9479268a4cd141d1771142
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 30f075dc4f27a07abb76088946b2944178d85347
@@ -407,7 +407,7 @@ F src/test_windirent.c 8f5fada630348558d5745b334702f301da1ffc61
 F src/test_windirent.h b12055cab6227f7be10f5c19296f67c60cc5e2a5
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
-F src/tokenize.c 32aeca12f0d57a5c1c9a88d63e46ed2ee795cdb4
+F src/tokenize.c c4c1d360fafa3dc458fcbb535691b134798dbb70
 F src/treeview.c dc39ccf04e9331237388b9cb73289c9d87ea050b
 F src/trigger.c e14840ee0c3e549e758ec9bf3e4146e166002280
 F src/update.c a7eeeaffad59c6506f01303a071dac11de8269ca
@@ -765,7 +765,7 @@ F test/fuzz2.test 76dc35b32b6d6f965259508508abce75a6c4d7e1
 F test/fuzz3.test b47377143f0c80f91ed29d722861077ff34415d5
 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b
 F test/fuzz_malloc.test 328f70aaca63adf29b4c6f06505ed0cf57ca7c26
-F test/fuzzcheck.c 19782d888c5542afe16d5c9336192761f38ea70b
+F test/fuzzcheck.c 93bb9d309888634615e21ef98d1c30d51483e942
 F test/fuzzdata1.db 7ee3227bad0e7ccdeb08a9e6822916777073c664
 F test/fuzzdata2.db f03a420d3b822cc82e4f894ca957618fbe9c4973
 F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba
@@ -856,7 +856,7 @@ F test/lock6.test ad5b387a3a8096afd3c68a55b9535056431b0cf5
 F test/lock7.test 49f1eaff1cdc491cc5dee3669f3c671d9f172431
 F test/lock_common.tcl 7ffb45accf6ee91c736df9bafe0806a44358f035
 F test/lookaside.test 90052e87282de256d613fcf8c9cbb845e4001d2f
-F test/main.test 16131264ea0c2b93b95201f0c92958e85f2ba11a
+F test/main.test bb75e406c9b64931f3dc7e7f04626633365bb22f
 F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9
 F test/malloc.test 21c213365f2cca95ab2d7dc078dc8525f96065f8
 F test/malloc3.test e3b32c724b5a124b57cb0ed177f675249ad0c66a
@@ -890,7 +890,7 @@ F test/minmax.test 42fbad0e81afaa6e0de41c960329f2b2c3526efd
 F test/minmax2.test b44bae787fc7b227597b01b0ca5575c7cb54d3bc
 F test/minmax3.test cc1e8b010136db0d01a6f2a29ba5a9f321034354
 F test/minmax4.test 936941484ebdceb8adec7c86b6cd9b6e5e897c1f
-F test/misc1.test 48ebfb5b22a6a058f7b7e1df211226dd1d21409c
+F test/misc1.test 6430dabfb4b4fa480633590118964201f94d3ccc
 F test/misc2.test 00d7de54eda90e237fc9a38b9e5ccc769ebf6d4d
 F test/misc3.test cf3dda47d5dda3e53fc5804a100d3c82be736c9d
 F test/misc4.test 0d8be3466adf123a7791a66ba2bc8e8d229e87f3
@@ -1428,7 +1428,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh a98af506df552f3b3c0d904f94e4cdc4e1a6d598
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 31d8b69e9e0747e573516570bfe2770384e99134
-R a027f18f6ed81f6dba546149a0b77304
+P c8cd7804dc905b2b20cd7c0192bcfaceaaa7e2a8
+R ff7407a00ef53a788829701876392bcd
 U drh
-Z 2e72dfb6b81d85b4231b0f2b20f67f3c
+Z e801ae846fbe7280611432828d271852

manifest.uuid

@@ -1 +1 @@
-c8cd7804dc905b2b20cd7c0192bcfaceaaa7e2a8
+2a8d97e7c8976df0312e1294e8c1da8b15686654

src/tokenize.c

@@ -285,7 +285,7 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){
     case CC_BANG: {
       if( z[1]!='=' ){
         *tokenType = TK_ILLEGAL;
-        return 2;
+        return 1;
       }else{
         *tokenType = TK_NE;
         return 2;

test/fuzzcheck.c

@@ -70,6 +70,7 @@
 #include <stdarg.h>
 #include <ctype.h>
 #include "sqlite3.h"
+#include <assert.h>
 #define ISSPACE(X) isspace((unsigned char)(X))
 #define ISDIGIT(X) isdigit((unsigned char)(X))
 
@@ -621,12 +622,14 @@ static void inmemVfsRegister(void){
 */
 static void runSql(sqlite3 *db, const char *zSql, unsigned  runFlags){
   const char *zMore;
+  const char *zEnd = &zSql[strlen(zSql)];
   sqlite3_stmt *pStmt;
 
   while( zSql && zSql[0] ){
     zMore = 0;
     pStmt = 0;
     sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zMore);
+    assert( zMore<=zEnd );
     if( zMore==zSql ) break;
     if( runFlags & SQL_TRACE ){
       const char *z = zSql;

test/main.test

@@ -319,7 +319,7 @@ do_test main-3.1 {
   sqlite3 db testdb
   set v [catch {execsql {SELECT * from T1 where x!!5}} msg]
   lappend v $msg
-} {1 {unrecognized token: "!!"}}
+} {1 {unrecognized token: "!"}}
 do_test main-3.2 {
   catch {db close}
   foreach f [glob -nocomplain testdb/*] {forcedelete $f}

test/misc1.test

@@ -699,7 +699,7 @@ do_catchsql_test misc1-23.3 {
 #
 do_test misc1-24.0 {
   list [catch { sqlite3_prepare_v2 db ! -1 dummy } msg] $msg
-} {1 {(1) unrecognized token: "!}}
+} {1 {(1) unrecognized token: "!"}}
 
 # The following query (provided by Kostya Serebryany) used to take 25
 # minutes to prepare.  This has been speeded up to about 250 milliseconds.