Allow the left-hand side of IN operators on virtual tables to have the

aConstraintUsage[].omit flag clear.

FossilOrigin-Name: 1622623cbbfc4325c53d731aba78ca9c382ec612

Makefile.in

@@ -1050,6 +1050,7 @@ sqlite3rbu.lo:	$(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR)
 TESTFIXTURE_FLAGS  = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
 TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE 
 TESTFIXTURE_FLAGS += -DBUILD_sqlite
+TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
 
 TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la
 TESTFIXTURE_SRC1 = sqlite3.c

Makefile.msc

@@ -1925,6 +1925,7 @@ sqlite3rbu.lo:	$(TOP)\ext\rbu\sqlite3rbu.c $(HDR) $(EXTHDR)
 TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE=""
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CORE $(NO_WARN)
+TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
 
 TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2)
 TESTFIXTURE_SRC1 = $(TESTEXT) $(SQLITE3C)

ext/misc/series.c

@@ -217,12 +217,12 @@ static int seriesEof(sqlite3_vtab_cursor *cur){
   }
 }
 
-/* True to omit run-time checking of the start=, stop=, and/or step= 
-** parameters.  The only reason to not omit these is for testing the
+/* True to cause run-time checking of the start=, stop=, and/or step= 
+** parameters.  The only reason to do this is for testing the
 ** constraint checking logic for virtual tables in the SQLite core.
 */
-#ifndef SERIES_OMIT_CONSTRAINT_VERIFY
-# define SERIES_OMIT_CONSTRAINT_VERIFY 1
+#ifndef SQLITE_SERIES_CONSTRAINT_VERIFY
+# define SQLITE_SERIES_CONSTRAINT_VERIFY 0
 #endif
 
 /*
@@ -332,15 +332,15 @@ static int seriesBestIndex(
   }
   if( startIdx>=0 ){
     pIdxInfo->aConstraintUsage[startIdx].argvIndex = ++nArg;
-    pIdxInfo->aConstraintUsage[startIdx].omit = SERIES_OMIT_CONSTRAINT_VERIFY;
+    pIdxInfo->aConstraintUsage[startIdx].omit= !SQLITE_SERIES_CONSTRAINT_VERIFY;
   }
   if( stopIdx>=0 ){
     pIdxInfo->aConstraintUsage[stopIdx].argvIndex = ++nArg;
-    pIdxInfo->aConstraintUsage[stopIdx].omit = SERIES_OMIT_CONSTRAINT_VERIFY;
+    pIdxInfo->aConstraintUsage[stopIdx].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY;
   }
   if( stepIdx>=0 ){
     pIdxInfo->aConstraintUsage[stepIdx].argvIndex = ++nArg;
-    pIdxInfo->aConstraintUsage[stepIdx].omit = SERIES_OMIT_CONSTRAINT_VERIFY;
+    pIdxInfo->aConstraintUsage[stepIdx].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY;
   }
   if( (idxNum & 3)==3 ){
     /* Both start= and stop= boundaries are available.  This is the 

main.mk

@@ -726,7 +726,8 @@ sqlite3_analyzer$(EXE): sqlite3_analyzer.c
 # Rules to build the 'testfixture' application.
 #
 TESTFIXTURE_FLAGS  = -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
-TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE 
+TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE
+TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
 
 testfixture$(EXE): $(TESTSRC2) libsqlite3.a $(TESTSRC) $(TOP)/src/tclsqlite.c
 	$(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS)                  \

manifest

@@ -1,8 +1,8 @@
-C Add\sthe\soptional\s-DSERIES_OMIT_CONSTRAINT_VERIFY=0\soption\sto\sthe\sseries.c\nextension\sthat\simplements\sthe\sgenerate_series()\svirtual\stable.
-D 2016-03-02T00:58:49.570
-F Makefile.in 4e90dc1521879022aa9479268a4cd141d1771142
+C Allow\sthe\sleft-hand\sside\sof\sIN\soperators\son\svirtual\stables\sto\shave\sthe\naConstraintUsage[].omit\sflag\sclear.
+D 2016-03-02T03:28:07.978
+F Makefile.in e335453db0b16da00c884ad51bb56d1c091a74de
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 4f319afb7c049d40aff7af6e8c4e7cc2ba18e079
+F Makefile.msc dbd4621ecc585c2ef0c2aa0874698c54675754f1
 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7
 F VERSION c6b1f51809551d60ad001e6d87cf3ab2c7f54b6f
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -212,7 +212,7 @@ F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342
 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63
 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc
 F ext/misc/rot13.c 1ac6f95f99b575907b9b09c81a349114cf9be45a
-F ext/misc/series.c e1ef8bc23328d4e2196835737f62b324bdcd1c0d
+F ext/misc/series.c e11e534ada797d5b816d7e7a93c022306563ca35
 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52
 F ext/misc/spellfix.c 525190484b7a9dbc6be646c4842274fff4f27d53
 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512
@@ -273,7 +273,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 31027cdd40130bdbb3a0862bb97a8c837c5987d9
+F main.mk 518d93f9f606d515628f99ce03f9e909f4f8a2e3
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -429,9 +429,9 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 10deb6b43887662691e5f53d10b3c171c401169b
 F src/wal.h 2f7c831cf3b071fa548bf2d5cac640846a7ff19c
 F src/walker.c 0f142b5bd3ed2041fc52d773880748b212e63354
-F src/where.c 6282e53fcc612a8918262f1432d6fd7fbc24af40
+F src/where.c 56948ada5aacc3bf2628db3776986e8bf4085383
 F src/whereInt.h 93297d56edd137b7ea004490690fb6e2ce028a34
-F src/wherecode.c 39c1ef4598bedf1d66249334c74efd23ddd182ac
+F src/wherecode.c 3ca820435c5b597bb50e63ed11e938786fe5c23e
 F src/whereexpr.c fb87944b1254234e5bba671aaf6dee476241506a
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd
@@ -1074,7 +1074,7 @@ F test/symlink.test c9ebe7330d228249e447038276bfc8a7b22f4849
 F test/sync.test 2f607e1821aa3af3c5c53b58835c05e511c95899
 F test/syscall.test f59ba4e25f7ba4a4c031026cc2ef8b6e4b4c639c
 F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04
-F test/tabfunc01.test cc33684f9480fcf1fd5ce287ac28d22971cad1cc
+F test/tabfunc01.test f977868fa8bb7beb4b2072883190411653473906
 F test/table.test b708f3e5fa2542fa51dfab21fc07b36ea445cb2f
 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126
 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930
@@ -1453,7 +1453,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 1d41c161165006d6c2af47e476f05fb13039f8b8
-R 7bc50a1ee335d764bdfff403c9ca0d5d
+P 3d9daa929c0abe6dc01e800ef343b0eef2f0c76a
+R c059a6ee769a0309e913921e43679d8c
+T *branch * vtab-IN-opt
+T *sym-vtab-IN-opt *
+T -sym-trunk *
 U drh
-Z d249d93e08651d834742ccb923f02a0e
+Z f36b44043e8c4337262495e2c87317b9

manifest.uuid

@@ -1 +1 @@
-3d9daa929c0abe6dc01e800ef343b0eef2f0c76a
+1622623cbbfc4325c53d731aba78ca9c382ec612

src/where.c

@@ -2898,13 +2898,6 @@ static int whereLoopAddVirtual(
         testcase( iTerm==16 );
         if( iTerm<16 && pUsage[i].omit ) pNew->u.vtab.omitMask |= 1<<iTerm;
         if( (pTerm->eOperator & WO_IN)!=0 ){
-          if( pUsage[i].omit==0 ){
-            /* Do not attempt to use an IN constraint if the virtual table
-            ** says that the equivalent EQ constraint cannot be safely omitted.
-            ** If we do attempt to use such a constraint, some rows might be
-            ** repeated in the output. */
-            break;
-          }
           /* A virtual table that is constrained by an IN clause may not
           ** consume the ORDER BY clause because (1) the order of IN terms
           ** is not necessarily related to the order of output terms and

src/wherecode.c

@@ -874,6 +874,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     int iReg;   /* P3 Value for OP_VFilter */
     int addrNotFound;
     int nConstraint = pLoop->nLTerm;
+    int iIn;    /* Counter for IN constraints */
 
     sqlite3ExprCachePush(pParse);
     iReg = sqlite3GetTempRange(pParse, nConstraint+2);
@@ -896,14 +897,48 @@ Bitmask sqlite3WhereCodeOneLoopStart(
                       pLoop->u.vtab.needFree ? P4_MPRINTF : P4_STATIC);
     VdbeCoverage(v);
     pLoop->u.vtab.needFree = 0;
-    for(j=0; j<nConstraint && j<16; j++){
-      if( (pLoop->u.vtab.omitMask>>j)&1 ){
-        disableTerm(pLevel, pLoop->aLTerm[j]);
-      }
-    }
     pLevel->p1 = iCur;
     pLevel->op = pWInfo->eOnePass ? OP_Noop : OP_VNext;
     pLevel->p2 = sqlite3VdbeCurrentAddr(v);
+    iIn = pLevel->u.in.nIn;
+    for(j=nConstraint-1; j>=0; j--){
+      pTerm = pLoop->aLTerm[j];
+      if( j<16 && (pLoop->u.vtab.omitMask>>j)&1 ){
+        disableTerm(pLevel, pTerm);
+      }else if( (pTerm->eOperator & WO_IN)!=0 ){
+        Expr *pCompare;  /* The comparison operator */
+        Expr *pRight;    /* RHS of the comparison */
+        VdbeOp *pOp;     /* Opcode to access the value of the IN constraint */
+
+        /* Reload the constraint value into reg[iReg+j+2].  The same value
+        ** was loaded into the same register prior to the OP_VFilter, but
+        ** the xFilter implementation might have changed the datatype or
+        ** encoding of the value in the register, so it *must* be reloaded. */
+        assert( pLevel->u.in.aInLoop!=0 || db->mallocFailed );
+        if( pLevel->u.in.aInLoop!=0 ){
+          assert( iIn>0 );
+          pOp = sqlite3VdbeGetOp(v, pLevel->u.in.aInLoop[--iIn].addrInTop);
+          assert( pOp->opcode==OP_Column || pOp->opcode==OP_Rowid );
+          assert( pOp->opcode!=OP_Column || pOp->p3==iReg+j+2 );
+          assert( pOp->opcode!=OP_Rowid || pOp->p2==iReg+j+2 );
+          testcase( pOp->opcode==OP_Rowid );
+          sqlite3VdbeAddOp3(v, pOp->opcode, pOp->p1, pOp->p2, pOp->p3);
+        }
+
+        /* Generate code that will continue to the next row if 
+        ** the IN constraint is not satisfied */
+        pCompare = sqlite3PExpr(pParse, TK_EQ, 0, 0, 0);
+        assert( pCompare!=0 || db->mallocFailed );
+        if( pCompare ){
+          pCompare->pLeft = pTerm->pExpr->pLeft;
+          pCompare->pRight = pRight = sqlite3Expr(db, TK_REGISTER, 0);
+          if( pRight ) pRight->iTable = iReg+j+2;
+          sqlite3ExprIfFalse(pParse, pCompare, pLevel->addrCont, 0);
+          pCompare->pLeft = 0;
+          sqlite3ExprDelete(db, pCompare);
+        }
+      }
+    }
     sqlite3ReleaseTempRange(pParse, iReg, nConstraint+2);
     sqlite3ExprCachePop(pParse);
   }else

test/tabfunc01.test

@@ -120,4 +120,19 @@ do_catchsql_test tabfunc01-4.3 {
   SELECT * FROM aux1.generate_series(1,4)
 } {1 {no such table: aux1.generate_series}}
 
+# The next series of tests is verifying that virtual table are able
+# to optimize the IN operator, even on terms that are not marked "omit".
+# When the generate_series virtual table is compiled for the testfixture,
+# the special -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 option is used, which
+# causes the xBestIndex method of generate_series to leave the
+# sqlite3_index_constraint_usage.omit flag set to 0, which should cause
+# the SQLite core to verify the start=, stop=, and step= constraints on
+# each step of output.  At one point, the IN operator could not be used
+# by virtual tables unless omit was set.
+#
+do_execsql_test tabfunc01-500 {
+  SELECT * FROM generate_series WHERE start IN (1,7) AND stop=20 AND step=10
+  ORDER BY +1;
+} {1 7 11 17}
+
 finish_test