diff --git a/manifest b/manifest index e9c314df02..bca1e660ac 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\scomment\susing\sthe\sname\s"predicate\spush-down\soptimization"\sto\swhat\swe\shave\nalso\scalled\s"WHERE-clause\spush\sdown".\s\sNo\schanges\sto\scode. -D 2024-07-04T11:15:32.292 +C Be\smore\saggressive\sabout\sreusing\ssubqueries\sthat\sappear\son\sthe\sRHS\sof\sIN\noperators\sthat\shave\sbeen\sreplicated\sdue\sto\sthe\spredicate\spush-down\soptimization. +D 2024-07-04T16:57:11.632 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -705,7 +705,7 @@ F src/date.c 13dd752847afb32ed70510ad7345a5b9c841f51ad904dba5d010f1fa3a6a324e F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782 F src/dbstat.c 3b677254d512fcafd4d0b341bf267b38b235ccfddbef24f9154e19360fa22e43 F src/delete.c cb766727c78e715f9fb7ec8a7d03658ed2a3016343ca687acfcec9083cdca500 -F src/expr.c d6f614d0c90cf8d72bbd54dbae96040e7feb495a77991fdc1b1c67db68eeb326 +F src/expr.c 06c1d1a0f8b5025f001bc8219ac63fded7c33494a1863afdc78032ed1085ffef F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 852f93c0ef995e0c2b8983059a2b97151c194cc8259e21f5bc2b7ac508348c2a F src/func.c 1f61e32e7a357e615b5d2e774bee563761fce4f2fd97ecb0f72c33e62a2ada5f @@ -826,10 +826,10 @@ F src/utf.c f23165685a67b4caf8ec08fb274cb3f319103decfb2a980b7cfd55d18dfa855e F src/util.c 5d1a0134cf4240648d1c6bb5cc8efaca0ea2b5d5c840985aec7e947271f04375 F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104 F src/vdbe.c f941fd8dbd1a9e073866a941e7fbb553f1de2d25b249611005d8c27d5eac8995 -F src/vdbe.h c2d78d15112c3fc5ab87f5e8e0b75d2db1c624409de2e858c3d1aafb1650bb4f +F src/vdbe.h c2549a215898a390de6669cfa32adba56f0d7e17ba5a7f7b14506d6fd5f0c36a F src/vdbeInt.h 949669dfd8a41550d27dcb905b494f2ccde9a2e6c1b0b04daa1227e2e74c2b2c F src/vdbeapi.c 80235ac380e9467fec1cb0883354d841f2a771976e766995f7e0c77f845406df -F src/vdbeaux.c 6e37cb918506c28fe7657454fcbc2e01e66bfba4164f306c2f075fd5c5fef609 +F src/vdbeaux.c 8d522d012c0545035be52e46b4cc5dfa752e376e17137816e558b3ab6bec9e5d F src/vdbeblob.c 13f9287b55b6356b4b1845410382d6bede203ceb29ef69388a4a3d007ffacbe5 F src/vdbemem.c 831a244831eaa45335f9ae276b50a7a82ee10d8c46c2c72492d4eb8c98d94d89 F src/vdbesort.c 237840ca1947511fa59bd4e18b9eeae93f2af2468c34d2427b059f896230a547 @@ -1518,7 +1518,7 @@ F test/printf.test 685fec5a0c5af2490ab0632775a301554361d674211d690f5bee0a97b0533 F test/printf2.test 3f55c1871a5a65507416076f6eb97e738d5210aeda7595a74ee895f2224cce60 F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc -F test/pushdown.test 9e655df51bc6559608dcc7af89a36f727eff520b4860261ed15a5f95a5ebbcd8 +F test/pushdown.test 84b525767442b3695d671f9df59dd91cf0ed8fb24cbbcdc55959f0dadeee8b39 F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca F test/quick.test 1681febc928d686362d50057c642f77a02c62e57 F test/quickcheck.test a4b7e878cd97e46108291c409b0bf8214f29e18fddd68a42bc5c1375ad1fb80a @@ -2195,8 +2195,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a204ffc06b468c2edf8f11ccf3de639edd4f8282e69a44ceeb68a4d3a43e77ea -R 7ddaa7e2074acfdf6d0130f12e8505c2 +P be77fe701c7b7d3407800e1ed6a4c8be14035831790419cc99362bff41c65b60 +R 53a737f2af23a108773b1a9f30c846b8 +T *branch * reuse-subqueries +T *sym-reuse-subqueries * +T -sym-trunk * U drh -Z 940dec88e4e92fbd5a212ce2f1e6eedc +Z 375183e5bacb7ab157909f081f98b1e8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3c3a8a5560..2a35b48263 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -be77fe701c7b7d3407800e1ed6a4c8be14035831790419cc99362bff41c65b60 +2accf32b6e45a396503c29eecc14a103bcc7b4c313cde921b26b489704060177 diff --git a/src/expr.c b/src/expr.c index 6d67d77f68..9ab1905050 100644 --- a/src/expr.c +++ b/src/expr.c @@ -3420,6 +3420,46 @@ void sqlite3VectorErrorMsg(Parse *pParse, Expr *pExpr){ } } +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Scan all previously generated bytecode looking for an OP_BeginSubrtn +** that is compatible with pExpr. If found, add the y.sub values +** to pExpr and return true. If not found, return false. +*/ +static int findCompatibleInRhsSubrtn( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* IN operator with RHS that we want to reuse */ + SubrtnSig *pNewSig /* Signature for the IN operator */ +){ + VdbeOp *pOp, *pEnd; + SubrtnSig *pSig; + Vdbe *v; + + if( pNewSig==0 ) return 0; + assert( pExpr->op==TK_IN ); + assert( !ExprUseYSub(pExpr) ); + assert( ExprUseXSelect(pExpr) ); + v = pParse->pVdbe; + assert( v!=0 ); + pOp = sqlite3VdbeGetOp(v, 1); + pEnd = sqlite3VdbeGetLastOp(v); + for(; pOpopcode!=OP_BeginSubrtn ) continue; + if( pOp->p4type!=P4_SUBRTNSIG ) continue; + pSig = pOp->p4.pSubrtnSig; + assert( pSig!=0 ); + if( pNewSig->selId!=pSig->selId ) continue; + if( strcmp(pNewSig->zAff,pSig->zAff)!=0 ) continue; + pExpr->y.sub.iAddr = pSig->iAddr; + pExpr->y.sub.regReturn = pSig->regReturn; + pExpr->iTable = pSig->iTable; + ExprSetProperty(pExpr, EP_Subrtn); + return 1; + } + return 0; +} +#endif /* SQLITE_OMIT_SUBQUERY */ + #ifndef SQLITE_OMIT_SUBQUERY /* ** Generate code that will construct an ephemeral table containing all terms @@ -3469,11 +3509,30 @@ void sqlite3CodeRhsOfIN( ** and reuse it many names. */ if( !ExprHasProperty(pExpr, EP_VarSelect) && pParse->iSelfTab==0 ){ - /* Reuse of the RHS is allowed */ - /* If this routine has already been coded, but the previous code - ** might not have been invoked yet, so invoke it now as a subroutine. + /* Reuse of the RHS is allowed + ** + ** Compute a signature for the RHS of the IN operator to facility + ** finding and reusing prior instances of the same IN operator. */ - if( ExprHasProperty(pExpr, EP_Subrtn) ){ + SubrtnSig *pSig; + if( !ExprUseXSelect(pExpr) ){ + pSig = 0; + }else{ + assert( pExpr->x.pSelect!=0 ); + pSig = sqlite3DbMallocRawNN(pParse->db, sizeof(pSig[0])); + if( pSig ){ + pSig->selId = pExpr->x.pSelect->selId; + pSig->zAff = exprINAffinity(pParse, pExpr); + } + } + + /* Check to see if there is a prior materialization of the RHS of + ** this IN operator. If there is, then make use of that prior + ** materialization rather than recomputing it. + */ + if( ExprHasProperty(pExpr, EP_Subrtn) + || findCompatibleInRhsSubrtn(pParse, pExpr, pSig) + ){ addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); if( ExprUseXSelect(pExpr) ){ ExplainQueryPlan((pParse, 0, "REUSE LIST SUBQUERY %d", @@ -3485,6 +3544,10 @@ void sqlite3CodeRhsOfIN( assert( iTab!=pExpr->iTable ); sqlite3VdbeAddOp2(v, OP_OpenDup, iTab, pExpr->iTable); sqlite3VdbeJumpHere(v, addrOnce); + if( pSig ){ + sqlite3DbFree(pParse->db, pSig->zAff); + sqlite3DbFree(pParse->db, pSig); + } return; } @@ -3495,7 +3558,12 @@ void sqlite3CodeRhsOfIN( pExpr->y.sub.regReturn = ++pParse->nMem; pExpr->y.sub.iAddr = sqlite3VdbeAddOp2(v, OP_BeginSubrtn, 0, pExpr->y.sub.regReturn) + 1; - + if( pSig ){ + pSig->iAddr = pExpr->y.sub.iAddr; + pSig->regReturn = pExpr->y.sub.regReturn; + pSig->iTable = iTab; + sqlite3VdbeChangeP4(v, -1, (const char*)pSig, P4_SUBRTNSIG); + } addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); } diff --git a/src/vdbe.h b/src/vdbe.h index 9001ace2ee..f40f68d24b 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -32,6 +32,19 @@ typedef struct Vdbe Vdbe; */ typedef struct sqlite3_value Mem; typedef struct SubProgram SubProgram; +typedef struct SubrtnSig SubrtnSig; + +/* +** A signature for a reusable subroutine that materializes the RHS of +** an IN operator. +*/ +struct SubrtnSig { + int selId; /* SELECT-id for the SELECT statement on the RHS */ + char *zAff; /* Affinity of the overall IN expression */ + int iTable; /* Ephemeral table generated by the subroutine */ + int iAddr; /* Subroutine entry address */ + int regReturn; /* Register used to hold return address */ +}; /* ** A single instruction of the virtual machine has an opcode @@ -60,6 +73,7 @@ struct VdbeOp { u32 *ai; /* Used when p4type is P4_INTARRAY */ SubProgram *pProgram; /* Used when p4type is P4_SUBPROGRAM */ Table *pTab; /* Used when p4type is P4_TABLE */ + SubrtnSig *pSubrtnSig; /* Used when p4type is P4_SUBRTNSIG */ #ifdef SQLITE_ENABLE_CURSOR_HINTS Expr *pExpr; /* Used when p4type is P4_EXPR */ #endif @@ -127,6 +141,7 @@ typedef struct VdbeOpList VdbeOpList; #define P4_INTARRAY (-14) /* P4 is a vector of 32-bit integers */ #define P4_FUNCCTX (-15) /* P4 is a pointer to an sqlite3_context object */ #define P4_TABLEREF (-16) /* Like P4_TABLE, but reference counted */ +#define P4_SUBRTNSIG (-17) /* P4 is a SubrtnSig pointer */ /* Error message codes for OP_Halt */ #define P5_ConstraintNotNull 1 diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 665f6cd17a..745d3eaba6 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1413,6 +1413,12 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ if( db->pnBytesFreed==0 ) sqlite3DeleteTable(db, (Table*)p4); break; } + case P4_SUBRTNSIG: { + SubrtnSig *pSig = (SubrtnSig*)p4; + sqlite3DbFree(db, pSig->zAff); + sqlite3DbFree(db, pSig); + break; + } } } @@ -1992,6 +1998,9 @@ char *sqlite3VdbeDisplayP4(sqlite3 *db, Op *pOp){ zP4 = pOp->p4.pTab->zName; break; } + case P4_SUBRTNSIG: { + break; + } default: { zP4 = pOp->p4.z; } diff --git a/test/pushdown.test b/test/pushdown.test index b3663bd6f5..271d412e7e 100644 --- a/test/pushdown.test +++ b/test/pushdown.test @@ -279,13 +279,9 @@ do_eqp_test 6.1 { | | `--CREATE BLOOM FILTER | `--UNION ALL | |--SEARCH t02 USING INDEX t02x (w=? AND x=? AND y>? AND y? AND y? AND y