Rework automatically acquired OPFS locks to be released during idle time. This eliminates the performance hit reported in [46304ba057707c].

FossilOrigin-Name: a7fe91afca473fe55c983bc81d214df4ef3699863c7423fa4b6b9cde23d6a3b4

ext/wasm/api/sqlite3-api-opfs.js

@@ -467,9 +467,11 @@ const installOpfsVfs = function callee(options){
       /**
          Returns an array of the deserialized state stored by the most
          recent serialize() operation (from from this thread or the
-         counterpart thread), or null if the serialization buffer is empty.
+         counterpart thread), or null if the serialization buffer is
+         empty.  If passed a truthy argument, the serialization buffer
+         is cleared after deserialization.
       */
-      state.s11n.deserialize = function(){
+      state.s11n.deserialize = function(clear=false){
         ++metrics.s11n.deserialize.count;
         const t = performance.now();
         const argc = viewU8[0];
@@ -494,6 +496,7 @@ const installOpfsVfs = function callee(options){
             rc.push(v);
           }
         }
+        if(clear) viewU8[0] = 0;
         //log("deserialize:",argc, rc);
         metrics.s11n.deserialize.time += performance.now() - t;
         return rc;

ext/wasm/api/sqlite3-opfs-async-proxy.js

@@ -44,6 +44,7 @@ if(self.window === self){
    this API.
 */
 const state = Object.create(null);
+
 /**
    verbose:
 
@@ -96,13 +97,27 @@ metrics.dump = ()=>{
 };
 
 /**
-   Map of sqlite3_file pointers (integers) to metadata related to a
-   given OPFS file handles. The pointers are, in this side of the
-   interface, opaque file handle IDs provided by the synchronous
-   part of this constellation. Each value is an object with a structure
-   demonstrated in the xOpen() impl.
+   __openFiles is a map of sqlite3_file pointers (integers) to
+   metadata related to a given OPFS file handles. The pointers are, in
+   this side of the interface, opaque file handle IDs provided by the
+   synchronous part of this constellation. Each value is an object
+   with a structure demonstrated in the xOpen() impl.
 */
 const __openFiles = Object.create(null);
+/**
+   __autoLocks is a Set of sqlite3_file pointers (integers) which were
+   "auto-locked".  i.e. those for which we obtained a sync access
+   handle without an explicit xLock() call. Such locks will be
+   released during db connection idle time, whereas a sync access
+   handle obtained via xLock(), or subsequently xLock()'d after
+   auto-acquisition, will not be released until xUnlock() is called.
+
+   Maintenance reminder: if we relinquish auto-locks at the end of the
+   operation which acquires them, we pay a massive performance
+   penalty: speedtest1 benchmarks take up to 4x as long. By delaying
+   the lock release until idle time, the hit is negligible.
+*/
+const __autoLocks = new Set();
 
 /**
    Expects an OPFS file path. It gets resolved, such that ".."
@@ -191,6 +206,10 @@ const getSyncHandle = async (fh)=>{
       }
     }
     log("Got sync handle for",fh.filenameAbs,'in',performance.now() - t,'ms');
+    if(!fh.xLock){
+      __autoLocks.add(fh.fid);
+      log("Auto-locked",fh.fid,fh.filenameAbs);
+    }
   }
   return fh.syncHandle;
 };
@@ -210,6 +229,8 @@ const closeSyncHandle = async (fh)=>{
     log("Closing sync handle for",fh.filenameAbs);
     const h = fh.syncHandle;
     delete fh.syncHandle;
+    delete fh.xLock;
+    __autoLocks.delete(fh.fid);
     return h.close();
   }
 };
@@ -360,9 +381,10 @@ const vfsAsyncImpls = {
   xClose: async function(fid/*sqlite3_file pointer*/){
     const opName = 'xClose';
     mTimeStart(opName);
+    __autoLocks.delete(fid);
     const fh = __openFiles[fid];
     let rc = 0;
-    wTimeStart('xClose');
+    wTimeStart(opName);
     if(fh){
       delete __openFiles[fid];
       await closeSyncHandle(fh);
@@ -421,7 +443,6 @@ const vfsAsyncImpls = {
     mTimeStart('xFileSize');
     const fh = __openFiles[fid];
     let rc;
-    const hadLock = !!fh.syncHandle;
     wTimeStart('xFileSize');
     try{
       affirmLocked('xFileSize',fh);
@@ -432,7 +453,6 @@ const vfsAsyncImpls = {
       state.s11n.storeException(2,e);
       rc = state.sq3Codes.SQLITE_IOERR;
     }
-    if(!hadLock) closeSyncHandleNoThrow(fh);
     wTimeEnd();
     storeAndNotify('xFileSize', rc);
     mTimeEnd();
@@ -442,12 +462,17 @@ const vfsAsyncImpls = {
     mTimeStart('xLock');
     const fh = __openFiles[fid];
     let rc = 0;
+    const oldLockType = fh.xLock;
+    fh.xLock = lockType;
     if( !fh.syncHandle ){
       wTimeStart('xLock');
-      try { await getSyncHandle(fh) }
-      catch(e){
+      try {
+        await getSyncHandle(fh);
+        __autoLocks.delete(fid);
+      }catch(e){
         state.s11n.storeException(1,e);
         rc = state.sq3Codes.SQLITE_IOERR_LOCK;
+        fh.xLock = oldLockType;
       }
       wTimeEnd();
     }
@@ -481,6 +506,7 @@ const vfsAsyncImpls = {
       */
       wTimeEnd();
       __openFiles[fid] = Object.assign(Object.create(null),{
+        fid: fid,
         filenameAbs: filename,
         filenamePart: filenamePart,
         dirHandle: hDir,
@@ -503,7 +529,6 @@ const vfsAsyncImpls = {
     mTimeStart('xRead');
     let rc = 0, nRead;
     const fh = __openFiles[fid];
-    const hadLock = !!fh.syncHandle;
     try{
       affirmLocked('xRead',fh);
       wTimeStart('xRead');
@@ -522,7 +547,6 @@ const vfsAsyncImpls = {
       state.s11n.storeException(1,e);
       rc = state.sq3Codes.SQLITE_IOERR_READ;
     }
-    if(!hadLock) closeSyncHandleNoThrow(fh);
     storeAndNotify('xRead',rc);
     mTimeEnd();
   },
@@ -547,7 +571,6 @@ const vfsAsyncImpls = {
     mTimeStart('xTruncate');
     let rc = 0;
     const fh = __openFiles[fid];
-    const hadLock = !!fh.syncHandle;
     wTimeStart('xTruncate');
     try{
       affirmLocked('xTruncate',fh);
@@ -558,7 +581,6 @@ const vfsAsyncImpls = {
       state.s11n.storeException(2,e);
       rc = state.sq3Codes.SQLITE_IOERR_TRUNCATE;
     }
-    if(!hadLock) closeSyncHandleNoThrow(fh);
     wTimeEnd();
     storeAndNotify('xTruncate',rc);
     mTimeEnd();
@@ -585,7 +607,6 @@ const vfsAsyncImpls = {
     mTimeStart('xWrite');
     let rc;
     const fh = __openFiles[fid];
-    const hadLock = !!fh.syncHandle;
     wTimeStart('xWrite');
     try{
       affirmLocked('xWrite',fh);
@@ -600,7 +621,6 @@ const vfsAsyncImpls = {
       state.s11n.storeException(1,e);
       rc = state.sq3Codes.SQLITE_IOERR_WRITE;
     }
-    if(!hadLock) closeSyncHandleNoThrow(fh);
     wTimeEnd();
     storeAndNotify('xWrite',rc);
     mTimeEnd();
@@ -636,7 +656,7 @@ const initS11n = ()=>{
       default: toss("Invalid type ID:",tid);
     }
   };
-  state.s11n.deserialize = function(){
+  state.s11n.deserialize = function(clear=false){
     ++metrics.s11n.deserialize.count;
     const t = performance.now();
     const argc = viewU8[0];
@@ -661,6 +681,7 @@ const initS11n = ()=>{
         rc.push(v);
       }
     }
+    if(clear) viewU8[0] = 0;
     //log("deserialize:",argc, rc);
     metrics.s11n.deserialize.time += performance.now() - t;
     return rc;
@@ -727,21 +748,30 @@ const waitLoop = async function f(){
      We need to wake up periodically to give the thread a chance
      to do other things.
   */
-  const waitTime = 1000;
+  const waitTime = 500;
   while(!flagAsyncShutdown){
     try {
       if('timed-out'===Atomics.wait(
         state.sabOPView, state.opIds.whichOp, 0, waitTime
       )){
+        if(__autoLocks.size){
+          /* Release all auto-locks. */
+          for(const fid of __autoLocks){
+            const fh = __openFiles[fid];
+            await closeSyncHandleNoThrow(fh);
+            log("Auto-unlocked",fid,fh.filenameAbs);
+          }
+        }
         continue;
       }
       const opId = Atomics.load(state.sabOPView, state.opIds.whichOp);
       Atomics.store(state.sabOPView, state.opIds.whichOp, 0);
       const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId);
-      const args = state.s11n.deserialize() || [];
-      state.s11n.serialize(/* clear s11n to keep the caller from
-                              confusing this with an exception string
-                              written by the upcoming operation */);
+      const args = state.s11n.deserialize(
+        true /* clear s11n to keep the caller from confusing this with
+                an exception string written by the upcoming
+                operation */
+      ) || [];
       //warn("waitLoop() whichOp =",opId, hnd, args);
       if(hnd.f) await hnd.f(...args);
       else error("Missing callback for opId",opId);

ext/wasm/speedtest1-worker.html

@@ -169,7 +169,11 @@
         return str+a.join(' ');
     };
 
-    const W = new Worker("speedtest1-worker.js?sqlite3.dir=jswasm");
+    const urlParams = new URL(self.location.href).searchParams;
+    const W = new Worker(
+        "speedtest1-worker.js?sqlite3.dir=jswasm"+
+            (urlParams.has('opfs-verbose') ? '&opfs-verbose' : '')
+    );
     const mPost = function(msgType,payload){
         W.postMessage({type: msgType, data: payload});
     };
@@ -179,7 +183,6 @@
     const eLinkMainThread = E('#link-main-thread');
     const eLinkWasmfs = E('#link-wasmfs');
     const eLinkKvvfs = E('#link-kvvfs');
-    const urlParams = new URL(self.location.href).searchParams;
     const getSelectedFlags = ()=>{
         const f = Array.prototype.map.call(eFlags.selectedOptions, (v)=>v.value);
         [

manifest

@@ -1,5 +1,5 @@
-C OPFS:\sif\san\sop\swhich\sneeds\sa\slock\sis\scalled\swhen\sno\slock\shas\sbeen\sobtained,\sautomatically\slock\sit\sat\sthe\sstart\sof\sthe\sop\sand\sunlock\sit\sat\sthe\send\sof\sthat\sop.\sThis\sis\san\sattempt\sto\salleviate\sthe\scross-tab\scontention\sdescribed\sin\s[forum\spost\s58a377083cd24a|forum:58a377083cd24a]\sbut\sit\sincreases\sspeedtest1\srun\stime\sby\sapproximately\s4x.\sPerhaps\sauto-lock\scan\sbe\scombined\swith\sthe\solder\sidle-time-based\sauto-unlock\sto\sunlock\ssuch\slocks\s(but\snot\sthose\sfrom\sxLock())\sto\simprove\sthis?
-D 2022-11-10T11:35:10.700
+C Rework\sautomatically\sacquired\sOPFS\slocks\sto\sbe\sreleased\sduring\sidle\stime.\sThis\seliminates\sthe\sperformance\shit\sreported\sin\s[46304ba057707c].
+D 2022-11-10T13:14:30.548
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -502,11 +502,11 @@ F ext/wasm/api/pre-js.js 287e462f969342b032c03900e668099fa1471d852df7a472de5bc34
 F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34
 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5
 F ext/wasm/api/sqlite3-api-oo1.js e9a83489bbb4838ce0aee46eaaa9350e0e25a5b926b565e4f5ae8e840e4fbaed
-F ext/wasm/api/sqlite3-api-opfs.js cdcbb57acc66f4569ac9e18f9d13d5a3657d8aae195725c6324943da56c1005d
+F ext/wasm/api/sqlite3-api-opfs.js 5c6d0903e100b2918920d8c596ab037cd9a67df19e388ebde4fe085218780af0
 F ext/wasm/api/sqlite3-api-prologue.js fd526fa017fa2578673ca18158354515c719e719a5d93f2f6d0e43f39170430e
 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f
 F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3
-F ext/wasm/api/sqlite3-opfs-async-proxy.js 721ea9f716749a47b71be618fb33881afa828a46756f32836fc75c3e4bf11b4e
+F ext/wasm/api/sqlite3-opfs-async-proxy.js 24d1c1982a012d998907105a4ff1ff6881bf462395e90c06326817701e69f093
 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9
 F ext/wasm/api/sqlite3-wasm.c 778409e00fb25a4d6989be17fc856c84460198fd3b05ba2ef8289e60c50157ca
 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b
@@ -540,7 +540,7 @@ F ext/wasm/module-symbols.html b8eebafef8e536624bbe5f7a3da40c07a9062b843dfd3161a
 F ext/wasm/scratchpad-wasmfs-main.html 20cf6f1a8f368e70d01e8c17200e3eaa90f1c8e1029186d836d14b83845fbe06
 F ext/wasm/scratchpad-wasmfs-main.js 4c140457f4d6da9d646a49addd91edb6e9ad1643c6c48e3258b5bce24725dc18
 F ext/wasm/speedtest1-wasmfs.html bc28eb29b69a73864b8d7aae428448f8b7e1de81d8bfb9bba99541322054dbd0
-F ext/wasm/speedtest1-worker.html 94246488e10af9daa1ebd0979b1b8d7a22a579e5a983fa2a5ad94591ecf55f2c
+F ext/wasm/speedtest1-worker.html d75d9de9347c6f2f3278b73ccb6bceb559080644aef385fe496fa389f58bcd66
 F ext/wasm/speedtest1-worker.js 13b57c4a41729678a1194014afec2bd5b94435dcfc8d1039dfa9a533ac819ee1
 F ext/wasm/speedtest1.html e4c4e5c1c8ec1ad13c995e346e4216a1df152fd2c5cd17e0793b865b2f3c5000
 F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x
@@ -2055,11 +2055,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ae43e97087a3207a5ca3ffae75fbe7a33c01f4a38ce0d1d7eed8591ae3083617
-R 2fca99324f13ba1d5a012e300d10f561
-T *branch * opfs-lock-without-xlock
-T *sym-opfs-lock-without-xlock *
-T -sym-trunk * Cancelled\sby\sbranch.
+P 46304ba057707c3b072b6e7bb8c4af774f653aa5814099f0866cd87b2b73abeb
+R 4034304670d7219ca07133486e9bbed5
 U stephan
-Z f1d3a3f55481503869b687bc0b9f2eae
+Z 9cbc867c57797caf44094dc93d412778
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-46304ba057707c3b072b6e7bb8c4af774f653aa5814099f0866cd87b2b73abeb
+a7fe91afca473fe55c983bc81d214df4ef3699863c7423fa4b6b9cde23d6a3b4