Import wiredtiger: 5abe3cff013c14f7285f0db4eb04dd8a388478c9 from branch mongodb-5.0

ref: fee51ab6b3..5abe3cff01
for: 5.0.0

WT-6576       Fix the aborted on-disk prepared key

src/third_party/wiredtiger/import.data

@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-5.0",
-    "commit": "fee51ab6b3550d3fb2a183ba7c1cee17bde38112"
+    "commit": "5abe3cff013c14f7285f0db4eb04dd8a388478c9"
 }

src/third_party/wiredtiger/src/btree/bt_page.c

@@ -686,6 +686,7 @@ __inmem_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page)
         upd->durable_ts = unpack.tw.durable_start_ts;
         upd->start_ts = unpack.tw.start_ts;
         upd->txnid = unpack.tw.start_txn;
+        F_SET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DS);
 
         /*
          * Instantiate both update and tombstone if the prepared update is a tombstone. This is
@@ -700,7 +701,6 @@ __inmem_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page)
             tombstone->txnid = unpack.tw.stop_txn;
             tombstone->prepare_state = WT_PREPARE_INPROGRESS;
             F_SET(tombstone, WT_UPDATE_PREPARE_RESTORED_FROM_DS);
-            F_SET(upd, WT_UPDATE_RESTORED_FROM_DS);
 
             /*
              * Mark the update also as in-progress if the update and tombstone are from same
@@ -712,14 +712,12 @@ __inmem_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page)
               unpack.tw.start_txn == unpack.tw.stop_txn) {
                 upd->durable_ts = WT_TS_NONE;
                 upd->prepare_state = WT_PREPARE_INPROGRESS;
-                F_SET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DS);
             }
 
             tombstone->next = upd;
         } else {
             upd->durable_ts = WT_TS_NONE;
             upd->prepare_state = WT_PREPARE_INPROGRESS;
-            F_SET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DS);
             tombstone = upd;
         }
 

src/third_party/wiredtiger/src/txn/txn.c

@@ -1054,6 +1054,7 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit,
 #endif
     size_t not_used;
     uint32_t hs_btree_id;
+    char ts_string[3][WT_TS_INT_STRING_SIZE];
     bool upd_appended;
 
     hs_cursor = NULL;
@@ -1063,9 +1064,18 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit,
 
     WT_RET(__txn_search_prepared_op(session, op, cursorp, &upd));
 
-    __wt_verbose(session, WT_VERB_TRANSACTION,
-      "resolving prepared op for txnid: %" PRIu64 " that %s", txn->id,
-      commit ? "committed" : "roll backed");
+    if (commit)
+        __wt_verbose(session, WT_VERB_TRANSACTION,
+          "commit resolving prepared transaction with txnid: %" PRIu64
+          "and timestamp: %s to commit and durable timestamps: %s,%s",
+          txn->id, __wt_timestamp_to_string(txn->prepare_timestamp, ts_string[0]),
+          __wt_timestamp_to_string(txn->commit_timestamp, ts_string[1]),
+          __wt_timestamp_to_string(txn->durable_timestamp, ts_string[2]));
+    else
+        __wt_verbose(session, WT_VERB_TRANSACTION,
+          "rollback resolving prepared transaction with txnid: %" PRIu64 "and timestamp:%s",
+          txn->id, __wt_timestamp_to_string(txn->prepare_timestamp, ts_string[0]));
+
     /*
      * Aborted updates can exist in the update chain of our transaction. Generally this will occur
      * due to a reserved update. As such we should skip over these updates.
@@ -1092,9 +1102,14 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit,
      * updates first, the history search logic may race with other sessions modifying the same key
      * and checkpoint moving the new updates to the history store.
      *
-     * For prepared delete, we don't need to fix the history store.
+     * For prepared delete commit, we don't need to fix the history store. Whereas for rollback, if
+     * the update is also from the same prepared transaction, restore the update from history store
+     * or remove the key.
      */
-    if (F_ISSET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DS) && upd->type != WT_UPDATE_TOMBSTONE) {
+    if (F_ISSET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DS) &&
+      (upd->type != WT_UPDATE_TOMBSTONE ||
+        (!commit && upd->next != NULL && upd->durable_ts == upd->next->durable_ts &&
+          upd->txnid == upd->next->txnid && upd->start_ts == upd->next->start_ts))) {
         cbt = (WT_CURSOR_BTREE *)(*cursorp);
         hs_btree_id = S2BT(session)->id;
         /* Open a history store table cursor. */

src/third_party/wiredtiger/test/suite/test_prepare15.py

@@ -0,0 +1,204 @@
+#!/usr/bin/env python
+#
+# Public Domain 2014-present MongoDB, Inc.
+# Public Domain 2008-2014 WiredTiger, Inc.
+#
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+
+import wttest
+from wiredtiger import WT_NOTFOUND
+from wtscenario import make_scenarios
+
+def timestamp_str(t):
+    return '%x' % t
+
+# test_prepare15.py
+# Test that the prepare transaction rollback removes the on-disk key
+# or replace it with history store and commit retains the changes when
+# both insert and remove operations are from the same transaction.
+class test_prepare15(wttest.WiredTigerTestCase):
+    in_memory_values = [
+        ('no_inmem', dict(in_memory=False)),
+        ('inmem', dict(in_memory=True))
+    ]
+
+    key_format_values = [
+        ('column', dict(key_format='r')),
+        ('integer_row', dict(key_format='i')),
+    ]
+
+    txn_end_values = [
+        ('commit', dict(commit=True)),
+        ('rollback', dict(commit=False)),
+    ]
+
+    scenarios = make_scenarios(in_memory_values, key_format_values, txn_end_values)
+
+    def conn_config(self):
+        config = 'cache_size=50MB'
+        if self.in_memory:
+            config += ',in_memory=true'
+        else:
+            config += ',in_memory=false'
+        return config
+
+    def test_prepare_restore_hs_update(self):
+        # Prepare transactions for column store table is not yet supported.
+        if self.key_format == 'r':
+            self.skipTest('Prepare transactions for column store table is not yet supported')
+
+        # Create a table without logging.
+        uri = "table:prepare15"
+        create_config = 'allocation_size=512,key_format=S,value_format=S'
+        self.session.create(uri, create_config)
+
+        # Pin oldest and stable timestamps to 10.
+        self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) +
+            ',stable_timestamp=' + timestamp_str(10))
+
+        valuea = 'a'
+        valueb = 'a'
+
+        # Perform an update and remove.
+        cursor = self.session.open_cursor(uri)
+        self.session.begin_transaction()
+        cursor[str(0)] = valuea
+        self.session.commit_transaction('commit_timestamp=' + timestamp_str(20))
+
+        self.session.begin_transaction()
+        cursor.set_key(str(0))
+        cursor.remove()
+        self.session.commit_transaction('commit_timestamp=' + timestamp_str(30))
+        cursor.close()
+
+        # Perform an update and remove.
+        s = self.conn.open_session()
+        cursor = s.open_cursor(uri)
+        s.begin_transaction()
+        cursor[str(0)] = valueb
+        cursor.set_key(str(0))
+        cursor.remove()
+        cursor.close()
+        s.prepare_transaction('prepare_timestamp=' + timestamp_str(40))
+
+        # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used.
+        evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)")
+
+        # Search for the key so we position our cursor on the page that we want to evict.
+        self.session.begin_transaction('ignore_prepare = true')
+        evict_cursor.set_key(str(0))
+        self.assertEquals(evict_cursor.search(), WT_NOTFOUND)
+        evict_cursor.reset()
+        evict_cursor.close()
+        self.session.commit_transaction()
+
+        if self.commit:
+            # Commit the prepared transaction
+            s.timestamp_transaction('commit_timestamp=' + timestamp_str(50))
+            s.timestamp_transaction('durable_timestamp=' + timestamp_str(60))
+            s.commit_transaction()
+        else:
+            # Rollback the prepared transaction
+            s.rollback_transaction()
+
+        # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used.
+        evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)")
+
+        # Search for the key so we position our cursor on the page that we want to evict.
+        self.session.begin_transaction()
+        evict_cursor.set_key(str(0))
+        self.assertEquals(evict_cursor.search(), WT_NOTFOUND)
+        evict_cursor.reset()
+        evict_cursor.close()
+        self.session.commit_transaction()
+
+        self.session.begin_transaction('read_timestamp=' + timestamp_str(20))
+        cursor2 = self.session.open_cursor(uri)
+        cursor2.set_key(str(0))
+        self.assertEquals(cursor2.search(), 0)
+        self.assertEqual(cursor2.get_value(), valuea)
+        self.session.commit_transaction()
+
+    def test_prepare_not_found(self):
+        # Prepare transactions for column store table is not yet supported.
+        if self.key_format == 'r':
+            self.skipTest('Prepare transactions for column store table is not yet supported')
+
+        # Create a table without logging.
+        uri = "table:prepare15"
+        create_config = 'allocation_size=512,key_format=S,value_format=S'
+        self.session.create(uri, create_config)
+
+        # Pin oldest and stable timestamps to 10.
+        self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) +
+            ',stable_timestamp=' + timestamp_str(10))
+
+        value = 'a'
+
+        # Perform an update and remove.
+        s = self.conn.open_session()
+        cursor = s.open_cursor(uri)
+        s.begin_transaction()
+        cursor[str(0)] = value
+        cursor.set_key(str(0))
+        cursor.remove()
+        cursor.close()
+        s.prepare_transaction('prepare_timestamp=' + timestamp_str(20))
+
+        # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used.
+        evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)")
+
+        # Search for the key so we position our cursor on the page that we want to evict.
+        self.session.begin_transaction("ignore_prepare = true")
+        evict_cursor.set_key(str(0))
+        self.assertEquals(evict_cursor.search(), WT_NOTFOUND)
+        evict_cursor.reset()
+        evict_cursor.close()
+        self.session.commit_transaction()
+
+        if self.commit:
+            # Commit the prepared transaction
+            s.timestamp_transaction('commit_timestamp=' + timestamp_str(30))
+            s.timestamp_transaction('durable_timestamp=' + timestamp_str(40))
+            s.commit_transaction()
+        else:
+            # Rollback the prepared transaction
+            s.rollback_transaction()
+
+        # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used.
+        evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)")
+
+        # Search for the key so we position our cursor on the page that we want to evict.
+        self.session.begin_transaction()
+        evict_cursor.set_key(str(0))
+        self.assertEquals(evict_cursor.search(), WT_NOTFOUND)
+        evict_cursor.reset()
+        evict_cursor.close()
+        self.session.commit_transaction()
+
+        self.session.begin_transaction()
+        cursor2 = self.session.open_cursor(uri)
+        cursor2.set_key(str(0))
+        self.assertEquals(cursor2.search(), WT_NOTFOUND)
+        self.session.commit_transaction()