mirrors/mongodb
mirrors/mongodb
0
0
Fork 0
mirror of https://github.com/mongodb/mongo.git synced 2024-11-24 00:17:37 +01:00
Code Issues Releases Activity

SERVER-96234 Implement missing save/restore calls in DistinctScan (#28986)

Browse Source 
GitOrigin-RevId: 7aa950ae51e671f89549dfa3ba63e86de0e19291
This commit is contained in:
Alya Carina Berciu 2024-11-12 19:19:45 +01:00 committed by MongoDB Bot
parent 4fbe381c85
commit c067f7246d
8 changed files with 589 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
jstests/query_golden_sharding/distinct_scan_multi_chunk.js
View File

@ -35,11 +35,13 @@ coll.createIndex({shardKey: 1});
coll.createIndex({shardKey: 1, notShardKey: 1});
const docs = [];
let _id = 0; // We don't want non-deterministic _ids in $$ROOT tests.
for (const chunk of allChunks) {
for (let i = 0; i < 3; i++) {
docs.push({shardKey: `${chunk}_${i}`, notShardKey: `1notShardKey_${chunk}_${i}`},
{shardKey: `${chunk}_${i}`, notShardKey: `2notShardKey_${chunk}_${i}`},
{shardKey: `${chunk}_${i}`, notShardKey: `3notShardKey_${chunk}_${i}`});
docs.push(
{_id: _id++, shardKey: `${chunk}_${i}`, notShardKey: `1notShardKey_${chunk}_${i}`},
{_id: _id++, shardKey: `${chunk}_${i}`, notShardKey: `2notShardKey_${chunk}_${i}`},
{_id: _id++, shardKey: `${chunk}_${i}`, notShardKey: `3notShardKey_${chunk}_${i}`});
}
}
coll.insertMany(docs);
@ -201,16 +203,16 @@ outputAggregationPlanAndResults(coll,
outputAggregationPlanAndResults(coll,
[{$group: {_id: "$shardKey", accum: {$last: "$notShardKey"}}}]);
// TODO SERVER-96234: Uncomment these tests.
// outputAggregationPlanAndResults(coll, [
// {$sort: {shardKey: 1, notShardKey: 1}},
// {$match: {shardKey: {$gt: "chunk1_s0"}}},
// {$group: {_id: "$shardKey", r: {$first: "$$ROOT"}}}
// ]);
// outputAggregationPlanAndResults(coll, [
// {$sort: {shardKey: 1, notShardKey: 1}},
// {$match: {shardKey: {$gt: "chunk1_s0"}}},
// {$group: {_id: "$shardKey", r: {$last: "$$ROOT"}}}
// ]);
subSection("with preceding $sort and intervening $match, output non-shard key field");
outputAggregationPlanAndResults(coll, [
{$sort: {shardKey: 1, notShardKey: 1}},
{$match: {shardKey: {$gt: "chunk1_s0"}}},
{$group: {_id: "$shardKey", r: {$first: "$$ROOT"}}}
]);
outputAggregationPlanAndResults(coll, [
{$sort: {shardKey: 1, notShardKey: 1}},
{$match: {shardKey: {$gt: "chunk1_s0"}}},
{$group: {_id: "$shardKey", r: {$last: "$$ROOT"}}}
]);
shardingTest.stop();

541
jstests/query_golden_sharding/expected_output/distinct_scan_multi_chunk.md
View File

@ -5463,3 +5463,544 @@
}
```
### with preceding $sort and intervening $match, output non-shard key field
### Pipeline
```json
[
{
"$sort" : {
"shardKey" : 1,
"notShardKey" : 1
}
},
{
"$match" : {
"shardKey" : {
"$gt" : "chunk1_s0"
}
}
},
{
"$group" : {
"_id" : "$shardKey",
"r" : {
"$first" : "$$ROOT"
}
}
}
]
```
### Results
```json
{ "_id" : "chunk1_s0_0", "r" : { "_id" : 0, "notShardKey" : "1notShardKey_chunk1_s0_0", "shardKey" : "chunk1_s0_0" } }
{ "_id" : "chunk1_s0_1", "r" : { "_id" : 3, "notShardKey" : "1notShardKey_chunk1_s0_1", "shardKey" : "chunk1_s0_1" } }
{ "_id" : "chunk1_s0_2", "r" : { "_id" : 6, "notShardKey" : "1notShardKey_chunk1_s0_2", "shardKey" : "chunk1_s0_2" } }
{ "_id" : "chunk1_s1_0", "r" : { "_id" : 27, "notShardKey" : "1notShardKey_chunk1_s1_0", "shardKey" : "chunk1_s1_0" } }
{ "_id" : "chunk1_s1_1", "r" : { "_id" : 30, "notShardKey" : "1notShardKey_chunk1_s1_1", "shardKey" : "chunk1_s1_1" } }
{ "_id" : "chunk1_s1_2", "r" : { "_id" : 33, "notShardKey" : "1notShardKey_chunk1_s1_2", "shardKey" : "chunk1_s1_2" } }
{ "_id" : "chunk2_s0_0", "r" : { "_id" : 9, "notShardKey" : "1notShardKey_chunk2_s0_0", "shardKey" : "chunk2_s0_0" } }
{ "_id" : "chunk2_s0_1", "r" : { "_id" : 12, "notShardKey" : "1notShardKey_chunk2_s0_1", "shardKey" : "chunk2_s0_1" } }
{ "_id" : "chunk2_s0_2", "r" : { "_id" : 15, "notShardKey" : "1notShardKey_chunk2_s0_2", "shardKey" : "chunk2_s0_2" } }
{ "_id" : "chunk2_s1_0", "r" : { "_id" : 36, "notShardKey" : "1notShardKey_chunk2_s1_0", "shardKey" : "chunk2_s1_0" } }
{ "_id" : "chunk2_s1_1", "r" : { "_id" : 39, "notShardKey" : "1notShardKey_chunk2_s1_1", "shardKey" : "chunk2_s1_1" } }
{ "_id" : "chunk2_s1_2", "r" : { "_id" : 42, "notShardKey" : "1notShardKey_chunk2_s1_2", "shardKey" : "chunk2_s1_2" } }
{ "_id" : "chunk3_s0_0", "r" : { "_id" : 18, "notShardKey" : "1notShardKey_chunk3_s0_0", "shardKey" : "chunk3_s0_0" } }
{ "_id" : "chunk3_s0_1", "r" : { "_id" : 21, "notShardKey" : "1notShardKey_chunk3_s0_1", "shardKey" : "chunk3_s0_1" } }
{ "_id" : "chunk3_s0_2", "r" : { "_id" : 24, "notShardKey" : "1notShardKey_chunk3_s0_2", "shardKey" : "chunk3_s0_2" } }
{ "_id" : "chunk3_s1_0", "r" : { "_id" : 45, "notShardKey" : "1notShardKey_chunk3_s1_0", "shardKey" : "chunk3_s1_0" } }
{ "_id" : "chunk3_s1_1", "r" : { "_id" : 48, "notShardKey" : "1notShardKey_chunk3_s1_1", "shardKey" : "chunk3_s1_1" } }
{ "_id" : "chunk3_s1_2", "r" : { "_id" : 51, "notShardKey" : "1notShardKey_chunk3_s1_2", "shardKey" : "chunk3_s1_2" } }
```
### Summarized explain
```json
{
"distinct_scan_multi_chunk-rs0" : [
{
"$cursor" : {
"rejectedPlans" : [
[
{
"memLimit" : 104857600,
"sortPattern" : {
"notShardKey" : 1,
"shardKey" : 1
},
"stage" : "SORT",
"type" : "simple"
},
{
"stage" : "FETCH"
},
{
"stage" : "SHARDING_FILTER"
},
{
"direction" : "forward",
"indexBounds" : {
"shardKey" : [
"(\"chunk1_s0\", {})"
]
},
"indexName" : "shardKey_1",
"isMultiKey" : false,
"isPartial" : false,
"isSparse" : false,
"isUnique" : false,
"keyPattern" : {
"shardKey" : 1
},
"multiKeyPaths" : {
"shardKey" : [ ]
},
"stage" : "IXSCAN"
}
]
],
"winningPlan" : [
{
"stage" : "SORT_KEY_GENERATOR"
},
{
"direction" : "forward",
"indexBounds" : {
"notShardKey" : [
"[MinKey, MaxKey]"
],
"shardKey" : [
"(\"chunk1_s0\", {})"
]
},
"indexName" : "shardKey_1_notShardKey_1",
"isFetching" : true,
"isMultiKey" : false,
"isPartial" : false,
"isShardFiltering" : true,
"isSparse" : false,
"isUnique" : false,
"keyPattern" : {
"notShardKey" : 1,
"shardKey" : 1
},
"multiKeyPaths" : {
"notShardKey" : [ ],
"shardKey" : [ ]
},
"stage" : "DISTINCT_SCAN"
}
]
}
},
{
"$group" : {
"_id" : "$shardKey",
"r" : {
"$first" : "$$ROOT"
}
}
}
],
"distinct_scan_multi_chunk-rs1" : [
{
"$cursor" : {
"rejectedPlans" : [
[
{
"memLimit" : 104857600,
"sortPattern" : {
"notShardKey" : 1,
"shardKey" : 1
},
"stage" : "SORT",
"type" : "simple"
},
{
"stage" : "FETCH"
},
{
"stage" : "SHARDING_FILTER"
},
{
"direction" : "forward",
"indexBounds" : {
"shardKey" : [
"(\"chunk1_s0\", {})"
]
},
"indexName" : "shardKey_1",
"isMultiKey" : false,
"isPartial" : false,
"isSparse" : false,
"isUnique" : false,
"keyPattern" : {
"shardKey" : 1
},
"multiKeyPaths" : {
"shardKey" : [ ]
},
"stage" : "IXSCAN"
}
]
],
"winningPlan" : [
{
"stage" : "SORT_KEY_GENERATOR"
},
{
"direction" : "forward",
"indexBounds" : {
"notShardKey" : [
"[MinKey, MaxKey]"
],
"shardKey" : [
"(\"chunk1_s0\", {})"
]
},
"indexName" : "shardKey_1_notShardKey_1",
"isFetching" : true,
"isMultiKey" : false,
"isPartial" : false,
"isShardFiltering" : true,
"isSparse" : false,
"isUnique" : false,
"keyPattern" : {
"notShardKey" : 1,
"shardKey" : 1
},
"multiKeyPaths" : {
"notShardKey" : [ ],
"shardKey" : [ ]
},
"stage" : "DISTINCT_SCAN"
}
]
}
},
{
"$group" : {
"_id" : "$shardKey",
"r" : {
"$first" : "$$ROOT"
}
}
}
],
"mergeType" : "router",
"mergerPart" : [
{
"$mergeCursors" : {
"allowPartialResults" : false,
"compareWholeSortKey" : false,
"nss" : "test.distinct_scan_multi_chunk",
"recordRemoteOpWaitTime" : false,
"requestQueryStatsFromRemotes" : false,
"tailableMode" : "normal"
}
},
{
"$group" : {
"$doingMerge" : true,
"_id" : "$$ROOT._id",
"r" : {
"$first" : "$$ROOT.r"
}
}
}
],
"shardsPart" : [
{
"$match" : {
"shardKey" : {
"$gt" : "chunk1_s0"
}
}
},
{
"$sort" : {
"sortKey" : {
"notShardKey" : 1,
"shardKey" : 1
}
}
},
{
"$group" : {
"_id" : "$shardKey",
"r" : {
"$first" : "$$ROOT"
}
}
}
]
}
```
### Pipeline
```json
[
{
"$sort" : {
"shardKey" : 1,
"notShardKey" : 1
}
},
{
"$match" : {
"shardKey" : {
"$gt" : "chunk1_s0"
}
}
},
{
"$group" : {
"_id" : "$shardKey",
"r" : {
"$last" : "$$ROOT"
}
}
}
]
```
### Results
```json
{ "_id" : "chunk1_s0_0", "r" : { "_id" : 2, "notShardKey" : "3notShardKey_chunk1_s0_0", "shardKey" : "chunk1_s0_0" } }
{ "_id" : "chunk1_s0_1", "r" : { "_id" : 5, "notShardKey" : "3notShardKey_chunk1_s0_1", "shardKey" : "chunk1_s0_1" } }
{ "_id" : "chunk1_s0_2", "r" : { "_id" : 8, "notShardKey" : "3notShardKey_chunk1_s0_2", "shardKey" : "chunk1_s0_2" } }
{ "_id" : "chunk1_s1_0", "r" : { "_id" : 29, "notShardKey" : "3notShardKey_chunk1_s1_0", "shardKey" : "chunk1_s1_0" } }
{ "_id" : "chunk1_s1_1", "r" : { "_id" : 32, "notShardKey" : "3notShardKey_chunk1_s1_1", "shardKey" : "chunk1_s1_1" } }
{ "_id" : "chunk1_s1_2", "r" : { "_id" : 35, "notShardKey" : "3notShardKey_chunk1_s1_2", "shardKey" : "chunk1_s1_2" } }
{ "_id" : "chunk2_s0_0", "r" : { "_id" : 11, "notShardKey" : "3notShardKey_chunk2_s0_0", "shardKey" : "chunk2_s0_0" } }
{ "_id" : "chunk2_s0_1", "r" : { "_id" : 14, "notShardKey" : "3notShardKey_chunk2_s0_1", "shardKey" : "chunk2_s0_1" } }
{ "_id" : "chunk2_s0_2", "r" : { "_id" : 17, "notShardKey" : "3notShardKey_chunk2_s0_2", "shardKey" : "chunk2_s0_2" } }
{ "_id" : "chunk2_s1_0", "r" : { "_id" : 38, "notShardKey" : "3notShardKey_chunk2_s1_0", "shardKey" : "chunk2_s1_0" } }
{ "_id" : "chunk2_s1_1", "r" : { "_id" : 41, "notShardKey" : "3notShardKey_chunk2_s1_1", "shardKey" : "chunk2_s1_1" } }
{ "_id" : "chunk2_s1_2", "r" : { "_id" : 44, "notShardKey" : "3notShardKey_chunk2_s1_2", "shardKey" : "chunk2_s1_2" } }
{ "_id" : "chunk3_s0_0", "r" : { "_id" : 20, "notShardKey" : "3notShardKey_chunk3_s0_0", "shardKey" : "chunk3_s0_0" } }
{ "_id" : "chunk3_s0_1", "r" : { "_id" : 23, "notShardKey" : "3notShardKey_chunk3_s0_1", "shardKey" : "chunk3_s0_1" } }
{ "_id" : "chunk3_s0_2", "r" : { "_id" : 26, "notShardKey" : "3notShardKey_chunk3_s0_2", "shardKey" : "chunk3_s0_2" } }
{ "_id" : "chunk3_s1_0", "r" : { "_id" : 47, "notShardKey" : "3notShardKey_chunk3_s1_0", "shardKey" : "chunk3_s1_0" } }
{ "_id" : "chunk3_s1_1", "r" : { "_id" : 50, "notShardKey" : "3notShardKey_chunk3_s1_1", "shardKey" : "chunk3_s1_1" } }
{ "_id" : "chunk3_s1_2", "r" : { "_id" : 53, "notShardKey" : "3notShardKey_chunk3_s1_2", "shardKey" : "chunk3_s1_2" } }
```
### Summarized explain
```json
{
"distinct_scan_multi_chunk-rs0" : [
{
"$cursor" : {
"rejectedPlans" : [
[
{
"memLimit" : 104857600,
"sortPattern" : {
"notShardKey" : 1,
"shardKey" : 1
},
"stage" : "SORT",
"type" : "simple"
},
{
"stage" : "FETCH"
},
{
"stage" : "SHARDING_FILTER"
},
{
"direction" : "forward",
"indexBounds" : {
"shardKey" : [
"(\"chunk1_s0\", {})"
]
},
"indexName" : "shardKey_1",
"isMultiKey" : false,
"isPartial" : false,
"isSparse" : false,
"isUnique" : false,
"keyPattern" : {
"shardKey" : 1
},
"multiKeyPaths" : {
"shardKey" : [ ]
},
"stage" : "IXSCAN"
}
]
],
"winningPlan" : [
{
"stage" : "SORT_KEY_GENERATOR"
},
{
"direction" : "backward",
"indexBounds" : {
"notShardKey" : [
"[MaxKey, MinKey]"
],
"shardKey" : [
"({}, \"chunk1_s0\")"
]
},
"indexName" : "shardKey_1_notShardKey_1",
"isFetching" : true,
"isMultiKey" : false,
"isPartial" : false,
"isShardFiltering" : true,
"isSparse" : false,
"isUnique" : false,
"keyPattern" : {
"notShardKey" : 1,
"shardKey" : 1
},
"multiKeyPaths" : {
"notShardKey" : [ ],
"shardKey" : [ ]
},
"stage" : "DISTINCT_SCAN"
}
]
}
},
{
"$group" : {
"_id" : "$shardKey",
"r" : {
"$last" : "$$ROOT"
}
}
}
],
"distinct_scan_multi_chunk-rs1" : [
{
"$cursor" : {
"rejectedPlans" : [
[
{
"memLimit" : 104857600,
"sortPattern" : {
"notShardKey" : 1,
"shardKey" : 1
},
"stage" : "SORT",
"type" : "simple"
},
{
"stage" : "FETCH"
},
{
"stage" : "SHARDING_FILTER"
},
{
"direction" : "forward",
"indexBounds" : {
"shardKey" : [
"(\"chunk1_s0\", {})"
]
},
"indexName" : "shardKey_1",
"isMultiKey" : false,
"isPartial" : false,
"isSparse" : false,
"isUnique" : false,
"keyPattern" : {
"shardKey" : 1
},
"multiKeyPaths" : {
"shardKey" : [ ]
},
"stage" : "IXSCAN"
}
]
],
"winningPlan" : [
{
"stage" : "SORT_KEY_GENERATOR"
},
{
"direction" : "backward",
"indexBounds" : {
"notShardKey" : [
"[MaxKey, MinKey]"
],
"shardKey" : [
"({}, \"chunk1_s0\")"
]
},
"indexName" : "shardKey_1_notShardKey_1",
"isFetching" : true,
"isMultiKey" : false,
"isPartial" : false,
"isShardFiltering" : true,
"isSparse" : false,
"isUnique" : false,
"keyPattern" : {
"notShardKey" : 1,
"shardKey" : 1
},
"multiKeyPaths" : {
"notShardKey" : [ ],
"shardKey" : [ ]
},
"stage" : "DISTINCT_SCAN"
}
]
}
},
{
"$group" : {
"_id" : "$shardKey",
"r" : {
"$last" : "$$ROOT"
}
}
}
],
"mergeType" : "router",
"mergerPart" : [
{
"$mergeCursors" : {
"allowPartialResults" : false,
"compareWholeSortKey" : false,
"nss" : "test.distinct_scan_multi_chunk",
"recordRemoteOpWaitTime" : false,
"requestQueryStatsFromRemotes" : false,
"tailableMode" : "normal"
}
},
{
"$group" : {
"$doingMerge" : true,
"_id" : "$$ROOT._id",
"r" : {
"$last" : "$$ROOT.r"
}
}
}
],
"shardsPart" : [
{
"$match" : {
"shardKey" : {
"$gt" : "chunk1_s0"
}
}
},
{
"$sort" : {
"sortKey" : {
"notShardKey" : 1,
"shardKey" : 1
}
}
},
{
"$group" : {
"_id" : "$shardKey",
"r" : {
"$last" : "$$ROOT"
}
}
}
]
}
```

2
jstests/sharding/query/distinct_shard_filtering.js
View File

@ -5,8 +5,6 @@
* featureFlagShardFilteringDistinctScan,
* do_not_wrap_aggregations_in_facets,
* not_allowed_with_signed_security_token,
* # TODO SERVER-95934: Remove tsan_incompatible.
* tsan_incompatible,
* ]
*/

2
jstests/sharding/query/group_conversion_to_distinct_scan_with_orphans.js
View File

@ -9,8 +9,6 @@
* expects_explicit_underscore_id_index,
* # Index filter commands do not support causal consistency.
* does_not_support_causal_consistency,
* # TODO SERVER-95934: Remove tsan_incompatible.
* tsan_incompatible,
* ]
*/

2
jstests/sharding/query/group_with_acc_n_to_distinct_scan_with_orphans.js
View File

@ -9,8 +9,6 @@
* expects_explicit_underscore_id_index,
* # Index filter commands do not support causal consistency.
* does_not_support_causal_consistency,
* # TODO SERVER-95934: Remove tsan_incompatible.
* tsan_incompatible,
* ]
*/

2
jstests/sharding/query/group_with_top_bottom_to_distinct_scan_with_orphans.js
View File

@ -9,8 +9,6 @@
* expects_explicit_underscore_id_index,
* # Index filter commands do not support causal consistency.
* does_not_support_causal_consistency,
* # TODO SERVER-95934: Remove tsan_incompatible.
* tsan_incompatible,
* ]
*/

35
src/mongo/db/exec/distinct_scan.cpp
View File

@ -40,6 +40,7 @@
#include "mongo/db/catalog/collection.h"
#include "mongo/db/exec/plan_stage.h"
#include "mongo/db/exec/requires_index_stage.h"
#include "mongo/db/exec/working_set.h"
#include "mongo/db/exec/working_set_common.h"
#include "mongo/db/index/index_access_method.h"
@ -313,24 +314,48 @@ bool DistinctScan::isEOF() {
}
void DistinctScan::doSaveStateRequiresIndex() {
// We always seek, so we don't care where the cursor is.
if (_cursor)
if (_cursor && !_needsSequentialScan) {
// Unless we are 1) shard filtering, 2) not using an orphan chunk skipper, and 3) scanning
// past orphans, we always seek, so we don't care where the cursor is.
_cursor->saveUnpositioned();
} else if (_cursor) {
// We are scanning past orphans; save the cursor position.
_cursor->save();
}
if (_fetchCursor) {
_fetchCursor->saveUnpositioned();
}
}
void DistinctScan::doRestoreStateRequiresIndex() {
if (_cursor)
if (_cursor) {
_cursor->restore();
}
if (_fetchCursor) {
uassert(9623400,
"Could not restore collection cursor for fetching DISTINCT_SCAN",
_fetchCursor->restore());
}
}
void DistinctScan::doDetachFromOperationContext() {
if (_cursor)
if (_cursor) {
_cursor->detachFromOperationContext();
}
if (_fetchCursor) {
_fetchCursor->detachFromOperationContext();
}
}
void DistinctScan::doReattachToOperationContext() {
if (_cursor)
if (_cursor) {
_cursor->reattachToOperationContext(opCtx());
}
if (_fetchCursor) {
_fetchCursor->reattachToOperationContext(opCtx());
}
}
unique_ptr<PlanStageStats> DistinctScan::getStats() {

4
src/mongo/db/exec/requires_index_stage.h
View File

@ -72,9 +72,9 @@ protected:
*/
virtual void doRestoreStateRequiresIndex() = 0;
void doSaveStateRequiresCollection() final;
void doSaveStateRequiresCollection() override;
void doRestoreStateRequiresCollection() final;
void doRestoreStateRequiresCollection() override;
const IndexDescriptor* indexDescriptor() const {
return _entry ? _entry->descriptor() : nullptr;