mirrors/mongodb
mirrors/mongodb
0
0
Fork 0
mirror of https://github.com/mongodb/mongo.git synced 2024-11-21 12:39:08 +01:00
Code Issues Releases Activity

SERVER-72262 Bump Python minimum version to 3.9/3.10

Browse Source
This commit is contained in:
Alex Neben 2023-08-02 19:00:48 +00:00 committed by Evergreen Agent
parent b5e3b9b8aa
commit 209ce30150
24 changed files with 148 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.mypy.ini
View File

@ -1,5 +1,5 @@
[mypy]
python_version = 3.7
python_version = 3.10
disallow_untyped_defs = False
# Do not error on imported files since all imported files may not be mypy clean.

4
SConstruct
View File

@ -38,7 +38,9 @@ import mongo.toolchain as mongo_toolchain
import mongo.generators as mongo_generators
import mongo.install_actions as install_actions
EnsurePythonVersion(3, 6)
# TODO SERVER-79172
# We cannot set the limit to python 3.10 since python 3.9 is needed for windows testing
EnsurePythonVersion(3, 9)
EnsureSConsVersion(3, 1, 1)
utc_starttime = datetime.utcnow()

36
buildscripts/ciconfig/evergreen.py
View File

@ -6,11 +6,11 @@ configuration file.
from __future__ import annotations
import datetime
import distutils.spawn
import shutil
import os
import subprocess
import structlog
import sys
import time
from typing import Set, List, Optional
import yaml
@ -20,20 +20,46 @@ from buildscripts.util import runcommand
ENTERPRISE_MODULE_NAME = "enterprise"
ASAN_SIGNATURE = "detect_leaks=1"
LOGGER = structlog.get_logger(__name__)
def parse_evergreen_file(path, evergreen_binary="evergreen"):
"""Read an Evergreen file and return EvergreenProjectConfig instance."""
if evergreen_binary:
if not distutils.spawn.find_executable(evergreen_binary):
print(f"os.environ={os.environ}")
if not shutil.which(evergreen_binary):
# On windows in python3.8 there was an update to no longer use HOME in os.path.expanduser
# However, cygwin is weird and has HOME but not USERPROFILE
# So we just check if HOME is set and USERPROFILE is not
# Then we just set USERPROFILE and unset it after
# Bug is here: https://bugs.python.org/issue36264
prev_environ = os.environ.copy()
if sys.platform in ("win32", "cygwin"):
LOGGER.info(f"Previous os.environ={os.environ} before updating 'USERPROFILE'")
if 'HOME' in os.environ:
os.environ['USERPROFILE'] = os.environ['HOME']
else:
LOGGER.warn(
"'HOME' enviorment variable unset. This will likely cause us to be unable to find evergreen binary."
)
default_evergreen_location = os.path.expanduser(os.path.join("~", "evergreen"))
# Restore enviorment if it was modified above on windows
os.environ.clear()
os.environ.update(prev_environ)
if os.path.exists(default_evergreen_location):
evergreen_binary = default_evergreen_location
elif os.path.exists(f"{default_evergreen_location}.exe"):
evergreen_binary = f"{default_evergreen_location}.exe"
else:
raise EnvironmentError(
"Executable '{}' does not exist or is not in the PATH.".format(
evergreen_binary))
f"Executable {evergreen_binary} (default location: {default_evergreen_location}) does not exist or is not in the PATH. PATH={os.environ.get('PATH')}"
)
else:
evergreen_binary = shutil.which(evergreen_binary)
# Call 'evergreen evaluate path' to pre-process the project configuration file.
cmd = [evergreen_binary, "evaluate", path]

2
buildscripts/linter/runner.py
View File

@ -82,7 +82,7 @@ def _find_linter(linter, config_dict):
python_dir = os.path.dirname(sys.executable)
if sys.platform == "win32":
# On Windows, these scripts are installed in %PYTHONDIR%\scripts like
# 'C:\Python37\scripts', and have .exe extensions.
# 'C:\python\python310\scripts', and have .exe extensions.
python_dir = os.path.join(python_dir, "scripts")
cmd_str = os.path.join(python_dir, linter.cmd_name)

23
buildscripts/resmokelib/utils/archival.py
View File

@ -126,6 +126,29 @@ class Archival(object):
def _get_s3_client():
# Since boto3 is a 3rd party module, we import locally.
import boto3
import botocore.session
botocore.session.Session()
if sys.platform in ("win32", "cygwin"):
# These overriden values can be found here
# https://github.com/boto/botocore/blob/13468bc9d8923eccd0816ce2dd9cd8de5a6f6e0e/botocore/configprovider.py#L49C7-L49C7
# This is due to the backwards breaking changed python introduced https://bugs.python.org/issue36264
botocore_session = botocore.session.Session(
session_vars={
'config_file': (
None,
'AWS_CONFIG_FILE',
os.path.join(os.environ['HOME'], '.aws', 'config'),
None,
),
'credentials_file': (
None,
'AWS_SHARED_CREDENTIALS_FILE',
os.path.join(os.environ['HOME'], '.aws', 'credentials'),
None,
),
})
boto3.setup_default_session(botocore_session=botocore_session)
return boto3.client("s3")
def archive_files_to_s3(self, display_name, input_files, s3_bucket, s3_path):

4
buildscripts/tests/ciconfig/test_evergreen.py
View File

@ -14,8 +14,12 @@ class TestEvergreenProjectConfig(unittest.TestCase):
@classmethod
def setUpClass(cls):
env = os.environ.copy()
cls.conf = _evergreen.parse_evergreen_file(TEST_FILE_PATH, evergreen_binary=None)
# Assert there is no leakage of env variables from this function
assert env == os.environ
def test_invalid_path(self):
invalid_path = "non_existing_file"
with self.assertRaises(IOError):

6
docs/building.md
View File

@ -19,7 +19,7 @@ To build MongoDB, you will need:
* `libcurl4-gnutls-dev`
* On Ubuntu, the lzma library is required. Install `liblzma-dev`
* On Amazon Linux, the xz-devel library is required. `yum install xz-devel`
* Python 3.7.x and Pip modules:
* Python 3.9.x and Pip modules:
* See the section "Python Prerequisites" below.
* About 13 GB of free disk space for the core binaries (`mongod`,
`mongos`, and `mongo`) and about 600 GB for the install-all target.
@ -43,7 +43,7 @@ The source for the tools is now available at
Python Prerequisites
---------------
In order to build MongoDB, Python 3.7+ is required, and several Python
In order to build MongoDB, Python 3.9+ is required, and several Python
modules must be installed. Python 3 is included in macOS 10.15 and later.
For earlier macOS versions, Python 3 can be installed using Homebrew or
MacPorts or similar.
@ -131,7 +131,7 @@ Windows
Build requirements:
* Visual Studio 2022 version 17.0 or newer
* Python 3.7
* Python 3.9
Or download a prebuilt binary for Windows at www.mongodb.org.

2
docs/golden_data_test_framework.md
View File

@ -180,7 +180,7 @@ buildscripts/golden_test.py setup
Run buildscripts/golden_test.py setup utility.
You may be asked for a password, when not running in "Run as administrator" shell.
```cmd
c:\python\Python37\python.exe buildscripts/golden_test.py setup
c:\python\python310\python.exe buildscripts/golden_test.py setup
```
### Manual Setup (Default config)

16
etc/evergreen.yml
View File

@ -168,7 +168,7 @@ variables:
-j$(( $(grep -c ^processor /proc/cpuinfo) / 2 ))
--win-version-min=win10
num_scons_link_jobs_available: 0.5
python: '/cygdrive/c/python/python37/python.exe'
python: '/cygdrive/c/python/python39/python.exe'
ext: zip
scons_cache_scope: shared
multiversion_platform: windows
@ -551,7 +551,7 @@ buildvariants:
-j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.8")
MONGO_DISTMOD=windows
num_scons_link_jobs_available: 0.2
python: '/cygdrive/c/python/python37/python.exe'
python: '/cygdrive/c/python/python39/python.exe'
ext: zip
scons_cache_scope: shared
multiversion_platform: windows
@ -631,7 +631,7 @@ buildvariants:
burn_in_tests_build_variant: enterprise-windows-all-feature-flags-required
exe: ".exe"
content_type: application/zip
python: '/cygdrive/c/python/python37/python.exe'
python: '/cygdrive/c/python/python39/python.exe'
ext: zip
multiversion_platform: windows
multiversion_edition: enterprise
@ -684,7 +684,7 @@ buildvariants:
burn_in_tests_build_variant: enterprise-windows-all-feature-flags-required
exe: ".exe"
content_type: application/zip
python: '/cygdrive/c/python/python37/python.exe'
python: '/cygdrive/c/python/python39/python.exe'
ext: zip
multiversion_platform: windows
multiversion_edition: enterprise
@ -773,7 +773,7 @@ buildvariants:
-j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.5")
--win-version-min=win10
num_scons_link_jobs_available: 0.25
python: '/cygdrive/c/python/python37/python.exe'
python: '/cygdrive/c/python/python39/python.exe'
ext: zip
resmoke_jobs_max: 1
scons_cache_scope: shared
@ -1890,7 +1890,7 @@ buildvariants:
LIBPATH="c:/sasl/lib"
-j$(( $(grep -c ^processor /proc/cpuinfo) / 2 ))
--win-version-min=win10
python: '/cygdrive/c/python/python37/python.exe'
python: '/cygdrive/c/python/python39/python.exe'
tasks:
- name: build_metrics_gen_TG
@ -2908,7 +2908,7 @@ buildvariants:
- windows-vsCurrent-small
stepback: false
expansions:
python: '/cygdrive/c/python/python37/python.exe'
python: '/cygdrive/c/python/python39/python.exe'
tasks:
- name: win_shared_scons_cache_pruning
@ -2975,7 +2975,7 @@ buildvariants:
LIBPATH="c:/sasl/lib"
-j$(( $(grep -c ^processor /proc/cpuinfo) / 2 ))
--win-version-min=win10
python: '/cygdrive/c/python/python37/python.exe'
python: '/cygdrive/c/python/python39/python.exe'
ext: zip
has_packages: false
scons_cache_scope: shared

2
etc/evergreen_yml_components/variants/compile_static_analysis.yml
View File

@ -205,7 +205,7 @@ buildvariants:
-j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.8")
--win-version-min=win10
num_scons_link_jobs_available: 0.2
python: '/cygdrive/c/python/python37/python.exe'
python: '/cygdrive/c/python/python39/python.exe'
scons_cache_scope: shared
compile_variant: *windows-compile-required
tasks:

2
etc/evergreen_yml_components/variants/in_memory.yml
View File

@ -95,7 +95,7 @@ buildvariants:
LIBPATH="c:/sasl/lib" -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.5")
--win-version-min=win10
num_scons_link_jobs_available: 0.25
python: '/cygdrive/c/python/python37/python.exe'
python: '/cygdrive/c/python/python39/python.exe'
test_flags: >-
--storageEngine=inMemory
--excludeWithAnyTags=requires_persistence,requires_journaling,incompatible_with_windows_tls

4
etc/evergreen_yml_components/variants/misc_release.yml
View File

@ -2514,7 +2514,7 @@ buildvariants:
--win-version-min=win10
--use-diagnostic-latches=off
num_scons_link_jobs_available: 0.25
python: '/cygdrive/c/python/python37/python.exe'
python: '/cygdrive/c/python/python39/python.exe'
ext: zip
scons_cache_scope: shared
large_distro_name: windows-vsCurrent-large
@ -2580,7 +2580,7 @@ buildvariants:
--win-version-min=win10
--use-diagnostic-latches=off
num_scons_link_jobs_available: 0.25
python: '/cygdrive/c/python/python37/python.exe'
python: '/cygdrive/c/python/python39/python.exe'
ext: zip
scons_cache_scope: shared
multiversion_platform: windows

2
etc/pip/components/external_auth.req
View File

@ -1,5 +1,5 @@
# These are the dependencies of ldaptor
passlib == 1.7.1
passlib == 1.7.4
pyOpenSSL == 19.0.0; platform_machine == "s390x" or platform_machine == "ppc64le" # Needed for pinned cryptography package - see SERVER-70845
pyOpenSSL == 22.0.0; platform_machine != "s390x" and platform_machine != "ppc64le"
pyparsing == 2.4.0

2
evergreen/prelude_python.sh
View File

@ -1,5 +1,5 @@
if [ "Windows_NT" = "$OS" ]; then
python='/cygdrive/c/python/python37/python.exe'
python='/cygdrive/c/python/python39/python.exe'
else
if [ -f /opt/mongodbtoolchain/v4/bin/python3 ]; then
python="/opt/mongodbtoolchain/v4/bin/python3"

8
jstests/free_mon/libs/free_mon.js
View File

@ -3,6 +3,8 @@
* Control the Free Monitoring Mock Webserver.
*/
load("jstests/libs/python.js");
// These faults must match the list of faults in mock_http_server.py, see the
// SUPPORTED_FAULT_TYPES list in mock_http_server.py
const FAULT_FAIL_REGISTER = "fail_register";
@ -23,14 +25,10 @@ class FreeMonWebServer {
* @param {bool} disableFaultsOnStartup optionally disable fault on startup
*/
constructor(fault_type, disableFaultsOnStartup) {
this.python = "python3";
this.python = getPython3Binary();
this.disableFaultsOnStartup = disableFaultsOnStartup || false;
this.fault_type = fault_type;
if (_isWindows()) {
this.python = "python.exe";
}
print("Using python interpreter: " + this.python);
this.web_server_py = "jstests/free_mon/libs/mock_http_server.py";
this.control_py = "jstests/free_mon/libs/mock_http_control.py";

48
jstests/libs/python.js
View File

@ -3,25 +3,41 @@
function getPython3Binary() {
'use strict';
let cmd = '/opt/mongodbtoolchain/v4/bin/python3';
if (_isWindows()) {
const paths = ["c:/python36/python.exe", "c:/python/python36/python.exe"];
for (let p of paths) {
if (fileExists(p)) {
cmd = p;
break;
}
// On windows it is important to use python vs python3
// or else we will pick up a python that is not in our venv
clearRawMongoProgramOutput();
assert.eq(runNonMongoProgram("python", "--version"), 0);
const pythonVersion = rawMongoProgramOutput(); // Will look like "Python 3.10.4\n"
const usingPython39 = /Python 3\.9/.exec(pythonVersion);
const usingPython310 = /Python 3\.10/.exec(pythonVersion);
if (usingPython310) {
print(
"Found python 3.10 by default. Likely this is because we are using a virtual enviorment.");
return "python";
} else if (usingPython39) {
// TODO: SERVER-79172
// Once the above ticket is complete we should stop using python 3.9 on windows and upgrade
// to python 310 everywhere To solve: grep for python39 and fix instances of it
print(
"Found python 3.9 by default. Likely this is because we are using a windows virtual enviorment.");
return "python";
}
const paths = [
"/opt/mongodbtoolchain/v4/bin/python3",
"/cygdrive/c/python/python310/python.exe",
"c:/python/python310/python.exe"
];
for (let p of paths) {
if (fileExists(p)) {
print("Found python3 in default location " + p);
return p;
}
}
if (fileExists(cmd)) {
return cmd;
}
clearRawMongoProgramOutput();
assert.eq(runNonMongoProgram("python", "--version"), 0);
const pythonVersion = rawMongoProgramOutput();
assert(/Python 3/.exec(pythonVersion));
return "python";
// We are probs running on mac
print("Did not find python3 in a virtualenv or default location");
return "python3";
}

5
jstests/noPassthrough/libs/backup_restore.js
View File

@ -19,6 +19,8 @@
* }
*/
load("jstests/libs/python.js");
var BackupRestoreTest = function(options) {
"use strict";
@ -123,7 +125,8 @@ var BackupRestoreTest = function(options) {
function _fsmClient(host) {
// Launch FSM client
const suite = 'concurrency_replication_for_backup_restore';
const resmokeCmd = 'python buildscripts/resmoke.py run --shuffle --continueOnFailure' +
const resmokeCmd = getPython3Binary() +
' buildscripts/resmoke.py run --shuffle --continueOnFailure' +
' --repeat=99999 --internalParam=is_inner_level --mongo=' +
MongoRunner.getMongoShellPath() + ' --shellConnString=mongodb://' + host +
' --suites=' + suite;

3
jstests/noPassthrough/libs/configExpand/lib.js
View File

@ -88,9 +88,6 @@ function makeReflectionCmd(arg, opts = {}) {
load('jstests/libs/python.js');
let cmd = getPython3Binary();
if (_isWindows()) {
cmd = '"' + cmd + '"';
}
cmd += ' jstests/noPassthrough/libs/configExpand/reflect.py';
if (opts.sleep && (opts.sleep > 0)) {

7
jstests/ocsp/lib/mock_ocsp.js
View File

@ -3,6 +3,7 @@
* OCSP certificate revocation.
*/
load("jstests/ocsp/lib/ocsp_helpers.js");
load("jstests/libs/python.js");
// These are a list of faults to match the list of faults
// in ocsp_mock.py.
@ -46,13 +47,9 @@ class MockOCSPServer {
response_delay_secs = 0,
include_extraneous_status = false,
issuer_hash_algorithm = "") {
this.python = "python3";
this.python = getPython3Binary();
this.fault_type = fault_type;
if (_isWindows()) {
this.python = "python.exe";
}
this.ca_file = responder_certificate_set.cafile;
this.ocsp_cert_file = responder_certificate_set.certfile;
this.ocsp_cert_key = responder_certificate_set.keyfile;

4
jstests/resmoke_selftest/end2end/timeout/nested/top_level_timeout.js
View File

@ -1,6 +1,8 @@
(function() {
'use strict';
load("jstests/libs/python.js");
const rst = new ReplSetTest({nodes: 2});
rst.startSet();
rst.initiate();
@ -10,7 +12,7 @@ function start() {
// because when we pass --taskId into resmoke it thinks that it is being ran in evergreen
// and cannot normally find an evergreen task associated with
// buildscripts/tests/resmoke_end2end/suites/resmoke_selftest_nested_timeout.yml
const resmokeCmd = 'python3 buildscripts/resmoke.py run ' +
const resmokeCmd = getPython3Binary() + ' buildscripts/resmoke.py run ' +
'--storageEngineCacheSizeGB=1 --dbpathPrefix=/data/db/selftest_inner ' +
'--internalParam=test_archival --taskId=123 ' +
'--originSuite=resmoke_end2end_tests ' +

8
jstests/sharding/libs/proxy_protocol.js
View File

@ -1,16 +1,14 @@
/**
* Control the proxy protocol server.
*/
load("jstests/libs/python.js");
class ProxyProtocolServer {
/**
* Create a new proxy protocol server.
*/
constructor(ingress_port, egress_port, version) {
this.python = "python3";
if (_isWindows()) {
this.python = "python.exe";
}
this.python = getPython3Binary();
print("Using python interpreter: " + this.python);
this.web_server_py = "jstests/sharding/libs/proxy_protocol_server.py";

7
jstests/ssl/ssl_ECDHE_suites.js
View File

@ -1,6 +1,7 @@
// Test that the server supports ECDHE and DHE tls cipher suites.
load('jstests/ssl/libs/ssl_helpers.js');
load("jstests/libs/python.js");
(function() {
"use strict";
@ -35,11 +36,7 @@ const x509_options = {
const mongod = MongoRunner.runMongod(x509_options);
// Use new toolchain python, if it exists
let python_binary = '/opt/mongodbtoolchain/v4/bin/python3';
if (runProgram('/bin/sh', '-c', 'ls ' + python_binary) !== 0) {
// We are running on mac
python_binary = 'python3';
}
let python_binary = getPython3Binary();
// Run the tls cipher suite enumerator
const python = '/usr/bin/env ' + python_binary;

5
jstests/watchdog/lib/charybdefs_lib.js
View File

@ -6,10 +6,13 @@ const EXIT_WATCHDOG = 61;
*
* @param {string} test_name unique name for test directories
*/
load("jstests/libs/python.js");
function CharybdefsControl(test_name) {
'use strict';
const python = "python";
const python = getPython3Binary();
let control_py = "/data/charybdefs/mongo/control.py";
// Use the minimum watchdog period

16
src/mongo/shell/servers.js
View File

@ -1,3 +1,5 @@
load("jstests/libs/python.js");
var MongoRunner, _startMongod, startMongoProgram, runMongoProgram, startMongoProgramNoConnect,
myPort;
@ -169,8 +171,18 @@ function runHangAnalyzer(pids) {
print(`Running hang analyzer for pids [${pids}]`);
const scriptPath = pathJoin('.', 'buildscripts', 'resmoke.py');
const args =
['python', scriptPath, 'hang-analyzer', '-k', '-o', 'file', '-o', 'stdout', '-d', pids];
const args = [
getPython3Binary(),
scriptPath,
'hang-analyzer',
'-k',
'-o',
'file',
'-o',
'stdout',
'-d',
pids
];
if (jsTest.options().evergreenTaskId) {
args.push('-t', jsTest.options().evergreenTaskId);