diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml index 91b2df21363..64960d50914 100644 --- a/etc/evergreen_yml_components/definitions.yml +++ b/etc/evergreen_yml_components/definitions.yml @@ -1473,16 +1473,36 @@ functions: "do jepsen setup": - *f_expansions_write + - command: github.generate_token + params: + owner: 10gen + repo: libfaketime + expansion_name: github_token + permissions: + metadata: read + contents: read - command: subprocess.exec params: binary: bash args: - "./src/evergreen/do_jepsen_setup/build_libfaketime.sh" + include_expansions_in_env: + - github_token + - command: github.generate_token + params: + owner: 10gen + repo: jepsen + expansion_name: github_token + permissions: + metadata: read + contents: read - command: subprocess.exec params: binary: bash args: - "./src/evergreen/do_jepsen_setup/install_jepsen.sh" + include_expansions_in_env: + - github_token - command: subprocess.exec params: binary: bash @@ -1496,12 +1516,22 @@ functions: "do jepsen docker setup": - *f_expansions_write + - command: github.generate_token + params: + owner: 10gen + repo: jepsen-io-mongodb + expansion_name: github_token + permissions: + metadata: read + contents: read - command: subprocess.exec type: system params: binary: bash args: - "./src/evergreen/jepsen_docker/setup.sh" + include_expansions_in_env: + - github_token "setup jepsen config fuzzer": - *f_expansions_write - command: subprocess.exec @@ -1584,12 +1614,22 @@ functions: "setup jstestfuzz": - *f_expansions_write + - command: github.generate_token + params: + owner: 10gen + repo: jstestfuzz + expansion_name: github_token + permissions: + metadata: read + contents: read - command: subprocess.exec display_name: "jstestfuzz setup sh" params: binary: bash args: - "./src/evergreen/jstestfuzz_setup.sh" + include_expansions_in_env: + - github_token "lint fuzzer sanity patch": - *f_expansions_write @@ -1632,6 +1672,14 @@ functions: # Used by generator "run jstestfuzz": + - command: github.generate_token + params: + owner: 10gen + repo: QA + expansion_name: github_token + permissions: + metadata: read + contents: read - *f_expansions_write - command: subprocess.exec display_name: "clone repos sh" @@ -1639,6 +1687,8 @@ functions: binary: bash args: - "./src/evergreen/run_jstestfuzz/clone_repos.sh" + include_expansions_in_env: + - github_token - *f_expansions_write - command: subprocess.exec display_name: "jstestfuzz run sh" diff --git a/etc/evergreen_yml_components/tasks/misc_tasks.yml b/etc/evergreen_yml_components/tasks/misc_tasks.yml index 561b49669e2..8c39eefc3bf 100644 --- a/etc/evergreen_yml_components/tasks/misc_tasks.yml +++ b/etc/evergreen_yml_components/tasks/misc_tasks.yml @@ -431,12 +431,22 @@ tasks: tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"] commands: - func: "f_expansions_write" + - command: github.generate_token + params: + owner: 10gen + repo: coredb-patchbuild-optimizer + expansion_name: github_token + permissions: + metadata: read + contents: read - command: subprocess.exec type: test params: binary: bash args: - "src/evergreen/gen_patch_test_tags.sh" + include_expansions_in_env: + - "github_token" - name: generate_buildid_to_debug_symbols_mapping tags: diff --git a/evergreen/do_jepsen_setup/build_libfaketime.sh b/evergreen/do_jepsen_setup/build_libfaketime.sh index fdba0af2f4d..a488ced84a8 100755 --- a/evergreen/do_jepsen_setup/build_libfaketime.sh +++ b/evergreen/do_jepsen_setup/build_libfaketime.sh @@ -6,7 +6,8 @@ cd src # Additionally, a version of libfaketime containing the changes mentioned in SERVER-29336 # is required to avoid needing to use libfaketimeMT.so.1 and serializing all calls to # fake_clock_gettime() with a mutex. -git clone --branch=for-jepsen --depth=1 git@github.com:10gen/libfaketime.git + +git clone --branch=for-jepsen --depth=1 https://x-access-token:${github_token}@github.com/10gen/libfaketime.git cd libfaketime branch=$(git symbolic-ref --short HEAD) commit=$(git show -s --pretty=format:"%h - %an, %ar: %s") diff --git a/evergreen/do_jepsen_setup/install_jepsen.sh b/evergreen/do_jepsen_setup/install_jepsen.sh index 3c2a6c0f08a..1cb03a279a2 100755 --- a/evergreen/do_jepsen_setup/install_jepsen.sh +++ b/evergreen/do_jepsen_setup/install_jepsen.sh @@ -1,7 +1,8 @@ set -o errexit cd src -git clone --branch=v0.3.0-jepsen-mongodb-master --depth=1 git@github.com:10gen/jepsen.git jepsen-mongodb + +git clone --branch=v0.3.0-jepsen-mongodb-master --depth=1 https://x-access-token:${github_token}@github.com/10gen/jepsen.git jepsen-mongodb cd jepsen-mongodb lein install diff --git a/evergreen/gen_patch_test_tags.sh b/evergreen/gen_patch_test_tags.sh index d1656f6475e..2d9e4ce0631 100644 --- a/evergreen/gen_patch_test_tags.sh +++ b/evergreen/gen_patch_test_tags.sh @@ -12,7 +12,7 @@ if [[ "${build_variant}" != *"query-patch-only"* ]]; then fi activate_venv -git clone git@github.com:10gen/coredb-patchbuild-optimizer.git +git clone https://x-access-token:${github_token}@github.com/10gen/coredb-patchbuild-optimizer.git pushd coredb-patchbuild-optimizer # Reusing bfsuggestion's password here to avoid having to diff --git a/evergreen/jepsen_docker/setup.sh b/evergreen/jepsen_docker/setup.sh index 56426d0de50..a5d906b4f0a 100644 --- a/evergreen/jepsen_docker/setup.sh +++ b/evergreen/jepsen_docker/setup.sh @@ -9,7 +9,7 @@ git clone --branch=v0.2.0-evergreen-master git@github.com:10gen/jepsen.git jepse cp -rf src/dist-test jepsen/docker/node # Clone our internal tests to run -git clone --branch=v0.2.1 git@github.com:10gen/jepsen-io-mongodb.git jepsen/docker/control/mongodb +git clone --branch=v0.2.1 https://x-access-token:${github_token}@github.com/10gen/jepsen-io-mongodb.git jepsen/docker/control/mongodb # Kill any running containers sudo docker container kill $(docker ps -q) || true diff --git a/evergreen/jstestfuzz_setup.sh b/evergreen/jstestfuzz_setup.sh index 83623ca6f16..3c35305b697 100644 --- a/evergreen/jstestfuzz_setup.sh +++ b/evergreen/jstestfuzz_setup.sh @@ -7,8 +7,8 @@ set -o errexit set -o verbose for i in {1..5}; do - git clone git@github.com:10gen/jstestfuzz.git && RET=0 && break || RET=$? && sleep 5 - echo "Failed to clone git@github.com:10gen/jstestfuzz.git, retrying..." + git clone https://x-access-token:${github_token}@github.com/10gen/jstestfuzz.git && RET=0 && break || RET=$? && sleep 5 + echo "Failed to clone github.com:10gen/jstestfuzz.git, retrying..." done if [ $RET -ne 0 ]; then diff --git a/evergreen/prelude_mongo_task_generator.sh b/evergreen/prelude_mongo_task_generator.sh index 717af290acb..343942139a3 100644 --- a/evergreen/prelude_mongo_task_generator.sh +++ b/evergreen/prelude_mongo_task_generator.sh @@ -1,6 +1,6 @@ function setup_mongo_task_generator { if [ ! -f mongo-task-generator ]; then - curl -L https://github.com/mongodb/mongo-task-generator/releases/download/v0.7.15/mongo-task-generator --output mongo-task-generator + curl -L https://github.com/mongodb/mongo-task-generator/releases/download/v0.7.17/mongo-task-generator --output mongo-task-generator chmod +x mongo-task-generator fi } diff --git a/evergreen/run_jstestfuzz/clone_repos.sh b/evergreen/run_jstestfuzz/clone_repos.sh index d4fc05368e8..8acbc78c36e 100755 --- a/evergreen/run_jstestfuzz/clone_repos.sh +++ b/evergreen/run_jstestfuzz/clone_repos.sh @@ -2,13 +2,14 @@ set -o errexit set -o verbose cd src + for i in {1..5}; do - git clone --depth 1 git@github.com:10gen/QA.git jstests/qa_tests && RET=0 && break || RET=$? && sleep 5 - echo "Failed to clone git@github.com:10gen/QA.git, retrying..." + git clone --depth 1 https://x-access-token:${github_token}@github.com/10gen/QA.git jstests/qa_tests && RET=0 && break || RET=$? && sleep 1 + echo "Failed to clone github.com:10gen/QA.git, retrying..." done if [ $RET -ne 0 ]; then - echo "Failed to clone git@github.com:10gen/QA.git" + echo "Failed to clone github.com/10gen/QA.git" exit $RET fi diff --git a/poetry.lock b/poetry.lock index 15083ff9206..4751d6e8f6c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -477,6 +477,20 @@ files = [ {file = "crashtest-0.4.1.tar.gz", hash = "sha256:80d7b1f316ebfbd429f648076d6275c877ba30ba48979de4191714a75266f0ce"}, ] +[[package]] +name = "croniter" +version = "1.4.1" +description = "croniter provides iteration for datetime object with cron like format" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "croniter-1.4.1-py2.py3-none-any.whl", hash = "sha256:9595da48af37ea06ec3a9f899738f1b2c1c13da3c38cea606ef7cd03ea421128"}, + {file = "croniter-1.4.1.tar.gz", hash = "sha256:1a6df60eacec3b7a0aa52a8f2ef251ae3dd2a7c7c8b9874e73e791636d55a361"}, +] + +[package.dependencies] +python-dateutil = "*" + [[package]] name = "cryptography" version = "2.3" @@ -3566,17 +3580,20 @@ files = [ [[package]] name = "shrub-py" -version = "1.1.4" +version = "3.3.0" description = "Library for creating evergreen configurations" optional = false -python-versions = ">=3.6,<4.0" +python-versions = ">3.7.0" files = [ - {file = "shrub.py-1.1.4-py3-none-any.whl", hash = "sha256:bb7149948447f718b150d500973e981e1d34014675a5ff1a3c22737a632f6aeb"}, - {file = "shrub.py-1.1.4.tar.gz", hash = "sha256:d062ad131c7121dc0c2ffbb275e0a1199418911b7202193bb38b3b30d0d8361c"}, + {file = "shrub_py-3.3.0-py3-none-any.whl", hash = "sha256:e7e663ebceeb088cdf4ebdce8df9e0581501f5cf7e5b3ea4295b8e76706dfb74"}, + {file = "shrub_py-3.3.0.tar.gz", hash = "sha256:974526ef9f058159b565d7302f84a5749b88e1512c68d4f5bc7650ab6fe663c4"}, ] [package.dependencies] -PyYaml = ">=5.1,<6.0" +croniter = ">=1.4.1,<2.0.0" +pydantic = ">=1.8,<3.0" +PyYaml = ">=5.1,<7.0" +typing-extensions = ">=4,<5" [[package]] name = "six" @@ -4382,4 +4399,4 @@ libdeps = ["cxxfilt", "eventlet", "flask", "flask-cors", "gevent", "lxml", "prog [metadata] lock-version = "2.0" python-versions = ">=3.10,<4.0" -content-hash = "e220a14c6f8ee1cbde79864922637ae9aef0cfb5b93e9c18e41b8609679d4fb1" +content-hash = "85b3d757dae015ed1f645a9875e1a39b36b518d4ef9a0ec8e4b2d1588a6b0248" diff --git a/pyproject.toml b/pyproject.toml index 9388b6dd42f..5a37557036a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -129,7 +129,7 @@ PyKMIP = "0.10.0" evergreen-py = "^3.9.0" jinja2 = "^3.1.4" mock = "^5.1.0" -shrub-py = "^1.1.4" +shrub-py = "^3.1.4" ocspresponder = "^0.5.0" flask = "^2.3.3" ocspbuilder = "^0.10.2"