SERVER-95813: fix for github token issue (#28190)

GitOrigin-RevId: 603a9fb8e00f358033bdbb541281226577a31676
2024-11-24 00:17:37 +01:00 · 2024-10-17 13:11:09 -04:00 · 2024-10-17 13:11:09 -04:00 · 23426d6ab2
commit 23426d6ab2
parent cd01a53d56
11 changed files with 97 additions and 17 deletions
--- a/etc/evergreen_yml_components/definitions.yml
+++ b/etc/evergreen_yml_components/definitions.yml
@ -1473,16 +1473,36 @@ functions:

  "do jepsen setup":
    - *f_expansions_write
+    - command: github.generate_token
+      params:
+        owner: 10gen
+        repo: libfaketime
+        expansion_name: github_token
+        permissions:
+          metadata: read
+          contents: read
    - command: subprocess.exec
      params:
        binary: bash
        args:
          - "./src/evergreen/do_jepsen_setup/build_libfaketime.sh"
+        include_expansions_in_env:
+          - github_token
+    - command: github.generate_token
+      params:
+        owner: 10gen
+        repo: jepsen
+        expansion_name: github_token
+        permissions:
+          metadata: read
+          contents: read
    - command: subprocess.exec
      params:
        binary: bash
        args:
          - "./src/evergreen/do_jepsen_setup/install_jepsen.sh"
+        include_expansions_in_env:
+          - github_token
    - command: subprocess.exec
      params:
        binary: bash
@ -1496,12 +1516,22 @@ functions:

  "do jepsen docker setup":
    - *f_expansions_write
+    - command: github.generate_token
+      params:
+        owner: 10gen
+        repo: jepsen-io-mongodb
+        expansion_name: github_token
+        permissions:
+          metadata: read
+          contents: read
    - command: subprocess.exec
      type: system
      params:
        binary: bash
        args:
          - "./src/evergreen/jepsen_docker/setup.sh"
+        include_expansions_in_env:
+          - github_token
  "setup jepsen config fuzzer":
    - *f_expansions_write
    - command: subprocess.exec
@ -1584,12 +1614,22 @@ functions:

  "setup jstestfuzz":
    - *f_expansions_write
+    - command: github.generate_token
+      params:
+        owner: 10gen
+        repo: jstestfuzz
+        expansion_name: github_token
+        permissions:
+          metadata: read
+          contents: read
    - command: subprocess.exec
      display_name: "jstestfuzz setup sh"
      params:
        binary: bash
        args:
          - "./src/evergreen/jstestfuzz_setup.sh"
+        include_expansions_in_env:
+          - github_token

  "lint fuzzer sanity patch":
    - *f_expansions_write
@ -1632,6 +1672,14 @@ functions:

  # Used by generator
  "run jstestfuzz":
+    - command: github.generate_token
+      params:
+        owner: 10gen
+        repo: QA
+        expansion_name: github_token
+        permissions:
+          metadata: read
+          contents: read
    - *f_expansions_write
    - command: subprocess.exec
      display_name: "clone repos sh"
@ -1639,6 +1687,8 @@ functions:
        binary: bash
        args:
          - "./src/evergreen/run_jstestfuzz/clone_repos.sh"
+        include_expansions_in_env:
+          - github_token
    - *f_expansions_write
    - command: subprocess.exec
      display_name: "jstestfuzz run sh"
--- a/etc/evergreen_yml_components/tasks/misc_tasks.yml
+++ b/etc/evergreen_yml_components/tasks/misc_tasks.yml
@ -431,12 +431,22 @@ tasks:
    tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"]
    commands:
      - func: "f_expansions_write"
+      - command: github.generate_token
+        params:
+          owner: 10gen
+          repo: coredb-patchbuild-optimizer
+          expansion_name: github_token
+          permissions:
+            metadata: read
+            contents: read
      - command: subprocess.exec
        type: test
        params:
          binary: bash
          args:
            - "src/evergreen/gen_patch_test_tags.sh"
+          include_expansions_in_env:
+            - "github_token"

  - name: generate_buildid_to_debug_symbols_mapping
    tags:
--- a/evergreen/do_jepsen_setup/build_libfaketime.sh
+++ b/evergreen/do_jepsen_setup/build_libfaketime.sh
@ -6,7 +6,8 @@ cd src
 # Additionally, a version of libfaketime containing the changes mentioned in SERVER-29336
 # is required to avoid needing to use libfaketimeMT.so.1 and serializing all calls to
 # fake_clock_gettime() with a mutex.
-git clone --branch=for-jepsen --depth=1 git@github.com:10gen/libfaketime.git
+
+git clone --branch=for-jepsen --depth=1 https://x-access-token:${github_token}@github.com/10gen/libfaketime.git
 cd libfaketime
 branch=$(git symbolic-ref --short HEAD)
 commit=$(git show -s --pretty=format:"%h - %an, %ar: %s")
--- a/evergreen/do_jepsen_setup/install_jepsen.sh
+++ b/evergreen/do_jepsen_setup/install_jepsen.sh
@ -1,7 +1,8 @@
 set -o errexit

 cd src
-git clone --branch=v0.3.0-jepsen-mongodb-master --depth=1 git@github.com:10gen/jepsen.git jepsen-mongodb
+
+git clone --branch=v0.3.0-jepsen-mongodb-master --depth=1 https://x-access-token:${github_token}@github.com/10gen/jepsen.git jepsen-mongodb
 cd jepsen-mongodb

 lein install
--- a/evergreen/gen_patch_test_tags.sh
+++ b/evergreen/gen_patch_test_tags.sh
@ -12,7 +12,7 @@ if [[ "${build_variant}" != *"query-patch-only"* ]]; then
 fi

 activate_venv
-git clone git@github.com:10gen/coredb-patchbuild-optimizer.git
+git clone https://x-access-token:${github_token}@github.com/10gen/coredb-patchbuild-optimizer.git

 pushd coredb-patchbuild-optimizer
 # Reusing bfsuggestion's password here to avoid having to
--- a/evergreen/jepsen_docker/setup.sh
+++ b/evergreen/jepsen_docker/setup.sh
@ -9,7 +9,7 @@ git clone --branch=v0.2.0-evergreen-master git@github.com:10gen/jepsen.git jepse
 cp -rf src/dist-test jepsen/docker/node

 # Clone our internal tests to run
-git clone --branch=v0.2.1 git@github.com:10gen/jepsen-io-mongodb.git jepsen/docker/control/mongodb
+git clone --branch=v0.2.1 https://x-access-token:${github_token}@github.com/10gen/jepsen-io-mongodb.git jepsen/docker/control/mongodb

 # Kill any running containers
 sudo docker container kill $(docker ps -q) || true
--- a/evergreen/jstestfuzz_setup.sh
+++ b/evergreen/jstestfuzz_setup.sh
@ -7,8 +7,8 @@ set -o errexit
 set -o verbose

 for i in {1..5}; do
-  git clone git@github.com:10gen/jstestfuzz.git && RET=0 && break || RET=$? && sleep 5
-  echo "Failed to clone git@github.com:10gen/jstestfuzz.git, retrying..."
+  git clone https://x-access-token:${github_token}@github.com/10gen/jstestfuzz.git && RET=0 && break || RET=$? && sleep 5
+  echo "Failed to clone github.com:10gen/jstestfuzz.git, retrying..."
 done

 if [ $RET -ne 0 ]; then
--- a/evergreen/prelude_mongo_task_generator.sh
+++ b/evergreen/prelude_mongo_task_generator.sh
@ -1,6 +1,6 @@
 function setup_mongo_task_generator {
  if [ ! -f mongo-task-generator ]; then
-    curl -L https://github.com/mongodb/mongo-task-generator/releases/download/v0.7.15/mongo-task-generator --output mongo-task-generator
+    curl -L https://github.com/mongodb/mongo-task-generator/releases/download/v0.7.17/mongo-task-generator --output mongo-task-generator
    chmod +x mongo-task-generator
  fi
 }
--- a/evergreen/run_jstestfuzz/clone_repos.sh
+++ b/evergreen/run_jstestfuzz/clone_repos.sh
@ -2,13 +2,14 @@ set -o errexit
 set -o verbose

 cd src
+
 for i in {1..5}; do
-  git clone --depth 1 git@github.com:10gen/QA.git jstests/qa_tests && RET=0 && break || RET=$? && sleep 5
-  echo "Failed to clone git@github.com:10gen/QA.git, retrying..."
+  git clone --depth 1 https://x-access-token:${github_token}@github.com/10gen/QA.git jstests/qa_tests && RET=0 && break || RET=$? && sleep 1
+  echo "Failed to clone github.com:10gen/QA.git, retrying..."
 done

 if [ $RET -ne 0 ]; then
-  echo "Failed to clone git@github.com:10gen/QA.git"
+  echo "Failed to clone github.com/10gen/QA.git"
  exit $RET
 fi

--- a/poetry.lock
+++ b/poetry.lock
@ -477,6 +477,20 @@ files = [
    {file = "crashtest-0.4.1.tar.gz", hash = "sha256:80d7b1f316ebfbd429f648076d6275c877ba30ba48979de4191714a75266f0ce"},
 ]

+[[package]]
+name = "croniter"
+version = "1.4.1"
+description = "croniter provides iteration for datetime object with cron like format"
+optional = false
+python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+files = [
+    {file = "croniter-1.4.1-py2.py3-none-any.whl", hash = "sha256:9595da48af37ea06ec3a9f899738f1b2c1c13da3c38cea606ef7cd03ea421128"},
+    {file = "croniter-1.4.1.tar.gz", hash = "sha256:1a6df60eacec3b7a0aa52a8f2ef251ae3dd2a7c7c8b9874e73e791636d55a361"},
+]
+
+[package.dependencies]
+python-dateutil = "*"
+
 [[package]]
 name = "cryptography"
 version = "2.3"
@ -3566,17 +3580,20 @@ files = [

 [[package]]
 name = "shrub-py"
-version = "1.1.4"
+version = "3.3.0"
 description = "Library for creating evergreen configurations"
 optional = false
-python-versions = ">=3.6,<4.0"
+python-versions = ">3.7.0"
 files = [
-    {file = "shrub.py-1.1.4-py3-none-any.whl", hash = "sha256:bb7149948447f718b150d500973e981e1d34014675a5ff1a3c22737a632f6aeb"},
-    {file = "shrub.py-1.1.4.tar.gz", hash = "sha256:d062ad131c7121dc0c2ffbb275e0a1199418911b7202193bb38b3b30d0d8361c"},
+    {file = "shrub_py-3.3.0-py3-none-any.whl", hash = "sha256:e7e663ebceeb088cdf4ebdce8df9e0581501f5cf7e5b3ea4295b8e76706dfb74"},
+    {file = "shrub_py-3.3.0.tar.gz", hash = "sha256:974526ef9f058159b565d7302f84a5749b88e1512c68d4f5bc7650ab6fe663c4"},
 ]

 [package.dependencies]
-PyYaml = ">=5.1,<6.0"
+croniter = ">=1.4.1,<2.0.0"
+pydantic = ">=1.8,<3.0"
+PyYaml = ">=5.1,<7.0"
+typing-extensions = ">=4,<5"

 [[package]]
 name = "six"
@ -4382,4 +4399,4 @@ libdeps = ["cxxfilt", "eventlet", "flask", "flask-cors", "gevent", "lxml", "prog
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.10,<4.0"
-content-hash = "e220a14c6f8ee1cbde79864922637ae9aef0cfb5b93e9c18e41b8609679d4fb1"
+content-hash = "85b3d757dae015ed1f645a9875e1a39b36b518d4ef9a0ec8e4b2d1588a6b0248"
--- a/pyproject.toml
+++ b/pyproject.toml
@ -129,7 +129,7 @@ PyKMIP = "0.10.0"
 evergreen-py = "^3.9.0"
 jinja2 = "^3.1.4"
 mock = "^5.1.0"
-shrub-py = "^1.1.4"
+shrub-py = "^3.1.4"
 ocspresponder = "^0.5.0"
 flask = "^2.3.3"
 ocspbuilder = "^0.10.2"