From f453fa36c72edb67a52f16f62d8b82d18794dbaf Mon Sep 17 00:00:00 2001 From: SimonEismann <31125121+SimonEismann@users.noreply.github.com> Date: Thu, 1 Aug 2024 15:43:12 +0200 Subject: [PATCH] SERVER-93061 Use shallow clones in mongodb-mongo-master (#25530) GitOrigin-RevId: bc86725a7bbffe12113c16dc70cf50c8ba11b400 --- .../evergreen_gen_powercycle_tasks.py | 3 +- etc/evergreen.yml | 3 +- etc/evergreen_yml_components/definitions.yml | 44 +++++++++++++++---- .../tasks/misc_tasks.yml | 12 +++-- evergreen/restore_git_history_and_tags.sh | 28 ++++++++++++ 5 files changed, 76 insertions(+), 14 deletions(-) create mode 100644 evergreen/restore_git_history_and_tags.sh diff --git a/buildscripts/evergreen_gen_powercycle_tasks.py b/buildscripts/evergreen_gen_powercycle_tasks.py index b665b99b6e7..28107058d64 100755 --- a/buildscripts/evergreen_gen_powercycle_tasks.py +++ b/buildscripts/evergreen_gen_powercycle_tasks.py @@ -79,7 +79,8 @@ def get_skip_compile_setup_commands() -> Tuple[List[FunctionCall], set]: """Return skip compile setup commands.""" return [ BuiltInCommand("manifest.load", {}), - FunctionCall("git get project"), + FunctionCall("git get shallow project"), + FunctionCall("restore git history and tags"), FunctionCall("f_expansions_write"), FunctionCall("kill processes"), FunctionCall("cleanup environment"), diff --git a/etc/evergreen.yml b/etc/evergreen.yml index e2a4d528982..c44e6a62dbd 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -44,7 +44,8 @@ # 2. If your task does not depend on 'archive_dist_test'/'archive_dist_test_debug' task use the following functions # call sequence # - command: manifest.load -# - func: "git get project" (clone the entire mongo and enterprise repos) +# - func: "git get shallow project" (clone the entire mongo and enterprise repos) +# - func: "restore git history and tags" # - func: "f_expansions_write" # - func: "kill processes" # - func: "cleanup environment" diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml index 3149566b725..b8d7ec8adf7 100644 --- a/etc/evergreen_yml_components/definitions.yml +++ b/etc/evergreen_yml_components/definitions.yml @@ -74,13 +74,24 @@ functions: args: - "src/evergreen/functions/evergreen_api_credentials_configure.sh" - "git get project": &git_get_project + "git get shallow project": &git_get_shallow_project command: git.get_project - display_name: "git get project" + display_name: "git get shallow project" params: directory: ${git_project_directory|src} revisions: # for each module include revision as : ${_rev} wtdevelop: ${wtdevelop_rev} + clone_depth: 1000 + + "restore git history and tags": &restore_git_history_and_tags + command: subprocess.exec + display_name: "restore git history and tags" + params: + binary: bash + args: + - "src/evergreen/restore_git_history_and_tags.sh" + env: + dir: src # Get get the mongo repo, no modules. Useful for inspecting the commit history with the # `git` Python tool. @@ -89,6 +100,8 @@ functions: - command: git.get_project params: directory: ${git_project_directory|src} + clone_depth: 1000 + - *restore_git_history_and_tags "add git tag": &add_git_tag command: subprocess.exec @@ -103,7 +116,8 @@ functions: "git get project and add git tag": - *f_expansions_write - - *git_get_project + - *git_get_shallow_project + - *restore_git_history_and_tags - *add_git_tag "gen feature flags": &gen_feature_flags @@ -725,12 +739,22 @@ functions: - command: git.get_project params: directory: src/buildscripts/antithesis/base_images/workload/src + clone_depth: 1000 + - command: subprocess.exec + display_name: "restore git history and tags" + params: + binary: bash + args: + - "src/evergreen/restore_git_history_and_tags.sh" + env: + dir: src/buildscripts/antithesis/base_images/workload/src - *configure_evergreen_api_credentials "do non-compile setup": - command: manifest.load - - *git_get_project + - *git_get_shallow_project - *f_expansions_write + - *restore_git_history_and_tags - *add_git_tag - *kill_processes - *cleanup_environment @@ -742,8 +766,9 @@ functions: "do benchmark setup": - command: manifest.load - - *git_get_project + - *git_get_shallow_project - *f_expansions_write + - *restore_git_history_and_tags - *add_git_tag - *kill_processes - *cleanup_environment @@ -779,8 +804,9 @@ functions: "generate powercycle tasks": - command: manifest.load - - *git_get_project + - *git_get_shallow_project - *f_expansions_write + - *restore_git_history_and_tags - *add_git_tag - *kill_processes - *cleanup_environment @@ -822,8 +848,9 @@ functions: "run powercycle sentinel": - command: manifest.load - - *git_get_project + - *git_get_shallow_project - *f_expansions_write + - *restore_git_history_and_tags - *add_git_tag - *kill_processes - *cleanup_environment @@ -2637,8 +2664,9 @@ functions: "do scons setup": - command: manifest.load + - *git_get_shallow_project - *f_expansions_write - - *git_get_project + - *restore_git_history_and_tags - *add_git_tag - *set_task_expansion_macros - *f_expansions_write diff --git a/etc/evergreen_yml_components/tasks/misc_tasks.yml b/etc/evergreen_yml_components/tasks/misc_tasks.yml index c00e9824fa6..f5363285c0d 100644 --- a/etc/evergreen_yml_components/tasks/misc_tasks.yml +++ b/etc/evergreen_yml_components/tasks/misc_tasks.yml @@ -1591,8 +1591,9 @@ tasks: tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"] commands: - command: manifest.load - - func: "git get project" + - func: "git get shallow project" - func: "f_expansions_write" + - func: "restore git history and tags" - func: "add git tag" - func: "kill processes" - func: "cleanup environment" @@ -1606,8 +1607,9 @@ tasks: priority: 10 commands: - command: manifest.load - - func: "git get project" + - func: "git get shallow project" - func: "f_expansions_write" + - func: "restore git history and tags" - func: "add git tag" - func: "kill processes" - func: "cleanup environment" @@ -1624,8 +1626,9 @@ tasks: tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"] commands: - command: manifest.load - - func: "git get project" + - func: "git get shallow project" - func: "f_expansions_write" + - func: "restore git history and tags" - func: "add git tag" - func: "kill processes" - func: "cleanup environment" @@ -1642,8 +1645,9 @@ tasks: ] commands: - command: manifest.load - - func: "git get project" + - func: "git get shallow project" - func: "f_expansions_write" + - func: "restore git history and tags" - func: "add git tag" - func: "kill processes" - func: "cleanup environment" diff --git a/evergreen/restore_git_history_and_tags.sh b/evergreen/restore_git_history_and_tags.sh new file mode 100644 index 00000000000..06a3fead436 --- /dev/null +++ b/evergreen/restore_git_history_and_tags.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# +# Usage: +# restore_git_history_and_tags.sh +# +# Required environment variables: +# * ${dir} - target directory + +cd "$dir" + +set -o errexit +set -o verbose + +# For older commits, evergreen will already unshallow the repository. +if [ -f .git/shallow ]; then + # Older versions of git require this to be set, newer versions don't mind + git config extensions.partialClone origin + + # Git versions prior to 2.20.0 do not support --filter=tree:0, so we fall + # back to doing a blobless fetch instead. + required_version="2.20.0" + git_version=$(git --version | awk '{print $3}') + if [ "$(printf '%s\n' "$required_version" "$git_version" | sort -V | head -n1)" = "$required_version" ]; then + git fetch origin --filter=tree:0 --unshallow --tags + else + git fetch origin --filter=blob:none --unshallow --tags + fi +fi