From 3d77dae8e5365f4305584898c9a92704341cb3b3 Mon Sep 17 00:00:00 2001 From: Adam Rayner Date: Thu, 22 Jun 2023 15:54:34 +0000 Subject: [PATCH] SERVER-77303: Add retries for all SELinux test connectivity / ssh steps --- etc/evergreen_yml_components/definitions.yml | 4 +- evergreen/selinux_run_test.sh | 153 +++++++++++++------ 2 files changed, 110 insertions(+), 47 deletions(-) diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml index 8e78456723a..f9eec7bd0ba 100644 --- a/etc/evergreen_yml_components/definitions.yml +++ b/etc/evergreen_yml_components/definitions.yml @@ -1750,8 +1750,8 @@ functions: args: - "./src/evergreen/selinux_run_test.sh" env: - test_list: ${test_list} - user: ec2-user + TEST_LIST: ${test_list} + SELINUX_USER: ec2-user ### Process & archive remote EC2 artifacts ### diff --git a/evergreen/selinux_run_test.sh b/evergreen/selinux_run_test.sh index 15fe8be87b6..c7ec9502a46 100755 --- a/evergreen/selinux_run_test.sh +++ b/evergreen/selinux_run_test.sh @@ -3,76 +3,139 @@ # Notes on how to run this manually: # - repo must be unpacked into source tree # -# export ssh_key=$HOME/.ssh/id_rsa -# export hostname=ec2-3-91-230-150.compute-1.amazonaws.com -# export user=ec2-user -# export bypass_prelude=yes +# export SSH_KEY=$HOME/.ssh/id_rsa +# export SELINUX_HOSTNAME=ec2-3-91-230-150.compute-1.amazonaws.com +# export SELINUX_USER=ec2-user +# export BYPASS_PRELUDE=yes +# export SRC="$(basename $(pwd) | tee /dev/stderr)" +# export TEST_LIST='jstests/selinux/*.js' # export workdir="$(dirname $(pwd) | tee /dev/stderr)" -# export src="$(basename $(pwd) | tee /dev/stderr)" -# export test_list='jstests/selinux/*.js' -# export pkg_variant=mongodb-enterprise # evergreen/selinux_run_test.sh set -o errexit -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" -if [ "$bypass_prelude" != "yes" ]; then - . "$DIR/prelude.sh" +readonly k_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" + +if [ "$BYPASS_PRELUDE" != "yes" ]; then + . "$k_dir/prelude.sh" activate_venv - src="src" + readonly k_src="src" +else + readonly k_src="$SRC" fi -if [ "$hostname" == "" ]; then - hostname="$(tr -d '"[]{}' < "$workdir"/$src/hosts.yml | cut -d , -f 1 | awk -F : '{print $2}')" +# If no selinux hostname is defined by external env, then we are running through evergreen, which has dumped spawn host +# properties about this host into hosts.yml via host.list +# (https://github.com/evergreen-ci/evergreen/blob/main/docs/Project-Configuration/Project-Commands.md#hostlist), +# from which we can derive the hostname of the remote host +# Also note that $workdir here is a built-in expansion from evergreen: see more info at +# https://github.com/evergreen-ci/evergreen/blob/main/docs/Project-Configuration/Project-Configuration-Files.md#default-expansions +if [ "$SELINUX_HOSTNAME" == "" ]; then + readonly k_selinux_hostname="$(tr -d '"[]{}' < "$workdir"/$k_src/hosts.yml | cut -d , -f 1 | awk -F : '{print $2}')" + cat "$workdir"/$k_src/hosts.yml +else + readonly k_selinux_hostname="$SELINUX_HOSTNAME" fi -if [ "$user" == "" ]; then - user=$USER -fi +# SELINUX_USER injected from evergreen config, do not change +readonly k_host="${SELINUX_USER}@${k_selinux_hostname}" -host="${user}@${hostname}" -python="${python:-python3}" +# Obtain the ssh key and properties from expansions.yml, output from evergreen via the expansions.write command +# (https://github.com/evergreen-ci/evergreen/blob/main/docs/Project-Configuration/Project-Commands.md#expansionswrite) +if [ "$SSH_KEY" == "" ]; then + readonly k_ssh_key="$workdir/selinux.pem" + + "$workdir"/$k_src/buildscripts/yaml_key_value.py --yamlFile="$workdir"/expansions.yml \ + --yamlKey=__project_aws_ssh_key_value > "$k_ssh_key" + + chmod 600 "$k_ssh_key" + + result="$(openssl rsa -in "$k_ssh_key" -check -noout | tee /dev/stderr)" -if [ "$ssh_key" == "" ]; then - ssh_key="$workdir/selinux.pem" - "$workdir"/$src/buildscripts/yaml_key_value.py --yamlFile="$workdir"/expansions.yml \ - --yamlKey=__project_aws_ssh_key_value > "$ssh_key" - chmod 600 "$ssh_key" - result="$(openssl rsa -in "$ssh_key" -check -noout | tee /dev/stderr)" if [ "$result" != "RSA key ok" ]; then exit 1 fi +else + readonly k_ssh_key="$SSH_KEY" fi -attempts=0 -connection_attempts=50 +readonly k_ssh_options="-i $k_ssh_key -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" -# Check for remote connectivity -set +o errexit -ssh_options="-i $ssh_key -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" -while ! ssh -q $ssh_options -o ConnectTimeout=10 "$host" echo "I am working"; do - if [ "$attempts" -ge "$connection_attempts" ]; then - printf "SSH connection attempt failed after %d attempts.\n" "$attempts" - exit 1 - fi - ((attempts++)) - sleep 10 -done +function copy_sources_to_target() { + + rsync -ar -e "ssh $k_ssh_options" \ + --exclude 'tmp' --exclude 'build' --exclude '.*' \ + "$workdir"/$k_src/* "$k_host": + + return $? +} + +function configure_target_machine() { + ssh $k_ssh_options "$k_host" evergreen/selinux_test_setup.sh + return $? +} + +function execute_tests_on_target() { + ssh $k_ssh_options "$k_host" evergreen/selinux_test_executor.sh "$1" + return $? +} + +function check_remote_connectivity() { + ssh -q $k_ssh_options -o ConnectTimeout=10 "$k_host" echo "I am working" + return $? +} + +function retry_command() { + + local connection_attempts=$1 + local cmd="$2" + shift 2 #eat the first 2 parameters to pass on any remaining to the calling function + + local attempts=0 + set +o errexit + + while true; do + "$cmd" "$@" + + local result=$? + + if [[ $result -eq 0 ]]; then + set -o errexit + return $result + fi + + if [[ $attempts -ge $connection_attempts ]]; then + printf "%s failed after %d attempts with final error code %s.\n" "$cmd" "$attempts" "$result" + exit 1 + fi + + sleep 10 + ((attempts++)) + + done +} + +echo "===> Checking for remote connectivity..." +retry_command 20 check_remote_connectivity -set -o errexit echo "===> Copying sources to target..." -rsync -ar -e "ssh $ssh_options" \ - --exclude 'tmp' --exclude 'build' --exclude '.*' \ - "$workdir"/$src/* "$host": +retry_command 5 copy_sources_to_target echo "===> Configuring target machine..." -ssh $ssh_options "$host" evergreen/selinux_test_setup.sh +retry_command 5 configure_target_machine echo "===> Executing tests..." -list="$( +readonly list="$( cd src - for x in $test_list; do echo "$x"; done + + # $TEST_LIST defined in evegreen "run selinux tests" function, do not change + for x in $TEST_LIST; do echo "$x"; done )" + for test in $list; do - ssh $ssh_options "$host" evergreen/selinux_test_executor.sh "$test" + execute_tests_on_target "$test" + res="$?" + if [[ $res -ne 0 ]]; then + exit "$res" + fi done