# exit immediately if virtualenv is not found set -o errexit evergreen_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)/.." . "$evergreen_dir/prelude_workdir.sh" . "$evergreen_dir/prelude_python.sh" python_loc=$(which ${python}) echo "python_loc set to $python_loc" venv_dir="${workdir}/venv" if [ -d "$venv_dir" ]; then exit 0 fi # We create a venv for poetry # We cannot install poetry into the same virtual enviorment as the rest of our tools # If there is a conflict between poetry and our other deps windows fails to upgrade the package # See issue SERVER-80781 POETRY_VENV="${workdir}/poetry_venv" if [ "Windows_NT" = "$OS" ]; then POETRY_VENV_PYTHON="$POETRY_VENV/Scripts/python.exe" else POETRY_VENV_PYTHON="$POETRY_VENV/bin/python3" fi "$python_loc" -m venv "$POETRY_VENV" # Loop 5 times to retry the poetry install # We have seen weird network errors that can sometimes mess up the pip install # By retrying we would like to only see errors that happen consistently if uname -a | grep -q 's390x\|ppc64le'; then # s390x and ppc64le both require these old versions for some reason # They are pinned deps as well EXTRA_IBM_ARGS="cryptography==2.3 pyOpenSSL==19.0.0" fi poetry_dir="${workdir}/poetry_dir" mkdir -p $poetry_dir export POETRY_CONFIG_DIR="$poetry_dir/config" export POETRY_DATA_DIR="$poetry_dir/data" export POETRY_CACHE_DIR="$poetry_dir/cache" export PIP_CACHE_DIR="$poetry_dir/pip_cache" for i in {1..5}; do $POETRY_VENV_PYTHON -m pip install "poetry==1.8.3" ${EXTRA_IBM_ARGS} && RET=0 && break || RET=$? && sleep 1 echo "Python failed to install poetry, retrying..." done if [ $RET -ne 0 ]; then echo "Pip install error for poetry" exit $RET fi "$python_loc" -m venv "$venv_dir" # Adding README file for using this venv locally cat << EOF >> venv_readme.txt This is an archive of the Python venv generated by this Evergreen build. You can use it locally to avoid needing to manually set up the Python environment. Before activating it you should adjust it for your local environment. Run the following commands to do that: echo "Updating virtual env directory in activate script" pushd venv; venv_dir=\$(pwd); popd EOF if [ "Windows_NT" = "$OS" ]; then cat << EOF >> venv_readme.txt sed -i -e "s:VIRTUAL_ENV=\".*\":VIRTUAL_ENV=\"\$venv_dir\":" "\$venv_dir/Scripts/activate" EOF else cat << EOF >> venv_readme.txt sed -i -e "s:VIRTUAL_ENV=\".*\":VIRTUAL_ENV=\"\$venv_dir\":" "\$venv_dir/bin/activate" echo "Adding back python symlinks" pushd venv/bin rm python python3 ln -s "$python_loc" python3 ln -s python3 python python3_dot_locs=\$(ls python3.*) for p in \$python3_dot_locs; do rm "\$p" ln -s python3 "\$p" done popd EOF fi # End of README file # venv creates its Scripts/activate file with CLRF endings, which # cygwin bash does not like. dos2unix it # (See https://bugs.python.org/issue32451) if [ "Windows_NT" = "$OS" ]; then dos2unix "${workdir}/venv/Scripts/activate" fi export VIRTUAL_ENV_DISABLE_PROMPT=yes # the whole prelude cannot be imported because it requires pyyaml to be # installed, which happens just below. . "$evergreen_dir/prelude_venv.sh" activate_venv echo "Upgrading pip to 21.0.1" # Loop 5 times to retry the pip install # We have seen weird network errors that can sometimes mess up the pip install # By retrying we would like to only see errors that happen consistently for i in {1..5}; do python -m pip --disable-pip-version-check install "pip==21.0.1" "wheel==0.37.0" && RET=0 && break || RET=$? && sleep 1 echo "Python failed to install pip and wheel, retrying..." done if [ $RET -ne 0 ]; then echo "Pip install error for wheel and pip version" exit $RET fi cd src # Loop 5 times to retry full venv install # We have seen weird network errors that can sometimes mess up the pip install # By retrying we would like to only see errors that happen consistently count=0 for i in {1..5}; do yes | $POETRY_VENV_PYTHON -m poetry cache clear . --all rm -rf $poetry_dir/* $POETRY_VENV_PYTHON -m poetry install --no-root --sync && RET=0 && break || RET=$? && sleep 1 echo "Python failed install required deps with poetry, retrying..." sleep $((count * count * 20)) count=$((count + 1)) done if [ $RET -ne 0 ]; then echo "Poetry install error for full venv" exit $RET fi # poetry will install cryptography in an isolated build environment # to conform to pep517, however this doesn't work for the old cryptography # version on these platforms, and ends up not building required shared libraries. # Here we go behing poetry's back and install with pip if uname -a | grep -q 's390x\|ppc64le'; then for i in {1..5}; do python -m pip uninstall -y cryptography==2.3 || true python -m pip install cryptography==2.3 && RET=0 && break || RET=$? && sleep 1 done if [ $RET -ne 0 ]; then echo "cryptography install error for full venv" exit $RET fi fi cd .. python -m pip freeze > pip-requirements.txt