From 73bffeff62d0d624480590f77af5524df2e9db59 Mon Sep 17 00:00:00 2001
From: Eli Uriegas <eliuriegas@fb.com>
Date: Wed, 22 Apr 2020 13:27:34 -0700
Subject: [PATCH] scripts: Distinguish between platforms in conda promote
 (#37089)

Summary:
Files that were named the same within the anaconda repository, i.e.
pytorch_1.5.0-cpu.bz2, were found to be clobbering each other,
especially amongst different platforms.

This lead to similarly named packages for different platforms to not get
promoted.

This also adds "--skip" to our anaconda upload so that we don't end up
overwriting our releases just in case this script gets run twice.

Also, conda search ends up erroring out if it doesn't find anything for
the current platform being searched for so we should just continue
forward if we don't find anything since we want to be able to use this
script for all of the packages we support which also do not release
packages for all of the same platforms. (torchtext for example only has
"noarch")

This should also probably be back-ported to the `release/1.5` branch since this changeset was used to release `v1.5.0`

Signed-off-by: Eli Uriegas <eliuriegas@fb.com>
Pull Request resolved: https://github.com/pytorch/pytorch/pull/37089

Differential Revision: D21184768

Pulled By: seemethere

fbshipit-source-id: dbe12d74df593b57405b178ddb2375691e128a49
---
 scripts/release/promote/conda_to_conda.sh | 26 +++++++++++++++++------
 1 file changed, 19 insertions(+), 7 deletions(-)

diff --git a/scripts/release/promote/conda_to_conda.sh b/scripts/release/promote/conda_to_conda.sh
index 415ad3cc6fa..c890f9c187f 100755
--- a/scripts/release/promote/conda_to_conda.sh
+++ b/scripts/release/promote/conda_to_conda.sh
@@ -11,7 +11,7 @@ source "${DIR}/common_utils.sh"
 PACKAGE_NAME=${PACKAGE_NAME:-pytorch}
 PYTORCH_CONDA_FROM=${PYTORCH_CONDA_FROM:-pytorch-test}
 PYTORCH_CONDA_TO=${PYTORCH_CONDA_TO:-pytorch}
-CONDA_PLATFORMS="linux-64 osx-64 win-64"
+CONDA_PLATFORMS="linux-64 osx-64 win-64 noarch"
 
 pytorch_version="$(get_pytorch_version)"
 
@@ -19,20 +19,31 @@ tmp_dir="$(mktemp -d)"
 pushd "${tmp_dir}"
 trap 'rm -rf ${tmp_dir}' EXIT
 
+conda_search() {
+    conda search -q "${PYTORCH_CONDA_FROM}::${PACKAGE_NAME}==${pytorch_version}" -c "${PYTORCH_CONDA_FROM}" --platform "${platform}" \
+        | grep -e "^${PACKAGE_NAME}" \
+        | awk -F ' *' '{print $3}' \
+        | xargs -I % echo "https://anaconda.org/${PYTORCH_CONDA_FROM}/${PACKAGE_NAME}/${pytorch_version}/download/${platform}/${PACKAGE_NAME}-${pytorch_version}-%.tar.bz2"
+}
+
 pkgs_to_download=()
 for platform in ${CONDA_PLATFORMS}; do
     pkgs_to_download+=($(\
-        conda search "${PYTORCH_CONDA_FROM}::${PACKAGE_NAME}==${pytorch_version}" -c "${PYTORCH_CONDA_FROM}" --platform "${platform}" \
-            | grep "${PACKAGE_NAME}" \
-            | awk -F ' *' '{print $3}' \
-            | xargs -I % echo "https://anaconda.org/${PYTORCH_CONDA_FROM}/${PACKAGE_NAME}/${pytorch_version}/download/${platform}/${PACKAGE_NAME}-${pytorch_version}-%.tar.bz2"
+        conda_search 2>/dev/null || true
     ))
+    # Create directory where packages will eventually be downloaded
+    mkdir -p "${platform}"
 done
 
 my_curl() {
     local dl_url=$1
     local start=$(date +%s)
-    curl -fsSL -O "${dl_url}"
+    # downloads should be distinguished by platform which should be the second
+    # to last field in the url, this is to avoid clobbering same named files
+    # for different platforms
+    dl_dir=$(echo "${dl_url}" | rev | cut -d'/' -f 2 | rev)
+    dl_name=$(echo "${dl_url}" | rev | cut -d'/' -f 1 | rev)
+    curl -fsSL -o "${dl_dir}/${dl_name}" "${dl_url}"
     local end=$(date +%s)
     local diff=$(( end - start ))
     echo "+ ${dl_url} took ${diff}s"
@@ -50,8 +61,9 @@ if [[ $DRY_RUN = "disabled" ]]; then
     ANACONDA="anaconda"
 fi
 (
+    # We use --skip here to avoid re-uploading files we've already uploaded
     set -x
-    ${ANACONDA} upload -u ${PYTORCH_CONDA_TO} *.bz2
+    ${ANACONDA} upload --skip -u ${PYTORCH_CONDA_TO} $(find . -name '*.bz2')
 )
 
 popd