Add promotion pipeline for S3 and conda artifacts (#34993)

Summary:
Adds a new promotion pipeline for both our wheel packages hosted on S3
as well as our conda packages hosted on anaconda.

Promotion is only run on tags that that match the following regex:

    /v[0-9]+(\.[0-9]+)*/

Example:

    v1.5.0

The promotion pipeline is also only run after a manual approval from
someone within the CircleCI security context "org-member"

> NOTE: This promotion pipeline does not cover promotion of packages that
>      are published to PyPI, this is an intentional choice as those
>      packages cannot be reverted after they have been published.

TODO: Write a proper testing pipeline for this

Signed-off-by: Eli Uriegas <eliuriegas@fb.com>
Pull Request resolved: https://github.com/pytorch/pytorch/pull/34993

Differential Revision: D20539497

Pulled By: seemethere

fbshipit-source-id: 104772d3c3898d77a24ef9bf25f7dbd2496613df

.circleci/config.yml

@@ -326,6 +326,20 @@ binary_mac_params: &binary_mac_params
   environment:
     BUILD_ENVIRONMENT: << parameters.build_environment >>
 
+
+promote_common: &promote_common
+  docker:
+    - image: pytorch/release
+  parameters:
+    package_name:
+      description: "package name to promote"
+      type: string
+      default: ""
+  environment:
+    PACKAGE_NAME: << parameters.package_name >>
+    ANACONDA_API_TOKEN: ${CONDA_PYTORCHBOT_TOKEN}
+    AWS_ACCESS_KEY_ID: ${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}
+    AWS_SECRET_ACCESS_KEY: ${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}
 ##############################################################################
 # Job specs
 ##############################################################################
@@ -1541,6 +1555,24 @@ jobs:
             instruments -s -devices
             fastlane scan
 
+  promote_s3:
+    <<: *promote_common
+    steps:
+      - checkout
+      - run:
+          name: Running promote script
+          command: |
+            scripts/release/promote/wheel_to_s3.sh
+
+  promote_conda:
+    <<: *promote_common
+    steps:
+      - checkout
+      - run:
+          name: Running promote script
+          command: |
+            scripts/release/promote/conda_to_conda.sh
+
   # update_s3_htmls job
   # These jobs create html files for every cpu/cu## folder in s3. The html
   # files just store the names of all the files in that folder (which are
@@ -6978,3 +7010,49 @@ workflows:
       - docker_hub_index_job
 
 
+  # Promotion workflow
+  promote:
+    jobs:
+      # Requires manual approval by someone in org-member
+      # CircleCI security context
+      - promote_approval:
+          context: org-member
+          type: approval
+          filters:
+            tags:
+              only: /v[0-9]+(\.[0-9]+)*/
+            branches:
+                ignore: /.*/
+      - promote_s3:
+          context: org-member
+          name: promote_s3_libtorch
+          package_name: libtorch
+          requires:
+            - promote_approval
+          filters:
+            tags:
+              only: /v[0-9]+(\.[0-9]+)*/
+            branches:
+                ignore: /.*/
+      - promote_s3:
+          context: org-member
+          name: promote_s3_torch
+          package_name: torch
+          requires:
+            - promote_approval
+          filters:
+            tags:
+              only: /v[0-9]+(\.[0-9]+)*/
+            branches:
+                ignore: /.*/
+      - promote_conda:
+          context: org-member
+          name: promote_conda_pytorch
+          package_name: pytorch
+          requires:
+            - promote_approval
+          filters:
+            tags:
+              only: /v[0-9]+(\.[0-9]+)*/
+            branches:
+                ignore: /.*/

.circleci/generate_config_yml.py

@@ -80,12 +80,14 @@ YAML_SOURCES = [
     File("pytorch-build-params.yml"),
     File("caffe2-build-params.yml"),
     File("binary-build-params.yml"),
+    File("promote-build-params.yml"),
     Header("Job specs"),
     File("pytorch-job-specs.yml"),
     File("caffe2-job-specs.yml"),
     File("binary-job-specs.yml"),
     File("job-specs-setup.yml"),
     File("job-specs-custom.yml"),
+    File("job-specs-promote.yml"),
     File("binary_update_htmls.yml"),
     File("binary-build-tests.yml"),
     File("docker_jobs.yml"),
@@ -113,6 +115,7 @@ YAML_SOURCES = [
     File("workflows-s3-html.yml"),
     File("workflows-docker-builder.yml"),
     File("workflows-ecr-gc.yml"),
+    File("workflows-promote.yml")
 ]
 
 

.circleci/verbatim-sources/job-specs-promote.yml

@@ -0,0 +1,18 @@
+
+  promote_s3:
+    <<: *promote_common
+    steps:
+      - checkout
+      - run:
+          name: Running promote script
+          command: |
+            scripts/release/promote/wheel_to_s3.sh
+
+  promote_conda:
+    <<: *promote_common
+    steps:
+      - checkout
+      - run:
+          name: Running promote script
+          command: |
+            scripts/release/promote/conda_to_conda.sh

.circleci/verbatim-sources/promote-build-params.yml

@@ -0,0 +1,14 @@
+
+promote_common: &promote_common
+  docker:
+    - image: pytorch/release
+  parameters:
+    package_name:
+      description: "package name to promote"
+      type: string
+      default: ""
+  environment:
+    PACKAGE_NAME: << parameters.package_name >>
+    ANACONDA_API_TOKEN: ${CONDA_PYTORCHBOT_TOKEN}
+    AWS_ACCESS_KEY_ID: ${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}
+    AWS_SECRET_ACCESS_KEY: ${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}

.circleci/verbatim-sources/workflows-promote.yml

@@ -0,0 +1,46 @@
+  # Promotion workflow
+  promote:
+    jobs:
+      # Requires manual approval by someone in org-member
+      # CircleCI security context
+      - promote_approval:
+          context: org-member
+          type: approval
+          filters:
+            tags:
+              only: /v[0-9]+(\.[0-9]+)*/
+            branches:
+                ignore: /.*/
+      - promote_s3:
+          context: org-member
+          name: promote_s3_libtorch
+          package_name: libtorch
+          requires:
+            - promote_approval
+          filters:
+            tags:
+              only: /v[0-9]+(\.[0-9]+)*/
+            branches:
+                ignore: /.*/
+      - promote_s3:
+          context: org-member
+          name: promote_s3_torch
+          package_name: torch
+          requires:
+            - promote_approval
+          filters:
+            tags:
+              only: /v[0-9]+(\.[0-9]+)*/
+            branches:
+                ignore: /.*/
+      - promote_conda:
+          context: org-member
+          name: promote_conda_pytorch
+          package_name: pytorch
+          requires:
+            - promote_approval
+          filters:
+            tags:
+              only: /v[0-9]+(\.[0-9]+)*/
+            branches:
+                ignore: /.*/

scripts/release/promote/conda_to_conda.sh

@@ -29,19 +29,19 @@ for platform in ${CONDA_PLATFORMS}; do
     ))
 done
 
-my_wget() {
+my_curl() {
     local dl_url=$1
     local start=$(date +%s)
-    wget -c -q "${dl_url}"
+    curl -fsSL -O "${dl_url}"
     local end=$(date +%s)
     local diff=$(( end - start ))
     echo "+ ${dl_url} took ${diff}s"
 }
-export -f my_wget
+export -f my_curl
 
 # Download all packages in parallel
 printf '%s\n' "${pkgs_to_download[@]}" \
-    | xargs -P 10 -I % bash -c '(declare -t my_wget); my_wget %'
+    | xargs -P 10 -I % bash -c '(declare -t my_curl); my_curl %'
 
 # dry run by default
 DRY_RUN=${DRY_RUN:-enabled}