ci: Enable tests for iOS on GHA

These were left out of the intial migration for some reason so this just
transfers over those tests

Signed-off-by: Eli Uriegas <eliuriegasfb.com>

Pull Request resolved: https://github.com/pytorch/pytorch/pull/71644

Signed-off-by: Eli Uriegas <eliuriegas@fb.com>

.github/scripts/generate_ci_workflows.py

@@ -169,6 +169,8 @@ class CIWorkflow:
     timeout_after: int = 240
     xcode_version: str = ''
     only_on_pr: bool = False
+    ios_arch: str = ''
+    ios_platform: str = ''
 
     # The following variables will be set as environment variables,
     # so it's easier for both shell and Python scripts to consume it if false is represented as the empty string.
@@ -713,6 +715,8 @@ IOS_WORKFLOWS = [
     CIWorkflow(
         arch="macos",
         build_environment="ios-12-5-1-arm64",
+        ios_arch="arm64",
+        ios_platform="OS",
         test_runner_type=MACOS_TEST_RUNNER_10_15,
         exclude_test=True,
         ciflow_config=CIFlowConfig(
@@ -722,6 +726,8 @@ IOS_WORKFLOWS = [
     CIWorkflow(
         arch="macos",
         build_environment="ios-12-5-1-arm64-coreml",
+        ios_arch="arm64",
+        ios_platform="OS",
         test_runner_type=MACOS_TEST_RUNNER_10_15,
         exclude_test=True,
         ciflow_config=CIFlowConfig(
@@ -731,6 +737,8 @@ IOS_WORKFLOWS = [
     CIWorkflow(
         arch="macos",
         build_environment="ios-12-5-1-arm64-full-jit",
+        ios_arch="arm64",
+        ios_platform="OS",
         test_runner_type=MACOS_TEST_RUNNER_10_15,
         exclude_test=True,
         ciflow_config=CIFlowConfig(
@@ -740,6 +748,8 @@ IOS_WORKFLOWS = [
     CIWorkflow(
         arch="macos",
         build_environment="ios-12-5-1-arm64-custom-ops",
+        ios_arch="arm64",
+        ios_platform="OS",
         test_runner_type=MACOS_TEST_RUNNER_10_15,
         exclude_test=True,
         ciflow_config=CIFlowConfig(
@@ -749,6 +759,8 @@ IOS_WORKFLOWS = [
     CIWorkflow(
         arch="macos",
         build_environment="ios-12-5-1-arm64-metal",
+        ios_arch="arm64",
+        ios_platform="OS",
         test_runner_type=MACOS_TEST_RUNNER_10_15,
         exclude_test=True,
         ciflow_config=CIFlowConfig(
@@ -758,6 +770,8 @@ IOS_WORKFLOWS = [
     CIWorkflow(
         arch="macos",
         build_environment="ios-12-5-1-x86-64",
+        ios_arch="x86_64",
+        ios_platform="SIMULATOR",
         test_runner_type=MACOS_TEST_RUNNER_10_15,
         exclude_test=True,
         ciflow_config=CIFlowConfig(
@@ -767,6 +781,8 @@ IOS_WORKFLOWS = [
     CIWorkflow(
         arch="macos",
         build_environment="ios-12-5-1-x86-64-coreml",
+        ios_arch="x86_64",
+        ios_platform="SIMULATOR",
         test_runner_type=MACOS_TEST_RUNNER_10_15,
         exclude_test=True,
         ciflow_config=CIFlowConfig(
@@ -776,6 +792,8 @@ IOS_WORKFLOWS = [
     CIWorkflow(
         arch="macos",
         build_environment="ios-12-5-1-x86-64-full-jit",
+        ios_arch="x86_64",
+        ios_platform="SIMULATOR",
         test_runner_type=MACOS_TEST_RUNNER_10_15,
         exclude_test=True,
         ciflow_config=CIFlowConfig(

.github/templates/ios_ci_workflow.yml.j2

@@ -30,38 +30,79 @@ on:
 {%- endfor %}
   workflow_dispatch:
 
-# For setup-miniconda, see https://github.com/conda-incubator/setup-miniconda/issues/179
-defaults:
-  run:
-    shell: bash -x -e -l {0}
 env:
   BUILD_ENVIRONMENT: !{{ build_environment }}
   IN_CI: 1
   IS_GHA: 1
+  IOS_PLATFORM: !{{ ios_platform }}
+  IOS_ARCH: !{{ ios_arch }}
 !{{ common.set_xcode_version(xcode_version) }}
 
 jobs:
 {% block build +%}
   build:
+    # NOTE: These builds will not run successfully without running on `pytorch/pytorch` due to the limitations
+    #       of accessing secrets from forked pull requests and IOS' dependency on secrets for their build/test
+    if: ${{ github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository }}
     runs-on: macos-10.15
     timeout-minutes: !{{ common.timeout_minutes }}
     env:
       JOB_BASE_NAME: !{{ build_environment }}-build
       IOS_CERT_KEY_2022: ${{ secrets.IOS_CERT_KEY_2022 }}
+      IOS_CERT_SECRET: ${{ secrets.IOS_CERT_SECRET }}
+      IOS_DEV_TEAM_ID: ${{ secrets.IOS_DEV_TEAM_ID }}
       IOS_SIGN_KEY_2022: ${{ secrets.IOS_SIGN_KEY_2022 }}
       PR_LABELS: ${{ toJson(github.event.pull_request.labels.*.name) }}
     steps:
       - name: print labels
         run: echo "${PR_LABELS}"
       !{{ common.checkout() }}
-      !{{ common.setup_miniconda("3.8") }}
+      - name: Populate CI build options
-      - name: Install ios / conda Dependencies
+        run: |
+          # Most builds use the lite interpreter, if certain builds shouldn't
+          # build the lite interpreter this env variable should get over-written
+          # in the following case statement
+          echo "BUILD_LITE_INTERPRETER=1" >> "${GITHUB_ENV}"
+
+          case ${BUILD_ENVIRONMENT} in
+            *metal*)
+              echo "USE_PYTORCH_METAL=1" >> "${GITHUB_ENV}"
+              ;;
+            *full_jit*)
+              echo "BUILD_LITE_INTERPRETER=0" >> "${GITHUB_ENV}"
+              ;;
+            *custom*)
+              echo "SELECTED_OP_LIST=${GITHUB_WORKSPACE}/ios/TestApp/custom_build/mobilenetv2.yaml" >> "${GITHUB_ENV}"
+              ;;
+            *coreml*)
+              echo "USE_COREML_DELEGATE=1" >> "${GITHUB_ENV}"
+              ;;
+          esac
+      - name: Install brew dependencies
         run: |
           # Install dependencies
           brew install libtool
-          conda install numpy ninja pyyaml mkl mkl-include setuptools cmake cffi requests typing_extensions --yes
+      - name: Install conda and dependencies
+        run: |
+          # Install conda, setup-miniconda messes with the path that messes with the ruby stuff we do later on
+          curl --retry 3 -o "${RUNNER_TEMP}/conda.sh" https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+          chmod +x "${RUNNER_TEMP}/conda.sh"
+          /bin/bash "${RUNNER_TEMP}/conda.sh" -b -p "${RUNNER_TEMP}/anaconda"
+          echo "${RUNNER_TEMP}/anaconda/bin" >> "${GITHUB_PATH}"
+          # shellcheck disable=SC1091
+          source "${RUNNER_TEMP}/anaconda/bin/activate"
+          conda install -y \
+            cffi \
+            cmake \
+            mkl \
+            mkl-include \
+            ninja \
+            numpy \
+            pyyaml \
+            requests \
+            setuptools \
+            typing_extensions
       - name: Run Fastlane
-        shell: bash -e {0}
         run: |
           set -x
           cd ios/TestApp
@@ -83,10 +124,60 @@ jobs:
           rm cert.txt
       - name: Build
         run: |
+          # shellcheck disable=SC1091
+          source "${RUNNER_TEMP}/anaconda/bin/activate"
           export TCLLIBPATH="/usr/local/lib"
           python -VV
           export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname "$(which conda)")/../"}
           scripts/build_ios.sh
+      - name: Run Build Test
+        run: |
+          PROFILE=PyTorch_CI_2022
+          # run the ruby build script
+          if ! [ -x "$(command -v xcodebuild)" ]; then
+            echo 'Error: xcodebuild is not installed.'
+            exit 1
+          fi
+          if [ "${IOS_PLATFORM}" != "SIMULATOR" ]; then
+            ruby scripts/xcode_build.rb -i build_ios/install -x ios/TestApp/TestApp.xcodeproj -p "${IOS_PLATFORM}" -c "${PROFILE}" -t "${IOS_DEV_TEAM_ID}"
+          else
+            ruby scripts/xcode_build.rb -i build_ios/install -x ios/TestApp/TestApp.xcodeproj -p "${IOS_PLATFORM}"
+          fi
+{%- if ios_platform == "SIMULATOR" %}
+      - name: Run Simulator Tests
+        run: |
+          # shellcheck disable=SC1091
+          source "${RUNNER_TEMP}/anaconda/bin/activate"
+          pip3 install --pre torch torchvision torchaudio -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
+          # generate models for differnet backends
+          cd "${GITHUB_WORKSPACE}/ios/TestApp/benchmark"
+          mkdir -p ../models
+          if [ "${USE_COREML_DELEGATE}" == 1 ]; then
+            pip install coremltools==5.0b5
+            pip install six==1.16.0
+            python coreml_backend.py
+          else
+            python trace_model.py
+          fi
+          if [ "${BUILD_LITE_INTERPRETER}" == 1 ]; then
+            echo "Setting up the TestApp for LiteInterpreter"
+            ruby setup.rb --lite 1
+          else
+            echo "Setting up the TestApp for Full JIT"
+            ruby setup.rb
+          fi
+          cd "${GITHUB_WORKSPACE}/ios/TestApp"
+          instruments -s -devices
+          if [ "${BUILD_LITE_INTERPRETER}" == 1 ]; then
+            if [ "${USE_COREML_DELEGATE}" == 1 ]; then
+              fastlane scan --only_testing TestAppTests/TestAppTests/testCoreML
+            else
+              fastlane scan --only_testing TestAppTests/TestAppTests/testLiteInterpreter
+            fi
+          else
+            fastlane scan --only_testing TestAppTests/TestAppTests/testFullJIT
+          fi
+{%- endif -%}
 {% endblock +%}
 
 !{{ common.concurrency(build_environment) }}

.github/workflows/generated-ios-12-5-1-arm64-coreml.yml

@@ -15,24 +15,27 @@ on:
       - 'ciflow/trunk/*'
   workflow_dispatch:
 
-# For setup-miniconda, see https://github.com/conda-incubator/setup-miniconda/issues/179
-defaults:
-  run:
-    shell: bash -x -e -l {0}
 env:
   BUILD_ENVIRONMENT: ios-12-5-1-arm64-coreml
   IN_CI: 1
   IS_GHA: 1
+  IOS_PLATFORM: OS
+  IOS_ARCH: arm64
 
 
 jobs:
 
   build:
+    # NOTE: These builds will not run successfully without running on `pytorch/pytorch` due to the limitations
+    #       of accessing secrets from forked pull requests and IOS' dependency on secrets for their build/test
+    if: ${{ github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository }}
     runs-on: macos-10.15
     timeout-minutes: 240
     env:
       JOB_BASE_NAME: ios-12-5-1-arm64-coreml-build
       IOS_CERT_KEY_2022: ${{ secrets.IOS_CERT_KEY_2022 }}
+      IOS_CERT_SECRET: ${{ secrets.IOS_CERT_SECRET }}
+      IOS_DEV_TEAM_ID: ${{ secrets.IOS_DEV_TEAM_ID }}
       IOS_SIGN_KEY_2022: ${{ secrets.IOS_SIGN_KEY_2022 }}
       PR_LABELS: ${{ toJson(github.event.pull_request.labels.*.name) }}
     steps:
@@ -48,19 +51,52 @@ jobs:
         run: |
           # Remove any artifacts from the previous checkouts
           git clean -fxd
-      - name: Setup miniconda
+      - name: Populate CI build options
-        uses: conda-incubator/setup-miniconda@v2
+        run: |
-        with:
+          # Most builds use the lite interpreter, if certain builds shouldn't
-          auto-update-conda: true
+          # build the lite interpreter this env variable should get over-written
-          python-version: 3.8
+          # in the following case statement
-          activate-environment: build
+          echo "BUILD_LITE_INTERPRETER=1" >> "${GITHUB_ENV}"
-      - name: Install ios / conda Dependencies
+
+          case ${BUILD_ENVIRONMENT} in
+            *metal*)
+              echo "USE_PYTORCH_METAL=1" >> "${GITHUB_ENV}"
+              ;;
+            *full_jit*)
+              echo "BUILD_LITE_INTERPRETER=0" >> "${GITHUB_ENV}"
+              ;;
+            *custom*)
+              echo "SELECTED_OP_LIST=${GITHUB_WORKSPACE}/ios/TestApp/custom_build/mobilenetv2.yaml" >> "${GITHUB_ENV}"
+              ;;
+            *coreml*)
+              echo "USE_COREML_DELEGATE=1" >> "${GITHUB_ENV}"
+              ;;
+          esac
+      - name: Install brew dependencies
         run: |
           # Install dependencies
           brew install libtool
-          conda install numpy ninja pyyaml mkl mkl-include setuptools cmake cffi requests typing_extensions --yes
+      - name: Install conda and dependencies
+        run: |
+          # Install conda, setup-miniconda messes with the path that messes with the ruby stuff we do later on
+          curl --retry 3 -o "${RUNNER_TEMP}/conda.sh" https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+          chmod +x "${RUNNER_TEMP}/conda.sh"
+          /bin/bash "${RUNNER_TEMP}/conda.sh" -b -p "${RUNNER_TEMP}/anaconda"
+          echo "${RUNNER_TEMP}/anaconda/bin" >> "${GITHUB_PATH}"
+          # shellcheck disable=SC1091
+          source "${RUNNER_TEMP}/anaconda/bin/activate"
+          conda install -y \
+            cffi \
+            cmake \
+            mkl \
+            mkl-include \
+            ninja \
+            numpy \
+            pyyaml \
+            requests \
+            setuptools \
+            typing_extensions
       - name: Run Fastlane
-        shell: bash -e {0}
         run: |
           set -x
           cd ios/TestApp
@@ -82,11 +118,25 @@ jobs:
           rm cert.txt
       - name: Build
         run: |
+          # shellcheck disable=SC1091
+          source "${RUNNER_TEMP}/anaconda/bin/activate"
           export TCLLIBPATH="/usr/local/lib"
           python -VV
           export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname "$(which conda)")/../"}
           scripts/build_ios.sh
-
+      - name: Run Build Test
+        run: |
+          PROFILE=PyTorch_CI_2022
+          # run the ruby build script
+          if ! [ -x "$(command -v xcodebuild)" ]; then
+            echo 'Error: xcodebuild is not installed.'
+            exit 1
+          fi
+          if [ "${IOS_PLATFORM}" != "SIMULATOR" ]; then
+            ruby scripts/xcode_build.rb -i build_ios/install -x ios/TestApp/TestApp.xcodeproj -p "${IOS_PLATFORM}" -c "${PROFILE}" -t "${IOS_DEV_TEAM_ID}"
+          else
+            ruby scripts/xcode_build.rb -i build_ios/install -x ios/TestApp/TestApp.xcodeproj -p "${IOS_PLATFORM}"
+          fi
 
 concurrency:
   group: ios-12-5-1-arm64-coreml-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}

.github/workflows/generated-ios-12-5-1-arm64-custom-ops.yml

@@ -15,24 +15,27 @@ on:
       - 'ciflow/trunk/*'
   workflow_dispatch:
 
-# For setup-miniconda, see https://github.com/conda-incubator/setup-miniconda/issues/179
-defaults:
-  run:
-    shell: bash -x -e -l {0}
 env:
   BUILD_ENVIRONMENT: ios-12-5-1-arm64-custom-ops
   IN_CI: 1
   IS_GHA: 1
+  IOS_PLATFORM: OS
+  IOS_ARCH: arm64
 
 
 jobs:
 
   build:
+    # NOTE: These builds will not run successfully without running on `pytorch/pytorch` due to the limitations
+    #       of accessing secrets from forked pull requests and IOS' dependency on secrets for their build/test
+    if: ${{ github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository }}
     runs-on: macos-10.15
     timeout-minutes: 240
     env:
       JOB_BASE_NAME: ios-12-5-1-arm64-custom-ops-build
       IOS_CERT_KEY_2022: ${{ secrets.IOS_CERT_KEY_2022 }}
+      IOS_CERT_SECRET: ${{ secrets.IOS_CERT_SECRET }}
+      IOS_DEV_TEAM_ID: ${{ secrets.IOS_DEV_TEAM_ID }}
       IOS_SIGN_KEY_2022: ${{ secrets.IOS_SIGN_KEY_2022 }}
       PR_LABELS: ${{ toJson(github.event.pull_request.labels.*.name) }}
     steps:
@@ -48,19 +51,52 @@ jobs:
         run: |
           # Remove any artifacts from the previous checkouts
           git clean -fxd
-      - name: Setup miniconda
+      - name: Populate CI build options
-        uses: conda-incubator/setup-miniconda@v2
+        run: |
-        with:
+          # Most builds use the lite interpreter, if certain builds shouldn't
-          auto-update-conda: true
+          # build the lite interpreter this env variable should get over-written
-          python-version: 3.8
+          # in the following case statement
-          activate-environment: build
+          echo "BUILD_LITE_INTERPRETER=1" >> "${GITHUB_ENV}"
-      - name: Install ios / conda Dependencies
+
+          case ${BUILD_ENVIRONMENT} in
+            *metal*)
+              echo "USE_PYTORCH_METAL=1" >> "${GITHUB_ENV}"
+              ;;
+            *full_jit*)
+              echo "BUILD_LITE_INTERPRETER=0" >> "${GITHUB_ENV}"
+              ;;
+            *custom*)
+              echo "SELECTED_OP_LIST=${GITHUB_WORKSPACE}/ios/TestApp/custom_build/mobilenetv2.yaml" >> "${GITHUB_ENV}"
+              ;;
+            *coreml*)
+              echo "USE_COREML_DELEGATE=1" >> "${GITHUB_ENV}"
+              ;;
+          esac
+      - name: Install brew dependencies
         run: |
           # Install dependencies
           brew install libtool
-          conda install numpy ninja pyyaml mkl mkl-include setuptools cmake cffi requests typing_extensions --yes
+      - name: Install conda and dependencies
+        run: |
+          # Install conda, setup-miniconda messes with the path that messes with the ruby stuff we do later on
+          curl --retry 3 -o "${RUNNER_TEMP}/conda.sh" https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+          chmod +x "${RUNNER_TEMP}/conda.sh"
+          /bin/bash "${RUNNER_TEMP}/conda.sh" -b -p "${RUNNER_TEMP}/anaconda"
+          echo "${RUNNER_TEMP}/anaconda/bin" >> "${GITHUB_PATH}"
+          # shellcheck disable=SC1091
+          source "${RUNNER_TEMP}/anaconda/bin/activate"
+          conda install -y \
+            cffi \
+            cmake \
+            mkl \
+            mkl-include \
+            ninja \
+            numpy \
+            pyyaml \
+            requests \
+            setuptools \
+            typing_extensions
       - name: Run Fastlane
-        shell: bash -e {0}
         run: |
           set -x
           cd ios/TestApp
@@ -82,11 +118,25 @@ jobs:
           rm cert.txt
       - name: Build
         run: |
+          # shellcheck disable=SC1091
+          source "${RUNNER_TEMP}/anaconda/bin/activate"
           export TCLLIBPATH="/usr/local/lib"
           python -VV
           export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname "$(which conda)")/../"}
           scripts/build_ios.sh
-
+      - name: Run Build Test
+        run: |
+          PROFILE=PyTorch_CI_2022
+          # run the ruby build script
+          if ! [ -x "$(command -v xcodebuild)" ]; then
+            echo 'Error: xcodebuild is not installed.'
+            exit 1
+          fi
+          if [ "${IOS_PLATFORM}" != "SIMULATOR" ]; then
+            ruby scripts/xcode_build.rb -i build_ios/install -x ios/TestApp/TestApp.xcodeproj -p "${IOS_PLATFORM}" -c "${PROFILE}" -t "${IOS_DEV_TEAM_ID}"
+          else
+            ruby scripts/xcode_build.rb -i build_ios/install -x ios/TestApp/TestApp.xcodeproj -p "${IOS_PLATFORM}"
+          fi
 
 concurrency:
   group: ios-12-5-1-arm64-custom-ops-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}

.github/workflows/generated-ios-12-5-1-arm64-full-jit.yml

@@ -15,24 +15,27 @@ on:
       - 'ciflow/trunk/*'
   workflow_dispatch:
 
-# For setup-miniconda, see https://github.com/conda-incubator/setup-miniconda/issues/179
-defaults:
-  run:
-    shell: bash -x -e -l {0}
 env:
   BUILD_ENVIRONMENT: ios-12-5-1-arm64-full-jit
   IN_CI: 1
   IS_GHA: 1
+  IOS_PLATFORM: OS
+  IOS_ARCH: arm64
 
 
 jobs:
 
   build:
+    # NOTE: These builds will not run successfully without running on `pytorch/pytorch` due to the limitations
+    #       of accessing secrets from forked pull requests and IOS' dependency on secrets for their build/test
+    if: ${{ github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository }}
     runs-on: macos-10.15
     timeout-minutes: 240
     env:
       JOB_BASE_NAME: ios-12-5-1-arm64-full-jit-build
       IOS_CERT_KEY_2022: ${{ secrets.IOS_CERT_KEY_2022 }}
+      IOS_CERT_SECRET: ${{ secrets.IOS_CERT_SECRET }}
+      IOS_DEV_TEAM_ID: ${{ secrets.IOS_DEV_TEAM_ID }}
       IOS_SIGN_KEY_2022: ${{ secrets.IOS_SIGN_KEY_2022 }}
       PR_LABELS: ${{ toJson(github.event.pull_request.labels.*.name) }}
     steps:
@@ -48,19 +51,52 @@ jobs:
         run: |
           # Remove any artifacts from the previous checkouts
           git clean -fxd
-      - name: Setup miniconda
+      - name: Populate CI build options
-        uses: conda-incubator/setup-miniconda@v2
+        run: |
-        with:
+          # Most builds use the lite interpreter, if certain builds shouldn't
-          auto-update-conda: true
+          # build the lite interpreter this env variable should get over-written
-          python-version: 3.8
+          # in the following case statement
-          activate-environment: build
+          echo "BUILD_LITE_INTERPRETER=1" >> "${GITHUB_ENV}"
-      - name: Install ios / conda Dependencies
+
+          case ${BUILD_ENVIRONMENT} in
+            *metal*)
+              echo "USE_PYTORCH_METAL=1" >> "${GITHUB_ENV}"
+              ;;
+            *full_jit*)
+              echo "BUILD_LITE_INTERPRETER=0" >> "${GITHUB_ENV}"
+              ;;
+            *custom*)
+              echo "SELECTED_OP_LIST=${GITHUB_WORKSPACE}/ios/TestApp/custom_build/mobilenetv2.yaml" >> "${GITHUB_ENV}"
+              ;;
+            *coreml*)
+              echo "USE_COREML_DELEGATE=1" >> "${GITHUB_ENV}"
+              ;;
+          esac
+      - name: Install brew dependencies
         run: |
           # Install dependencies
           brew install libtool
-          conda install numpy ninja pyyaml mkl mkl-include setuptools cmake cffi requests typing_extensions --yes
+      - name: Install conda and dependencies
+        run: |
+          # Install conda, setup-miniconda messes with the path that messes with the ruby stuff we do later on
+          curl --retry 3 -o "${RUNNER_TEMP}/conda.sh" https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+          chmod +x "${RUNNER_TEMP}/conda.sh"
+          /bin/bash "${RUNNER_TEMP}/conda.sh" -b -p "${RUNNER_TEMP}/anaconda"
+          echo "${RUNNER_TEMP}/anaconda/bin" >> "${GITHUB_PATH}"
+          # shellcheck disable=SC1091
+          source "${RUNNER_TEMP}/anaconda/bin/activate"
+          conda install -y \
+            cffi \
+            cmake \
+            mkl \
+            mkl-include \
+            ninja \
+            numpy \
+            pyyaml \
+            requests \
+            setuptools \
+            typing_extensions
       - name: Run Fastlane
-        shell: bash -e {0}
         run: |
           set -x
           cd ios/TestApp
@@ -82,11 +118,25 @@ jobs:
           rm cert.txt
       - name: Build
         run: |
+          # shellcheck disable=SC1091
+          source "${RUNNER_TEMP}/anaconda/bin/activate"
           export TCLLIBPATH="/usr/local/lib"
           python -VV
           export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname "$(which conda)")/../"}
           scripts/build_ios.sh
-
+      - name: Run Build Test
+        run: |
+          PROFILE=PyTorch_CI_2022
+          # run the ruby build script
+          if ! [ -x "$(command -v xcodebuild)" ]; then
+            echo 'Error: xcodebuild is not installed.'
+            exit 1
+          fi
+          if [ "${IOS_PLATFORM}" != "SIMULATOR" ]; then
+            ruby scripts/xcode_build.rb -i build_ios/install -x ios/TestApp/TestApp.xcodeproj -p "${IOS_PLATFORM}" -c "${PROFILE}" -t "${IOS_DEV_TEAM_ID}"
+          else
+            ruby scripts/xcode_build.rb -i build_ios/install -x ios/TestApp/TestApp.xcodeproj -p "${IOS_PLATFORM}"
+          fi
 
 concurrency:
   group: ios-12-5-1-arm64-full-jit-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}

.github/workflows/generated-ios-12-5-1-arm64-metal.yml

@@ -15,24 +15,27 @@ on:
       - 'ciflow/trunk/*'
   workflow_dispatch:
 
-# For setup-miniconda, see https://github.com/conda-incubator/setup-miniconda/issues/179
-defaults:
-  run:
-    shell: bash -x -e -l {0}
 env:
   BUILD_ENVIRONMENT: ios-12-5-1-arm64-metal
   IN_CI: 1
   IS_GHA: 1
+  IOS_PLATFORM: OS
+  IOS_ARCH: arm64
 
 
 jobs:
 
   build:
+    # NOTE: These builds will not run successfully without running on `pytorch/pytorch` due to the limitations
+    #       of accessing secrets from forked pull requests and IOS' dependency on secrets for their build/test
+    if: ${{ github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository }}
     runs-on: macos-10.15
     timeout-minutes: 240
     env:
       JOB_BASE_NAME: ios-12-5-1-arm64-metal-build
       IOS_CERT_KEY_2022: ${{ secrets.IOS_CERT_KEY_2022 }}
+      IOS_CERT_SECRET: ${{ secrets.IOS_CERT_SECRET }}
+      IOS_DEV_TEAM_ID: ${{ secrets.IOS_DEV_TEAM_ID }}
       IOS_SIGN_KEY_2022: ${{ secrets.IOS_SIGN_KEY_2022 }}
       PR_LABELS: ${{ toJson(github.event.pull_request.labels.*.name) }}
     steps:
@@ -48,19 +51,52 @@ jobs:
         run: |
           # Remove any artifacts from the previous checkouts
           git clean -fxd
-      - name: Setup miniconda
+      - name: Populate CI build options
-        uses: conda-incubator/setup-miniconda@v2
+        run: |
-        with:
+          # Most builds use the lite interpreter, if certain builds shouldn't
-          auto-update-conda: true
+          # build the lite interpreter this env variable should get over-written
-          python-version: 3.8
+          # in the following case statement
-          activate-environment: build
+          echo "BUILD_LITE_INTERPRETER=1" >> "${GITHUB_ENV}"
-      - name: Install ios / conda Dependencies
+
+          case ${BUILD_ENVIRONMENT} in
+            *metal*)
+              echo "USE_PYTORCH_METAL=1" >> "${GITHUB_ENV}"
+              ;;
+            *full_jit*)
+              echo "BUILD_LITE_INTERPRETER=0" >> "${GITHUB_ENV}"
+              ;;
+            *custom*)
+              echo "SELECTED_OP_LIST=${GITHUB_WORKSPACE}/ios/TestApp/custom_build/mobilenetv2.yaml" >> "${GITHUB_ENV}"
+              ;;
+            *coreml*)
+              echo "USE_COREML_DELEGATE=1" >> "${GITHUB_ENV}"
+              ;;
+          esac
+      - name: Install brew dependencies
         run: |
           # Install dependencies
           brew install libtool
-          conda install numpy ninja pyyaml mkl mkl-include setuptools cmake cffi requests typing_extensions --yes
+      - name: Install conda and dependencies
+        run: |
+          # Install conda, setup-miniconda messes with the path that messes with the ruby stuff we do later on
+          curl --retry 3 -o "${RUNNER_TEMP}/conda.sh" https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+          chmod +x "${RUNNER_TEMP}/conda.sh"
+          /bin/bash "${RUNNER_TEMP}/conda.sh" -b -p "${RUNNER_TEMP}/anaconda"
+          echo "${RUNNER_TEMP}/anaconda/bin" >> "${GITHUB_PATH}"
+          # shellcheck disable=SC1091
+          source "${RUNNER_TEMP}/anaconda/bin/activate"
+          conda install -y \
+            cffi \
+            cmake \
+            mkl \
+            mkl-include \
+            ninja \
+            numpy \
+            pyyaml \
+            requests \
+            setuptools \
+            typing_extensions
       - name: Run Fastlane
-        shell: bash -e {0}
         run: |
           set -x
           cd ios/TestApp
@@ -82,11 +118,25 @@ jobs:
           rm cert.txt
       - name: Build
         run: |
+          # shellcheck disable=SC1091
+          source "${RUNNER_TEMP}/anaconda/bin/activate"
           export TCLLIBPATH="/usr/local/lib"
           python -VV
           export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname "$(which conda)")/../"}
           scripts/build_ios.sh
-
+      - name: Run Build Test
+        run: |
+          PROFILE=PyTorch_CI_2022
+          # run the ruby build script
+          if ! [ -x "$(command -v xcodebuild)" ]; then
+            echo 'Error: xcodebuild is not installed.'
+            exit 1
+          fi
+          if [ "${IOS_PLATFORM}" != "SIMULATOR" ]; then
+            ruby scripts/xcode_build.rb -i build_ios/install -x ios/TestApp/TestApp.xcodeproj -p "${IOS_PLATFORM}" -c "${PROFILE}" -t "${IOS_DEV_TEAM_ID}"
+          else
+            ruby scripts/xcode_build.rb -i build_ios/install -x ios/TestApp/TestApp.xcodeproj -p "${IOS_PLATFORM}"
+          fi
 
 concurrency:
   group: ios-12-5-1-arm64-metal-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}

.github/workflows/generated-ios-12-5-1-arm64.yml

@@ -15,24 +15,27 @@ on:
       - 'ciflow/trunk/*'
   workflow_dispatch:
 
-# For setup-miniconda, see https://github.com/conda-incubator/setup-miniconda/issues/179
-defaults:
-  run:
-    shell: bash -x -e -l {0}
 env:
   BUILD_ENVIRONMENT: ios-12-5-1-arm64
   IN_CI: 1
   IS_GHA: 1
+  IOS_PLATFORM: OS
+  IOS_ARCH: arm64
 
 
 jobs:
 
   build:
+    # NOTE: These builds will not run successfully without running on `pytorch/pytorch` due to the limitations
+    #       of accessing secrets from forked pull requests and IOS' dependency on secrets for their build/test
+    if: ${{ github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository }}
     runs-on: macos-10.15
     timeout-minutes: 240
     env:
       JOB_BASE_NAME: ios-12-5-1-arm64-build
       IOS_CERT_KEY_2022: ${{ secrets.IOS_CERT_KEY_2022 }}
+      IOS_CERT_SECRET: ${{ secrets.IOS_CERT_SECRET }}
+      IOS_DEV_TEAM_ID: ${{ secrets.IOS_DEV_TEAM_ID }}
       IOS_SIGN_KEY_2022: ${{ secrets.IOS_SIGN_KEY_2022 }}
       PR_LABELS: ${{ toJson(github.event.pull_request.labels.*.name) }}
     steps:
@@ -48,19 +51,52 @@ jobs:
         run: |
           # Remove any artifacts from the previous checkouts
           git clean -fxd
-      - name: Setup miniconda
+      - name: Populate CI build options
-        uses: conda-incubator/setup-miniconda@v2
+        run: |
-        with:
+          # Most builds use the lite interpreter, if certain builds shouldn't
-          auto-update-conda: true
+          # build the lite interpreter this env variable should get over-written
-          python-version: 3.8
+          # in the following case statement
-          activate-environment: build
+          echo "BUILD_LITE_INTERPRETER=1" >> "${GITHUB_ENV}"
-      - name: Install ios / conda Dependencies
+
+          case ${BUILD_ENVIRONMENT} in
+            *metal*)
+              echo "USE_PYTORCH_METAL=1" >> "${GITHUB_ENV}"
+              ;;
+            *full_jit*)
+              echo "BUILD_LITE_INTERPRETER=0" >> "${GITHUB_ENV}"
+              ;;
+            *custom*)
+              echo "SELECTED_OP_LIST=${GITHUB_WORKSPACE}/ios/TestApp/custom_build/mobilenetv2.yaml" >> "${GITHUB_ENV}"
+              ;;
+            *coreml*)
+              echo "USE_COREML_DELEGATE=1" >> "${GITHUB_ENV}"
+              ;;
+          esac
+      - name: Install brew dependencies
         run: |
           # Install dependencies
           brew install libtool
-          conda install numpy ninja pyyaml mkl mkl-include setuptools cmake cffi requests typing_extensions --yes
+      - name: Install conda and dependencies
+        run: |
+          # Install conda, setup-miniconda messes with the path that messes with the ruby stuff we do later on
+          curl --retry 3 -o "${RUNNER_TEMP}/conda.sh" https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+          chmod +x "${RUNNER_TEMP}/conda.sh"
+          /bin/bash "${RUNNER_TEMP}/conda.sh" -b -p "${RUNNER_TEMP}/anaconda"
+          echo "${RUNNER_TEMP}/anaconda/bin" >> "${GITHUB_PATH}"
+          # shellcheck disable=SC1091
+          source "${RUNNER_TEMP}/anaconda/bin/activate"
+          conda install -y \
+            cffi \
+            cmake \
+            mkl \
+            mkl-include \
+            ninja \
+            numpy \
+            pyyaml \
+            requests \
+            setuptools \
+            typing_extensions
       - name: Run Fastlane
-        shell: bash -e {0}
         run: |
           set -x
           cd ios/TestApp
@@ -82,11 +118,25 @@ jobs:
           rm cert.txt
       - name: Build
         run: |
+          # shellcheck disable=SC1091
+          source "${RUNNER_TEMP}/anaconda/bin/activate"
           export TCLLIBPATH="/usr/local/lib"
           python -VV
           export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname "$(which conda)")/../"}
           scripts/build_ios.sh
-
+      - name: Run Build Test
+        run: |
+          PROFILE=PyTorch_CI_2022
+          # run the ruby build script
+          if ! [ -x "$(command -v xcodebuild)" ]; then
+            echo 'Error: xcodebuild is not installed.'
+            exit 1
+          fi
+          if [ "${IOS_PLATFORM}" != "SIMULATOR" ]; then
+            ruby scripts/xcode_build.rb -i build_ios/install -x ios/TestApp/TestApp.xcodeproj -p "${IOS_PLATFORM}" -c "${PROFILE}" -t "${IOS_DEV_TEAM_ID}"
+          else
+            ruby scripts/xcode_build.rb -i build_ios/install -x ios/TestApp/TestApp.xcodeproj -p "${IOS_PLATFORM}"
+          fi
 
 concurrency:
   group: ios-12-5-1-arm64-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}

.github/workflows/generated-ios-12-5-1-x86-64-coreml.yml

@@ -15,24 +15,27 @@ on:
       - 'ciflow/trunk/*'
   workflow_dispatch:
 
-# For setup-miniconda, see https://github.com/conda-incubator/setup-miniconda/issues/179
-defaults:
-  run:
-    shell: bash -x -e -l {0}
 env:
   BUILD_ENVIRONMENT: ios-12-5-1-x86-64-coreml
   IN_CI: 1
   IS_GHA: 1
+  IOS_PLATFORM: SIMULATOR
+  IOS_ARCH: x86_64
 
 
 jobs:
 
   build:
+    # NOTE: These builds will not run successfully without running on `pytorch/pytorch` due to the limitations
+    #       of accessing secrets from forked pull requests and IOS' dependency on secrets for their build/test
+    if: ${{ github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository }}
     runs-on: macos-10.15
     timeout-minutes: 240
     env:
       JOB_BASE_NAME: ios-12-5-1-x86-64-coreml-build
       IOS_CERT_KEY_2022: ${{ secrets.IOS_CERT_KEY_2022 }}
+      IOS_CERT_SECRET: ${{ secrets.IOS_CERT_SECRET }}
+      IOS_DEV_TEAM_ID: ${{ secrets.IOS_DEV_TEAM_ID }}
       IOS_SIGN_KEY_2022: ${{ secrets.IOS_SIGN_KEY_2022 }}
       PR_LABELS: ${{ toJson(github.event.pull_request.labels.*.name) }}
     steps:
@@ -48,19 +51,52 @@ jobs:
         run: |
           # Remove any artifacts from the previous checkouts
           git clean -fxd
-      - name: Setup miniconda
+      - name: Populate CI build options
-        uses: conda-incubator/setup-miniconda@v2
+        run: |
-        with:
+          # Most builds use the lite interpreter, if certain builds shouldn't
-          auto-update-conda: true
+          # build the lite interpreter this env variable should get over-written
-          python-version: 3.8
+          # in the following case statement
-          activate-environment: build
+          echo "BUILD_LITE_INTERPRETER=1" >> "${GITHUB_ENV}"
-      - name: Install ios / conda Dependencies
+
+          case ${BUILD_ENVIRONMENT} in
+            *metal*)
+              echo "USE_PYTORCH_METAL=1" >> "${GITHUB_ENV}"
+              ;;
+            *full_jit*)
+              echo "BUILD_LITE_INTERPRETER=0" >> "${GITHUB_ENV}"
+              ;;
+            *custom*)
+              echo "SELECTED_OP_LIST=${GITHUB_WORKSPACE}/ios/TestApp/custom_build/mobilenetv2.yaml" >> "${GITHUB_ENV}"
+              ;;
+            *coreml*)
+              echo "USE_COREML_DELEGATE=1" >> "${GITHUB_ENV}"
+              ;;
+          esac
+      - name: Install brew dependencies
         run: |
           # Install dependencies
           brew install libtool
-          conda install numpy ninja pyyaml mkl mkl-include setuptools cmake cffi requests typing_extensions --yes
+      - name: Install conda and dependencies
+        run: |
+          # Install conda, setup-miniconda messes with the path that messes with the ruby stuff we do later on
+          curl --retry 3 -o "${RUNNER_TEMP}/conda.sh" https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+          chmod +x "${RUNNER_TEMP}/conda.sh"
+          /bin/bash "${RUNNER_TEMP}/conda.sh" -b -p "${RUNNER_TEMP}/anaconda"
+          echo "${RUNNER_TEMP}/anaconda/bin" >> "${GITHUB_PATH}"
+          # shellcheck disable=SC1091
+          source "${RUNNER_TEMP}/anaconda/bin/activate"
+          conda install -y \
+            cffi \
+            cmake \
+            mkl \
+            mkl-include \
+            ninja \
+            numpy \
+            pyyaml \
+            requests \
+            setuptools \
+            typing_extensions
       - name: Run Fastlane
-        shell: bash -e {0}
         run: |
           set -x
           cd ios/TestApp
@@ -82,11 +118,58 @@ jobs:
           rm cert.txt
       - name: Build
         run: |
+          # shellcheck disable=SC1091
+          source "${RUNNER_TEMP}/anaconda/bin/activate"
           export TCLLIBPATH="/usr/local/lib"
           python -VV
           export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname "$(which conda)")/../"}
           scripts/build_ios.sh
-
+      - name: Run Build Test
+        run: |
+          PROFILE=PyTorch_CI_2022
+          # run the ruby build script
+          if ! [ -x "$(command -v xcodebuild)" ]; then
+            echo 'Error: xcodebuild is not installed.'
+            exit 1
+          fi
+          if [ "${IOS_PLATFORM}" != "SIMULATOR" ]; then
+            ruby scripts/xcode_build.rb -i build_ios/install -x ios/TestApp/TestApp.xcodeproj -p "${IOS_PLATFORM}" -c "${PROFILE}" -t "${IOS_DEV_TEAM_ID}"
+          else
+            ruby scripts/xcode_build.rb -i build_ios/install -x ios/TestApp/TestApp.xcodeproj -p "${IOS_PLATFORM}"
+          fi
+      - name: Run Simulator Tests
+        run: |
+          # shellcheck disable=SC1091
+          source "${RUNNER_TEMP}/anaconda/bin/activate"
+          pip3 install --pre torch torchvision torchaudio -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
+          # generate models for differnet backends
+          cd "${GITHUB_WORKSPACE}/ios/TestApp/benchmark"
+          mkdir -p ../models
+          if [ "${USE_COREML_DELEGATE}" == 1 ]; then
+            pip install coremltools==5.0b5
+            pip install six==1.16.0
+            python coreml_backend.py
+          else
+            python trace_model.py
+          fi
+          if [ "${BUILD_LITE_INTERPRETER}" == 1 ]; then
+            echo "Setting up the TestApp for LiteInterpreter"
+            ruby setup.rb --lite 1
+          else
+            echo "Setting up the TestApp for Full JIT"
+            ruby setup.rb
+          fi
+          cd "${GITHUB_WORKSPACE}/ios/TestApp"
+          instruments -s -devices
+          if [ "${BUILD_LITE_INTERPRETER}" == 1 ]; then
+            if [ "${USE_COREML_DELEGATE}" == 1 ]; then
+              fastlane scan --only_testing TestAppTests/TestAppTests/testCoreML
+            else
+              fastlane scan --only_testing TestAppTests/TestAppTests/testLiteInterpreter
+            fi
+          else
+            fastlane scan --only_testing TestAppTests/TestAppTests/testFullJIT
+          fi
 
 concurrency:
   group: ios-12-5-1-x86-64-coreml-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}

.github/workflows/generated-ios-12-5-1-x86-64-full-jit.yml

@@ -15,24 +15,27 @@ on:
       - 'ciflow/trunk/*'
   workflow_dispatch:
 
-# For setup-miniconda, see https://github.com/conda-incubator/setup-miniconda/issues/179
-defaults:
-  run:
-    shell: bash -x -e -l {0}
 env:
   BUILD_ENVIRONMENT: ios-12-5-1-x86-64-full-jit
   IN_CI: 1
   IS_GHA: 1
+  IOS_PLATFORM: SIMULATOR
+  IOS_ARCH: x86_64
 
 
 jobs:
 
   build:
+    # NOTE: These builds will not run successfully without running on `pytorch/pytorch` due to the limitations
+    #       of accessing secrets from forked pull requests and IOS' dependency on secrets for their build/test
+    if: ${{ github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository }}
     runs-on: macos-10.15
     timeout-minutes: 240
     env:
       JOB_BASE_NAME: ios-12-5-1-x86-64-full-jit-build
       IOS_CERT_KEY_2022: ${{ secrets.IOS_CERT_KEY_2022 }}
+      IOS_CERT_SECRET: ${{ secrets.IOS_CERT_SECRET }}
+      IOS_DEV_TEAM_ID: ${{ secrets.IOS_DEV_TEAM_ID }}
       IOS_SIGN_KEY_2022: ${{ secrets.IOS_SIGN_KEY_2022 }}
       PR_LABELS: ${{ toJson(github.event.pull_request.labels.*.name) }}
     steps:
@@ -48,19 +51,52 @@ jobs:
         run: |
           # Remove any artifacts from the previous checkouts
           git clean -fxd
-      - name: Setup miniconda
+      - name: Populate CI build options
-        uses: conda-incubator/setup-miniconda@v2
+        run: |
-        with:
+          # Most builds use the lite interpreter, if certain builds shouldn't
-          auto-update-conda: true
+          # build the lite interpreter this env variable should get over-written
-          python-version: 3.8
+          # in the following case statement
-          activate-environment: build
+          echo "BUILD_LITE_INTERPRETER=1" >> "${GITHUB_ENV}"
-      - name: Install ios / conda Dependencies
+
+          case ${BUILD_ENVIRONMENT} in
+            *metal*)
+              echo "USE_PYTORCH_METAL=1" >> "${GITHUB_ENV}"
+              ;;
+            *full_jit*)
+              echo "BUILD_LITE_INTERPRETER=0" >> "${GITHUB_ENV}"
+              ;;
+            *custom*)
+              echo "SELECTED_OP_LIST=${GITHUB_WORKSPACE}/ios/TestApp/custom_build/mobilenetv2.yaml" >> "${GITHUB_ENV}"
+              ;;
+            *coreml*)
+              echo "USE_COREML_DELEGATE=1" >> "${GITHUB_ENV}"
+              ;;
+          esac
+      - name: Install brew dependencies
         run: |
           # Install dependencies
           brew install libtool
-          conda install numpy ninja pyyaml mkl mkl-include setuptools cmake cffi requests typing_extensions --yes
+      - name: Install conda and dependencies
+        run: |
+          # Install conda, setup-miniconda messes with the path that messes with the ruby stuff we do later on
+          curl --retry 3 -o "${RUNNER_TEMP}/conda.sh" https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+          chmod +x "${RUNNER_TEMP}/conda.sh"
+          /bin/bash "${RUNNER_TEMP}/conda.sh" -b -p "${RUNNER_TEMP}/anaconda"
+          echo "${RUNNER_TEMP}/anaconda/bin" >> "${GITHUB_PATH}"
+          # shellcheck disable=SC1091
+          source "${RUNNER_TEMP}/anaconda/bin/activate"
+          conda install -y \
+            cffi \
+            cmake \
+            mkl \
+            mkl-include \
+            ninja \
+            numpy \
+            pyyaml \
+            requests \
+            setuptools \
+            typing_extensions
       - name: Run Fastlane
-        shell: bash -e {0}
         run: |
           set -x
           cd ios/TestApp
@@ -82,11 +118,58 @@ jobs:
           rm cert.txt
       - name: Build
         run: |
+          # shellcheck disable=SC1091
+          source "${RUNNER_TEMP}/anaconda/bin/activate"
           export TCLLIBPATH="/usr/local/lib"
           python -VV
           export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname "$(which conda)")/../"}
           scripts/build_ios.sh
-
+      - name: Run Build Test
+        run: |
+          PROFILE=PyTorch_CI_2022
+          # run the ruby build script
+          if ! [ -x "$(command -v xcodebuild)" ]; then
+            echo 'Error: xcodebuild is not installed.'
+            exit 1
+          fi
+          if [ "${IOS_PLATFORM}" != "SIMULATOR" ]; then
+            ruby scripts/xcode_build.rb -i build_ios/install -x ios/TestApp/TestApp.xcodeproj -p "${IOS_PLATFORM}" -c "${PROFILE}" -t "${IOS_DEV_TEAM_ID}"
+          else
+            ruby scripts/xcode_build.rb -i build_ios/install -x ios/TestApp/TestApp.xcodeproj -p "${IOS_PLATFORM}"
+          fi
+      - name: Run Simulator Tests
+        run: |
+          # shellcheck disable=SC1091
+          source "${RUNNER_TEMP}/anaconda/bin/activate"
+          pip3 install --pre torch torchvision torchaudio -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
+          # generate models for differnet backends
+          cd "${GITHUB_WORKSPACE}/ios/TestApp/benchmark"
+          mkdir -p ../models
+          if [ "${USE_COREML_DELEGATE}" == 1 ]; then
+            pip install coremltools==5.0b5
+            pip install six==1.16.0
+            python coreml_backend.py
+          else
+            python trace_model.py
+          fi
+          if [ "${BUILD_LITE_INTERPRETER}" == 1 ]; then
+            echo "Setting up the TestApp for LiteInterpreter"
+            ruby setup.rb --lite 1
+          else
+            echo "Setting up the TestApp for Full JIT"
+            ruby setup.rb
+          fi
+          cd "${GITHUB_WORKSPACE}/ios/TestApp"
+          instruments -s -devices
+          if [ "${BUILD_LITE_INTERPRETER}" == 1 ]; then
+            if [ "${USE_COREML_DELEGATE}" == 1 ]; then
+              fastlane scan --only_testing TestAppTests/TestAppTests/testCoreML
+            else
+              fastlane scan --only_testing TestAppTests/TestAppTests/testLiteInterpreter
+            fi
+          else
+            fastlane scan --only_testing TestAppTests/TestAppTests/testFullJIT
+          fi
 
 concurrency:
   group: ios-12-5-1-x86-64-full-jit-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}

.github/workflows/generated-ios-12-5-1-x86-64.yml

@@ -15,24 +15,27 @@ on:
       - 'ciflow/trunk/*'
   workflow_dispatch:
 
-# For setup-miniconda, see https://github.com/conda-incubator/setup-miniconda/issues/179
-defaults:
-  run:
-    shell: bash -x -e -l {0}
 env:
   BUILD_ENVIRONMENT: ios-12-5-1-x86-64
   IN_CI: 1
   IS_GHA: 1
+  IOS_PLATFORM: SIMULATOR
+  IOS_ARCH: x86_64
 
 
 jobs:
 
   build:
+    # NOTE: These builds will not run successfully without running on `pytorch/pytorch` due to the limitations
+    #       of accessing secrets from forked pull requests and IOS' dependency on secrets for their build/test
+    if: ${{ github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository }}
     runs-on: macos-10.15
     timeout-minutes: 240
     env:
       JOB_BASE_NAME: ios-12-5-1-x86-64-build
       IOS_CERT_KEY_2022: ${{ secrets.IOS_CERT_KEY_2022 }}
+      IOS_CERT_SECRET: ${{ secrets.IOS_CERT_SECRET }}
+      IOS_DEV_TEAM_ID: ${{ secrets.IOS_DEV_TEAM_ID }}
       IOS_SIGN_KEY_2022: ${{ secrets.IOS_SIGN_KEY_2022 }}
       PR_LABELS: ${{ toJson(github.event.pull_request.labels.*.name) }}
     steps:
@@ -48,19 +51,52 @@ jobs:
         run: |
           # Remove any artifacts from the previous checkouts
           git clean -fxd
-      - name: Setup miniconda
+      - name: Populate CI build options
-        uses: conda-incubator/setup-miniconda@v2
+        run: |
-        with:
+          # Most builds use the lite interpreter, if certain builds shouldn't
-          auto-update-conda: true
+          # build the lite interpreter this env variable should get over-written
-          python-version: 3.8
+          # in the following case statement
-          activate-environment: build
+          echo "BUILD_LITE_INTERPRETER=1" >> "${GITHUB_ENV}"
-      - name: Install ios / conda Dependencies
+
+          case ${BUILD_ENVIRONMENT} in
+            *metal*)
+              echo "USE_PYTORCH_METAL=1" >> "${GITHUB_ENV}"
+              ;;
+            *full_jit*)
+              echo "BUILD_LITE_INTERPRETER=0" >> "${GITHUB_ENV}"
+              ;;
+            *custom*)
+              echo "SELECTED_OP_LIST=${GITHUB_WORKSPACE}/ios/TestApp/custom_build/mobilenetv2.yaml" >> "${GITHUB_ENV}"
+              ;;
+            *coreml*)
+              echo "USE_COREML_DELEGATE=1" >> "${GITHUB_ENV}"
+              ;;
+          esac
+      - name: Install brew dependencies
         run: |
           # Install dependencies
           brew install libtool
-          conda install numpy ninja pyyaml mkl mkl-include setuptools cmake cffi requests typing_extensions --yes
+      - name: Install conda and dependencies
+        run: |
+          # Install conda, setup-miniconda messes with the path that messes with the ruby stuff we do later on
+          curl --retry 3 -o "${RUNNER_TEMP}/conda.sh" https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+          chmod +x "${RUNNER_TEMP}/conda.sh"
+          /bin/bash "${RUNNER_TEMP}/conda.sh" -b -p "${RUNNER_TEMP}/anaconda"
+          echo "${RUNNER_TEMP}/anaconda/bin" >> "${GITHUB_PATH}"
+          # shellcheck disable=SC1091
+          source "${RUNNER_TEMP}/anaconda/bin/activate"
+          conda install -y \
+            cffi \
+            cmake \
+            mkl \
+            mkl-include \
+            ninja \
+            numpy \
+            pyyaml \
+            requests \
+            setuptools \
+            typing_extensions
       - name: Run Fastlane
-        shell: bash -e {0}
         run: |
           set -x
           cd ios/TestApp
@@ -82,11 +118,58 @@ jobs:
           rm cert.txt
       - name: Build
         run: |
+          # shellcheck disable=SC1091
+          source "${RUNNER_TEMP}/anaconda/bin/activate"
           export TCLLIBPATH="/usr/local/lib"
           python -VV
           export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname "$(which conda)")/../"}
           scripts/build_ios.sh
-
+      - name: Run Build Test
+        run: |
+          PROFILE=PyTorch_CI_2022
+          # run the ruby build script
+          if ! [ -x "$(command -v xcodebuild)" ]; then
+            echo 'Error: xcodebuild is not installed.'
+            exit 1
+          fi
+          if [ "${IOS_PLATFORM}" != "SIMULATOR" ]; then
+            ruby scripts/xcode_build.rb -i build_ios/install -x ios/TestApp/TestApp.xcodeproj -p "${IOS_PLATFORM}" -c "${PROFILE}" -t "${IOS_DEV_TEAM_ID}"
+          else
+            ruby scripts/xcode_build.rb -i build_ios/install -x ios/TestApp/TestApp.xcodeproj -p "${IOS_PLATFORM}"
+          fi
+      - name: Run Simulator Tests
+        run: |
+          # shellcheck disable=SC1091
+          source "${RUNNER_TEMP}/anaconda/bin/activate"
+          pip3 install --pre torch torchvision torchaudio -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
+          # generate models for differnet backends
+          cd "${GITHUB_WORKSPACE}/ios/TestApp/benchmark"
+          mkdir -p ../models
+          if [ "${USE_COREML_DELEGATE}" == 1 ]; then
+            pip install coremltools==5.0b5
+            pip install six==1.16.0
+            python coreml_backend.py
+          else
+            python trace_model.py
+          fi
+          if [ "${BUILD_LITE_INTERPRETER}" == 1 ]; then
+            echo "Setting up the TestApp for LiteInterpreter"
+            ruby setup.rb --lite 1
+          else
+            echo "Setting up the TestApp for Full JIT"
+            ruby setup.rb
+          fi
+          cd "${GITHUB_WORKSPACE}/ios/TestApp"
+          instruments -s -devices
+          if [ "${BUILD_LITE_INTERPRETER}" == 1 ]; then
+            if [ "${USE_COREML_DELEGATE}" == 1 ]; then
+              fastlane scan --only_testing TestAppTests/TestAppTests/testCoreML
+            else
+              fastlane scan --only_testing TestAppTests/TestAppTests/testLiteInterpreter
+            fi
+          else
+            fastlane scan --only_testing TestAppTests/TestAppTests/testFullJIT
+          fi
 
 concurrency:
   group: ios-12-5-1-x86-64-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}

scripts/build_ios.sh

@@ -7,55 +7,6 @@
 # using ios-cmake. This is very similar to the android-cmake - see
 # build_android.sh for more details.
 
-# For CI purposes
-populate_ci_build_options() {
-  # For use within CI, should be a no-op for everyone else
-
-  # BUILD_ENVIRONMENT's should look somewhat like pytorch_ios_12_5_1_arm64_coreml
-  BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT:-}
-
-  case ${BUILD_ENVIRONMENT} in
-    *x86_64*)
-      export IOS_ARCH="x86_64"
-      export IOS_PLATFORM="SIMULATOR"
-      ;;
-    *arm64*)
-      export IOS_ARCH="arm64"
-      export IOS_PLATFORM="OS"
-      ;;
-  esac
-
-  # Most builds use the lite interpreter, if certain builds shouldn't
-  # build the lite interpreter this env variable should get over-written
-  # in the following case statement
-  export BUILD_LITE_INTERPRETER="1"
-
-  case ${BUILD_ENVIRONMENT} in
-    *metal*)
-      export USE_PYTORCH_METAL=1
-      ;;
-    *full_jit*)
-      export BUILD_LITE_INTERPRETER="0"
-      ;;
-    *custom*)
-      PROJ_ROOT=$(git rev-parse --show-toplevel)
-      export SELECTED_OP_LIST="${PROJ_ROOT}/ios/TestApp/custom_build/mobilenetv2.yaml"
-      ;;
-    *coreml*)
-      export USE_COREML_DELEGATE="1"
-      ;;
-  esac
-  if [[ -n ${BUILD_ENVIRONMENT} ]]; then
-    echo "IOS_ARCH: ${IOS_ARCH}"
-    echo "IOS_PLATFORM: ${IOS_PLATFORM}"
-    echo "BUILD_LITE_INTERPRETER": "${BUILD_LITE_INTERPRETER}"
-    echo "USE_PYTORCH_METAL": "${USE_METAL:-}"
-    echo "USE_COREML_DELEGATE": "${USE_COREML_DELEGATE:-}"
-  fi
-}
-
-populate_ci_build_options
-
 CAFFE2_ROOT="$( cd "$(dirname "$0")"/.. ; pwd -P)"
 
 CMAKE_ARGS=()