[CI][MacOS] Move more dependencies to pypi (#154309)

Hopefully last step before all Mac build/tests could be switched away from conda
- Update cmake version from 3.22 to 3.25 as 3.22 from pipy seems  to be unusable with python-3.12
- Add `--plat-name macosx_11_0_arm64` to setup.py command
- Remove `codesign` for cmake workaround (that was probably never really necessary
-  Install `libpng` and `jpeg-turbo` when building torchbench and build torchaudio without OpenMP (to be fixed)
Pull Request resolved: https://github.com/pytorch/pytorch/pull/154309
Approved by: https://github.com/Skylion007, https://github.com/cyyever

.ci/pytorch/macos-build.sh

@@ -40,7 +40,7 @@ if [[ ${BUILD_ENVIRONMENT} == *"distributed"* ]]; then
 else
   # Explicitly set USE_DISTRIBUTED=0 to align with the default build config on mac. This also serves as the sole CI config that tests
   # that building with USE_DISTRIBUTED=0 works at all. See https://github.com/pytorch/pytorch/issues/86448
-  USE_DISTRIBUTED=0 USE_OPENMP=1 MACOSX_DEPLOYMENT_TARGET=11.0 WERROR=1 BUILD_TEST=OFF USE_PYTORCH_METAL=1 python setup.py bdist_wheel
+  USE_DISTRIBUTED=0 USE_OPENMP=1 MACOSX_DEPLOYMENT_TARGET=11.0 WERROR=1 BUILD_TEST=OFF USE_PYTORCH_METAL=1 python setup.py bdist_wheel --plat-name macosx_11_0_arm64
 fi
 if which sccache > /dev/null; then
   print_sccache_stats

.ci/pytorch/macos-common.sh

@@ -20,14 +20,4 @@ print_cmake_info() {
   CONDA_INSTALLATION_DIR=$(dirname "$CMAKE_EXEC")
   # Print all libraries under cmake rpath for debugging
   ls -la "$CONDA_INSTALLATION_DIR/../lib"
-
-  export CMAKE_EXEC
-  # Explicitly add conda env lib folder to cmake rpath to address the flaky issue
-  # where cmake dependencies couldn't be found. This seems to point to how conda
-  # links $CMAKE_EXEC to its package cache when cloning a new environment
-  install_name_tool -add_rpath @executable_path/../lib "${CMAKE_EXEC}" || true
-  # Adding the rpath will invalidate cmake signature, so signing it again here
-  # to trust the executable. EXC_BAD_ACCESS (SIGKILL (Code Signature Invalid))
-  # with an exit code 137 otherwise
-  codesign -f -s - "${CMAKE_EXEC}" || true
 }

.ci/pytorch/macos-test.sh

@@ -165,6 +165,7 @@ test_jit_hooks() {
 torchbench_setup_macos() {
   git clone --recursive https://github.com/pytorch/vision torchvision
   git clone --recursive https://github.com/pytorch/audio torchaudio
+  brew install jpeg-turbo libpng
 
   pushd torchvision
   git fetch
@@ -179,7 +180,8 @@ torchbench_setup_macos() {
   git checkout "$(cat ../.github/ci_commit_pins/audio.txt)"
   git submodule update --init --recursive
   python setup.py clean
-  python setup.py develop
+  #TODO: Remove me, when figure out how to make TorchAudio find brew installed openmp
+  USE_OPENMP=0 python setup.py develop
   popd
 
   # Shellcheck doesn't like it when you pass no arguments to a function that can take args. See https://www.shellcheck.net/wiki/SC2120
@@ -187,8 +189,8 @@ torchbench_setup_macos() {
   checkout_install_torchbench
 }
 
-conda_benchmark_deps() {
-  conda install -y astunparse numpy scipy ninja pyyaml setuptools cmake typing-extensions requests protobuf numba cython scikit-learn
+pip_benchmark_deps() {
+  python -mpip install --no-input astunparse requests cython scikit-learn
 }
 
 
@@ -196,7 +198,7 @@ test_torchbench_perf() {
   print_cmake_info
 
   echo "Launching torchbench setup"
-  conda_benchmark_deps
+  pip_benchmark_deps
   torchbench_setup_macos
 
   TEST_REPORTS_DIR=$(pwd)/test/test-reports
@@ -223,7 +225,7 @@ test_torchbench_smoketest() {
   print_cmake_info
 
   echo "Launching torchbench setup"
-  conda_benchmark_deps
+  pip_benchmark_deps
   # shellcheck disable=SC2119,SC2120
   torchbench_setup_macos
 
@@ -288,7 +290,7 @@ test_hf_perf() {
   print_cmake_info
   TEST_REPORTS_DIR=$(pwd)/test/test-reports
   mkdir -p "$TEST_REPORTS_DIR"
-  conda_benchmark_deps
+  pip_benchmark_deps
   torchbench_setup_macos
 
   echo "Launching HuggingFace training perf run"
@@ -304,7 +306,7 @@ test_timm_perf() {
   print_cmake_info
   TEST_REPORTS_DIR=$(pwd)/test/test-reports
   mkdir -p "$TEST_REPORTS_DIR"
-  conda_benchmark_deps
+  pip_benchmark_deps
   torchbench_setup_macos
 
   echo "Launching timm training perf run"

.github/requirements/conda-env-macOS-ARM64

@@ -1,12 +1,6 @@
 # Not pinning certifi so that we can always get the latest certificates
 certifi
-cmake=3.22.*
-expecttest=0.3.0
-numpy=1.26.4
-pillow=10.0.1
 pip=23.2.1
 pkg-config=0.29.2
-pyyaml=6.0.2
 setuptools=72.1.0
-typing-extensions=4.11.0
 wheel=0.37.1

.github/requirements/pip-requirements-macOS.txt

@@ -1,4 +1,5 @@
 boto3==1.35.42
+cmake==3.25.*
 expecttest==0.3.0
 fbscribelogger==0.1.7
 filelock==3.6.0
@@ -6,11 +7,14 @@ hypothesis==6.56.4
 librosa>=0.6.2
 mpmath==1.3.0
 networkx==2.8.7
+ninja==1.10.2.4
 numba==0.59.0
+numpy==1.26.4
 opt-einsum>=3.3
 optree==0.13.0
 packaging==23.1
 parameterized==0.8.1
+pillow==10.0.1
 protobuf==5.29.4
 psutil==5.9.1
 pygments==2.15.0
@@ -20,9 +24,11 @@ pytest-rerunfailures==10.3
 pytest-subtests==0.13.1
 pytest-xdist==3.3.1
 pytest==7.3.2
+pyyaml==6.0.2
 scipy==1.12.0
 sympy==1.13.3
 tensorboard==2.13.0
+typing-extensions==4.12.2
 unittest-xml-reporting<=3.2.0,>=2.0.0
 xdoctest==1.1.0
 z3-solver==4.12.2.0

.github/workflows/_mac-build.yml

@@ -87,6 +87,7 @@ jobs:
           python-version: ${{ inputs.python-version }}
           environment-file: .github/requirements/conda-env-macOS-ARM64
           pip-requirements-file: .github/requirements/pip-requirements-macOS.txt
+          default-packages: ""
 
       - name: Install sccache (only for non-forked PRs, and pushes to trunk)
         uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0

.github/workflows/_mac-test.yml

@@ -146,6 +146,7 @@ jobs:
           python-version: ${{ inputs.python-version }}
           environment-file: .github/requirements/conda-env-macOS-ARM64
           pip-requirements-file: .github/requirements/pip-requirements-macOS.txt
+          default-packages: ""
 
       - name: Parse ref
         id: parse-ref