diff --git a/.github/workflows/_linux-test.yml b/.github/workflows/_linux-test.yml index aa7a2bdf8f1..41a976b18c7 100644 --- a/.github/workflows/_linux-test.yml +++ b/.github/workflows/_linux-test.yml @@ -54,7 +54,7 @@ on: since we are investigating the behaviour of the monitor script with different tests. required: false type: boolean - default: true + default: false secrets: HUGGING_FACE_HUB_TOKEN: required: false diff --git a/.github/workflows/_mac-test.yml b/.github/workflows/_mac-test.yml index c030911caf1..f7f0902584c 100644 --- a/.github/workflows/_mac-test.yml +++ b/.github/workflows/_mac-test.yml @@ -88,6 +88,14 @@ jobs: - name: Checkout PyTorch uses: pytorch/pytorch/.github/actions/checkout-pytorch@main + - name: Start monitoring script + id: monitor-script + if: ${{ !inputs.disable-monitor }} + continue-on-error: true + run: | + ${CONDA_RUN} python3 -m tools.stats.monitor > usage_log.txt 2>&1 & + echo "monitor-script-pid=${!}" >> "${GITHUB_OUTPUT}" + - name: Download build artifacts uses: ./.github/actions/download-build-artifacts with: @@ -107,14 +115,6 @@ jobs: environment-file: .github/requirements/conda-env-${{ runner.os }}-${{ runner.arch }} pip-requirements-file: .github/requirements/pip-requirements-${{ runner.os }}.txt - - name: Start monitoring script - id: monitor-script - if: ${{ !inputs.disable-monitor }} - continue-on-error: true - run: | - ${CONDA_RUN} python3 -m tools.stats.monitor > usage_log.txt 2>&1 & - echo "monitor-script-pid=${!}" >> "${GITHUB_OUTPUT}" - - name: Parse ref id: parse-ref run: .github/scripts/parse_ref.py diff --git a/.github/workflows/inductor-perf-compare.yml b/.github/workflows/inductor-perf-compare.yml index d8322abe2d4..8b341d2c44d 100644 --- a/.github/workflows/inductor-perf-compare.yml +++ b/.github/workflows/inductor-perf-compare.yml @@ -63,4 +63,6 @@ jobs: docker-image: ${{ needs.linux-focal-cuda12_1-py3_10-gcc9-inductor-build.outputs.docker-image }} test-matrix: ${{ needs.linux-focal-cuda12_1-py3_10-gcc9-inductor-build.outputs.test-matrix }} use-gha: anything-non-empty-to-use-gha + # disable monitor in perf tests for more investigation + disable-monitor: true secrets: inherit diff --git a/.github/workflows/inductor-perf-test-nightly-aarch64.yml b/.github/workflows/inductor-perf-test-nightly-aarch64.yml index 93d261875fc..459c7901b06 100644 --- a/.github/workflows/inductor-perf-test-nightly-aarch64.yml +++ b/.github/workflows/inductor-perf-test-nightly-aarch64.yml @@ -124,6 +124,8 @@ jobs: test-matrix: ${{ needs.linux-jammy-aarch64-py3_10-inductor-build.outputs.test-matrix }} use-gha: anything-non-empty-to-use-gha timeout-minutes: 720 + # disable monitor in perf tests for more investigation + disable-monitor: true secrets: inherit diff --git a/.github/workflows/inductor-perf-test-nightly-macos.yml b/.github/workflows/inductor-perf-test-nightly-macos.yml index 9a884e4d222..a63731f759b 100644 --- a/.github/workflows/inductor-perf-test-nightly-macos.yml +++ b/.github/workflows/inductor-perf-test-nightly-macos.yml @@ -55,4 +55,6 @@ jobs: # Same as the build job python-version: 3.9.12 test-matrix: ${{ needs.macos-perf-py3-arm64-build.outputs.test-matrix }} + # disable monitor in perf tests for more investigation + disable-monitor: true secrets: inherit diff --git a/.github/workflows/inductor-perf-test-nightly-x86.yml b/.github/workflows/inductor-perf-test-nightly-x86.yml index 21eaca01aa4..d4e325d8fd7 100644 --- a/.github/workflows/inductor-perf-test-nightly-x86.yml +++ b/.github/workflows/inductor-perf-test-nightly-x86.yml @@ -97,6 +97,8 @@ jobs: test-matrix: ${{ needs.linux-jammy-cpu-py3_9-gcc11-inductor-build.outputs.test-matrix }} use-gha: anything-non-empty-to-use-gha timeout-minutes: 720 + # disable monitor in perf tests for more investigation + disable-monitor: true secrets: inherit @@ -112,4 +114,6 @@ jobs: test-matrix: ${{ needs.linux-jammy-cpu-py3_9-gcc11-inductor-build.outputs.test-matrix }} use-gha: anything-non-empty-to-use-gha timeout-minutes: 720 + # disable monitor in perf tests for more investigation + disable-monitor: true secrets: inherit diff --git a/.github/workflows/inductor-perf-test-nightly.yml b/.github/workflows/inductor-perf-test-nightly.yml index 8ad63166089..94f642ae2f5 100644 --- a/.github/workflows/inductor-perf-test-nightly.yml +++ b/.github/workflows/inductor-perf-test-nightly.yml @@ -116,6 +116,8 @@ jobs: test-matrix: ${{ needs.linux-focal-cuda12_1-py3_10-gcc9-inductor-build.outputs.test-matrix }} use-gha: anything-non-empty-to-use-gha timeout-minutes: 720 + # disable monitor in perf tests for more investigation + disable-monitor: true secrets: inherit linux-focal-cuda12_1-py3_10-gcc9-inductor-test-weekly: @@ -130,6 +132,8 @@ jobs: test-matrix: ${{ needs.linux-focal-cuda12_1-py3_10-gcc9-inductor-build.outputs.test-matrix }} use-gha: anything-non-empty-to-use-gha timeout-minutes: 1440 + # disable monitor in perf tests for more investigation + disable-monitor: true secrets: inherit linux-focal-cuda12_1-py3_10-gcc9-inductor-test: @@ -144,4 +148,6 @@ jobs: test-matrix: ${{ needs.linux-focal-cuda12_1-py3_10-gcc9-inductor-build.outputs.test-matrix }} use-gha: anything-non-empty-to-use-gha timeout-minutes: 720 + # disable monitor in perf tests for more investigation + disable-monitor: true secrets: inherit diff --git a/.github/workflows/inductor-periodic.yml b/.github/workflows/inductor-periodic.yml index 71662df7a6d..402cff71df9 100644 --- a/.github/workflows/inductor-periodic.yml +++ b/.github/workflows/inductor-periodic.yml @@ -104,6 +104,8 @@ jobs: docker-image: ${{ needs.linux-focal-cuda12_1-py3_10-gcc9-inductor-build-gcp.outputs.docker-image }} test-matrix: ${{ needs.linux-focal-cuda12_1-py3_10-gcc9-inductor-build-gcp.outputs.test-matrix }} use-gha: anything-non-empty-to-use-gha + # disable monitor in smoke perf tests for more investigation + disable-monitor: true secrets: inherit linux-jammy-cpu-py3_9-gcc11-periodic-dynamo-benchmarks-build: