OSSForks/pytorch
1
0
Fork 0
mirror of https://github.com/zebrajr/pytorch.git synced 2025-12-07 00:21:07 +01:00
Code Issues Actions 38 Packages Projects Releases Wiki Activity

Revert D29703523: [pytorch][PR] Fix scribe logs

Browse Source 
Test Plan: revert-hammer

Differential Revision:
D29703523 (eb5a56fb74)

Original commit changeset: 829ad3630d35

fbshipit-source-id: 2b2196d58791b995a008b6d810b3248ed27e7d94
This commit is contained in:
Jeffrey Wan 2021-07-16 08:46:45 -07:00 committed by Facebook GitHub Bot
parent b963607d50
commit 3624d75864
22 changed files with 188 additions and 181 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.circleci/config.yml
View File

@ -212,7 +212,7 @@ commands:
cd ~/project cd ~/project
export ANDROID_BUILD_TYPE="<< parameters.build_type >>" export ANDROID_BUILD_TYPE="<< parameters.build_type >>"
export COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0) export COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0)
python3 -m tools.stats.upload_binary_size_to_scuba android python3 tools/stats/upload_binary_size_to_scuba.py android
############################################################################## ##############################################################################
# Binary build (nightlies nightly build) defaults # Binary build (nightlies nightly build) defaults
@ -551,7 +551,7 @@ jobs:
cd /pytorch && export COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0) cd /pytorch && export COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0)
python3 -mpip install requests && \ python3 -mpip install requests && \
SCRIBE_GRAPHQL_ACCESS_TOKEN=${SCRIBE_GRAPHQL_ACCESS_TOKEN} \ SCRIBE_GRAPHQL_ACCESS_TOKEN=${SCRIBE_GRAPHQL_ACCESS_TOKEN} \
python3 -m tools.stats.upload_binary_size_to_scuba || exit 0 python3 tools/stats/upload_binary_size_to_scuba.py || exit 0
- store_artifacts: - store_artifacts:
path: /home/circleci/project/dist path: /home/circleci/project/dist
@ -889,7 +889,7 @@ jobs:
cd /pytorch && export COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0) cd /pytorch && export COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0)
python3 -mpip install requests && \ python3 -mpip install requests && \
SCRIBE_GRAPHQL_ACCESS_TOKEN=${SCRIBE_GRAPHQL_ACCESS_TOKEN} \ SCRIBE_GRAPHQL_ACCESS_TOKEN=${SCRIBE_GRAPHQL_ACCESS_TOKEN} \
python3 -m tools.stats.upload_binary_size_to_scuba || exit 0 python3 /pytorch/tools/stats/upload_binary_size_to_scuba.py || exit 0
- persist_to_workspace: - persist_to_workspace:
root: / root: /
paths: final_pkgs paths: final_pkgs

2
.circleci/verbatim-sources/commands.yml
View File

@ -171,4 +171,4 @@ commands:
cd ~/project cd ~/project
export ANDROID_BUILD_TYPE="<< parameters.build_type >>" export ANDROID_BUILD_TYPE="<< parameters.build_type >>"
export COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0) export COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0)
python3 -m tools.stats.upload_binary_size_to_scuba android python3 tools/stats/upload_binary_size_to_scuba.py android

2
.circleci/verbatim-sources/job-specs/binary-job-specs.yml
View File

@ -29,7 +29,7 @@
cd /pytorch && export COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0) cd /pytorch && export COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0)
python3 -mpip install requests && \ python3 -mpip install requests && \
SCRIBE_GRAPHQL_ACCESS_TOKEN=${SCRIBE_GRAPHQL_ACCESS_TOKEN} \ SCRIBE_GRAPHQL_ACCESS_TOKEN=${SCRIBE_GRAPHQL_ACCESS_TOKEN} \
python3 -m tools.stats.upload_binary_size_to_scuba || exit 0 python3 /pytorch/tools/stats/upload_binary_size_to_scuba.py || exit 0
- persist_to_workspace: - persist_to_workspace:
root: / root: /
paths: final_pkgs paths: final_pkgs

2
.circleci/verbatim-sources/job-specs/pytorch-job-specs.yml
View File

@ -81,7 +81,7 @@ jobs:
cd /pytorch && export COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0) cd /pytorch && export COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0)
python3 -mpip install requests && \ python3 -mpip install requests && \
SCRIBE_GRAPHQL_ACCESS_TOKEN=${SCRIBE_GRAPHQL_ACCESS_TOKEN} \ SCRIBE_GRAPHQL_ACCESS_TOKEN=${SCRIBE_GRAPHQL_ACCESS_TOKEN} \
python3 -m tools.stats.upload_binary_size_to_scuba || exit 0 python3 tools/stats/upload_binary_size_to_scuba.py || exit 0
- store_artifacts: - store_artifacts:
path: /home/circleci/project/dist path: /home/circleci/project/dist

15
.github/templates/bazel_ci_workflow.yml.j2 vendored
View File

@ -74,7 +74,6 @@ name: Bazel Linux CI (!{{ build_environment }})
# temporary hack: set CIRCLE_* vars, until we update # temporary hack: set CIRCLE_* vars, until we update
# tools/stats/print_test_stats.py to natively support GitHub Actions # tools/stats/print_test_stats.py to natively support GitHub Actions
env: env:
AWS_DEFAULT_REGION: us-east-1
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }} CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }} CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
@ -150,7 +149,7 @@ name: Bazel Linux CI (!{{ build_environment }})
if: always() if: always()
needs: needs:
- build-and-test - build-and-test
runs-on: linux.2xlarge runs-on: ubuntu-18.04
steps: steps:
- name: Checkout PyTorch - name: Checkout PyTorch
uses: actions/checkout@v2 uses: actions/checkout@v2
@ -166,14 +165,17 @@ name: Bazel Linux CI (!{{ build_environment }})
run: | run: |
# Should preserve paths so reports should still be in test/test-reports # Should preserve paths so reports should still be in test/test-reports
unzip -o 'test-reports-*.zip' unzip -o 'test-reports-*.zip'
- uses: actions/setup-python@v2
with:
python-version: 3.9
- name: Install dependencies - name: Install dependencies
# boto3 version copied from .circleci/docker/common/install_conda.sh # boto3 version copied from .circleci/docker/common/install_conda.sh
run: | run: |
pip3 install -r requirements.txt pip install -r requirements.txt
pip3 install boto3==1.16.34 junitparser rich pip install boto3==1.16.34 junitparser rich
- name: Output Test Results (Click Me) - name: Output Test Results (Click Me)
run: | run: |
python3 tools/render_junit.py test python tools/render_junit.py test
- name: Parse ref - name: Parse ref
id: parse-ref id: parse-ref
run: .github/scripts/parse_ref.py run: .github/scripts/parse_ref.py
@ -184,7 +186,6 @@ name: Bazel Linux CI (!{{ build_environment }})
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: us-east-1
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }} CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
JOB_BASE_NAME: pytorch-linux-xenial-py3.6-gcc7-bazel-test-test JOB_BASE_NAME: pytorch-linux-xenial-py3.6-gcc7-bazel-test-test
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }} CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
@ -192,5 +193,5 @@ name: Bazel Linux CI (!{{ build_environment }})
CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }} CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}' CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
run: | run: |
python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
{%- endblock %} {%- endblock %}

15
.github/templates/linux_ci_workflow.yml.j2 vendored
View File

@ -161,7 +161,6 @@ jobs:
# temporary hack: set CIRCLE_* vars, until we update # temporary hack: set CIRCLE_* vars, until we update
# tools/stats/print_test_stats.py to natively support GitHub Actions # tools/stats/print_test_stats.py to natively support GitHub Actions
env: env:
AWS_DEFAULT_REGION: us-east-1
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }} CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }} CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
@ -383,7 +382,7 @@ jobs:
needs: needs:
- generate-test-matrix - generate-test-matrix
- test - test
runs-on: linux.2xlarge runs-on: ubuntu-18.04
strategy: strategy:
matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }} matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }}
fail-fast: false fail-fast: false
@ -402,14 +401,17 @@ jobs:
run: | run: |
# Should preserve paths so reports should still be in test/test-reports # Should preserve paths so reports should still be in test/test-reports
unzip -o 'test-reports-*.zip' unzip -o 'test-reports-*.zip'
- uses: actions/setup-python@v2
with:
python-version: 3.9
- name: Install dependencies - name: Install dependencies
# boto3 version copied from .circleci/docker/common/install_conda.sh # boto3 version copied from .circleci/docker/common/install_conda.sh
run: | run: |
pip3 install -r requirements.txt pip install -r requirements.txt
pip3 install boto3==1.16.34 junitparser rich pip install boto3==1.16.34 junitparser rich
- name: Output Test Results (Click Me) - name: Output Test Results (Click Me)
run: | run: |
python3 tools/render_junit.py test python tools/render_junit.py test
- name: Parse ref - name: Parse ref
id: parse-ref id: parse-ref
run: .github/scripts/parse_ref.py run: .github/scripts/parse_ref.py
@ -420,7 +422,6 @@ jobs:
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: us-east-1
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }} CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
JOB_BASE_NAME: !{{ build_environment }}-test JOB_BASE_NAME: !{{ build_environment }}-test
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }} CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
@ -428,7 +429,7 @@ jobs:
CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }} CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}' CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
run: | run: |
python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
{%- endblock %} {%- endblock %}
{%- if enable_doc_jobs %} {%- if enable_doc_jobs %}

14
.github/templates/windows_ci_workflow.yml.j2 vendored
View File

@ -216,7 +216,7 @@ jobs:
needs: needs:
- generate-test-matrix - generate-test-matrix
- test - test
runs-on: linux.2xlarge runs-on: ubuntu-18.04
strategy: strategy:
matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }} matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }}
fail-fast: false fail-fast: false
@ -235,14 +235,17 @@ jobs:
- name: Unzip test reports - name: Unzip test reports
run: | run: |
unzip -o 'test-reports-*.zip' unzip -o 'test-reports-*.zip'
- uses: actions/setup-python@v2
with:
python-version: 3.9
- name: Install dependencies - name: Install dependencies
# boto3 version copied from .circleci/docker/common/install_conda.sh # boto3 version copied from .circleci/docker/common/install_conda.sh
run: | run: |
pip3 install -r requirements.txt pip install -r requirements.txt
pip3 install boto3==1.16.34 junitparser rich pip install boto3==1.16.34 junitparser rich
- name: Output Test Results (Click Me) - name: Output Test Results (Click Me)
run: | run: |
python3 tools/render_junit.py test python tools/render_junit.py test
- name: Parse ref - name: Parse ref
id: parse-ref id: parse-ref
run: .github/scripts/parse_ref.py run: .github/scripts/parse_ref.py
@ -253,7 +256,6 @@ jobs:
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: us-east-1
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }} CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
JOB_BASE_NAME: !{{ build_environment }}-test JOB_BASE_NAME: !{{ build_environment }}-test
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }} CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
@ -261,4 +263,4 @@ jobs:
CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }} CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}' CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
run: | run: |
python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test

2
.github/workflows/build_linux_conda.yml vendored
View File

@ -105,7 +105,7 @@ jobs:
COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0) COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0)
export COMMIT_TIME export COMMIT_TIME
pip3 install requests pip3 install requests
python3 -m tools.stats.upload_binary_size_to_scuba || exit 0 python3 tools.stats.upload_binary_size_to_scuba || exit 0
concurrency: concurrency:
group: build-linux-conda-${{ github.event.pull_request.number || github.sha }} group: build-linux-conda-${{ github.event.pull_request.number || github.sha }}

14
.github/workflows/periodic-pytorch-win-vs2019-cuda11-cudnn8-py3.yml vendored
View File

@ -190,7 +190,7 @@ jobs:
needs: needs:
- generate-test-matrix - generate-test-matrix
- test - test
runs-on: linux.2xlarge runs-on: ubuntu-18.04
strategy: strategy:
matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }} matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }}
fail-fast: false fail-fast: false
@ -209,14 +209,17 @@ jobs:
- name: Unzip test reports - name: Unzip test reports
run: | run: |
unzip -o 'test-reports-*.zip' unzip -o 'test-reports-*.zip'
- uses: actions/setup-python@v2
with:
python-version: 3.9
- name: Install dependencies - name: Install dependencies
# boto3 version copied from .circleci/docker/common/install_conda.sh # boto3 version copied from .circleci/docker/common/install_conda.sh
run: | run: |
pip3 install -r requirements.txt pip install -r requirements.txt
pip3 install boto3==1.16.34 junitparser rich pip install boto3==1.16.34 junitparser rich
- name: Output Test Results (Click Me) - name: Output Test Results (Click Me)
run: | run: |
python3 tools/render_junit.py test python tools/render_junit.py test
- name: Parse ref - name: Parse ref
id: parse-ref id: parse-ref
run: .github/scripts/parse_ref.py run: .github/scripts/parse_ref.py
@ -227,7 +230,6 @@ jobs:
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: us-east-1
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }} CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
JOB_BASE_NAME: periodic-pytorch-win-vs2019-cuda11-cudnn8-py3-test JOB_BASE_NAME: periodic-pytorch-win-vs2019-cuda11-cudnn8-py3-test
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }} CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
@ -235,4 +237,4 @@ jobs:
CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }} CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}' CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
run: | run: |
python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test

15
.github/workflows/pytorch-linux-bionic-cuda10.2-cudnn7-py3.9-gcc7.yml vendored
View File

@ -151,7 +151,6 @@ jobs:
# temporary hack: set CIRCLE_* vars, until we update # temporary hack: set CIRCLE_* vars, until we update
# tools/stats/print_test_stats.py to natively support GitHub Actions # tools/stats/print_test_stats.py to natively support GitHub Actions
env: env:
AWS_DEFAULT_REGION: us-east-1
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }} CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }} CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
@ -369,7 +368,7 @@ jobs:
needs: needs:
- generate-test-matrix - generate-test-matrix
- test - test
runs-on: linux.2xlarge runs-on: ubuntu-18.04
strategy: strategy:
matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }} matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }}
fail-fast: false fail-fast: false
@ -388,14 +387,17 @@ jobs:
run: | run: |
# Should preserve paths so reports should still be in test/test-reports # Should preserve paths so reports should still be in test/test-reports
unzip -o 'test-reports-*.zip' unzip -o 'test-reports-*.zip'
- uses: actions/setup-python@v2
with:
python-version: 3.9
- name: Install dependencies - name: Install dependencies
# boto3 version copied from .circleci/docker/common/install_conda.sh # boto3 version copied from .circleci/docker/common/install_conda.sh
run: | run: |
pip3 install -r requirements.txt pip install -r requirements.txt
pip3 install boto3==1.16.34 junitparser rich pip install boto3==1.16.34 junitparser rich
- name: Output Test Results (Click Me) - name: Output Test Results (Click Me)
run: | run: |
python3 tools/render_junit.py test python tools/render_junit.py test
- name: Parse ref - name: Parse ref
id: parse-ref id: parse-ref
run: .github/scripts/parse_ref.py run: .github/scripts/parse_ref.py
@ -406,7 +408,6 @@ jobs:
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: us-east-1
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }} CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
JOB_BASE_NAME: pytorch-linux-bionic-cuda10.2-cudnn7-py3.9-gcc7-test JOB_BASE_NAME: pytorch-linux-bionic-cuda10.2-cudnn7-py3.9-gcc7-test
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }} CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
@ -414,4 +415,4 @@ jobs:
CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }} CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}' CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
run: | run: |
python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test

15
.github/workflows/pytorch-linux-bionic-py3.8-gcc9-coverage.yml vendored
View File

@ -152,7 +152,6 @@ jobs:
# temporary hack: set CIRCLE_* vars, until we update # temporary hack: set CIRCLE_* vars, until we update
# tools/stats/print_test_stats.py to natively support GitHub Actions # tools/stats/print_test_stats.py to natively support GitHub Actions
env: env:
AWS_DEFAULT_REGION: us-east-1
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }} CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }} CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
@ -370,7 +369,7 @@ jobs:
needs: needs:
- generate-test-matrix - generate-test-matrix
- test - test
runs-on: linux.2xlarge runs-on: ubuntu-18.04
strategy: strategy:
matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }} matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }}
fail-fast: false fail-fast: false
@ -389,14 +388,17 @@ jobs:
run: | run: |
# Should preserve paths so reports should still be in test/test-reports # Should preserve paths so reports should still be in test/test-reports
unzip -o 'test-reports-*.zip' unzip -o 'test-reports-*.zip'
- uses: actions/setup-python@v2
with:
python-version: 3.9
- name: Install dependencies - name: Install dependencies
# boto3 version copied from .circleci/docker/common/install_conda.sh # boto3 version copied from .circleci/docker/common/install_conda.sh
run: | run: |
pip3 install -r requirements.txt pip install -r requirements.txt
pip3 install boto3==1.16.34 junitparser rich pip install boto3==1.16.34 junitparser rich
- name: Output Test Results (Click Me) - name: Output Test Results (Click Me)
run: | run: |
python3 tools/render_junit.py test python tools/render_junit.py test
- name: Parse ref - name: Parse ref
id: parse-ref id: parse-ref
run: .github/scripts/parse_ref.py run: .github/scripts/parse_ref.py
@ -407,7 +409,6 @@ jobs:
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: us-east-1
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }} CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
JOB_BASE_NAME: pytorch-linux-bionic-py3.8-gcc9-coverage-test JOB_BASE_NAME: pytorch-linux-bionic-py3.8-gcc9-coverage-test
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }} CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
@ -415,4 +416,4 @@ jobs:
CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }} CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}' CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
run: | run: |
python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test

15
.github/workflows/pytorch-linux-xenial-cuda10.2-cudnn7-py3.6-gcc7.yml vendored
View File

@ -151,7 +151,6 @@ jobs:
# temporary hack: set CIRCLE_* vars, until we update # temporary hack: set CIRCLE_* vars, until we update
# tools/stats/print_test_stats.py to natively support GitHub Actions # tools/stats/print_test_stats.py to natively support GitHub Actions
env: env:
AWS_DEFAULT_REGION: us-east-1
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }} CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }} CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
@ -369,7 +368,7 @@ jobs:
needs: needs:
- generate-test-matrix - generate-test-matrix
- test - test
runs-on: linux.2xlarge runs-on: ubuntu-18.04
strategy: strategy:
matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }} matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }}
fail-fast: false fail-fast: false
@ -388,14 +387,17 @@ jobs:
run: | run: |
# Should preserve paths so reports should still be in test/test-reports # Should preserve paths so reports should still be in test/test-reports
unzip -o 'test-reports-*.zip' unzip -o 'test-reports-*.zip'
- uses: actions/setup-python@v2
with:
python-version: 3.9
- name: Install dependencies - name: Install dependencies
# boto3 version copied from .circleci/docker/common/install_conda.sh # boto3 version copied from .circleci/docker/common/install_conda.sh
run: | run: |
pip3 install -r requirements.txt pip install -r requirements.txt
pip3 install boto3==1.16.34 junitparser rich pip install boto3==1.16.34 junitparser rich
- name: Output Test Results (Click Me) - name: Output Test Results (Click Me)
run: | run: |
python3 tools/render_junit.py test python tools/render_junit.py test
- name: Parse ref - name: Parse ref
id: parse-ref id: parse-ref
run: .github/scripts/parse_ref.py run: .github/scripts/parse_ref.py
@ -406,7 +408,6 @@ jobs:
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: us-east-1
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }} CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
JOB_BASE_NAME: pytorch-linux-xenial-cuda10.2-cudnn7-py3.6-gcc7-test JOB_BASE_NAME: pytorch-linux-xenial-cuda10.2-cudnn7-py3.6-gcc7-test
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }} CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
@ -414,4 +415,4 @@ jobs:
CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }} CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}' CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
run: | run: |
python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test

15
.github/workflows/pytorch-linux-xenial-cuda11.1-cudnn8-py3.6-gcc7.yml vendored
View File

@ -151,7 +151,6 @@ jobs:
# temporary hack: set CIRCLE_* vars, until we update # temporary hack: set CIRCLE_* vars, until we update
# tools/stats/print_test_stats.py to natively support GitHub Actions # tools/stats/print_test_stats.py to natively support GitHub Actions
env: env:
AWS_DEFAULT_REGION: us-east-1
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }} CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }} CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
@ -369,7 +368,7 @@ jobs:
needs: needs:
- generate-test-matrix - generate-test-matrix
- test - test
runs-on: linux.2xlarge runs-on: ubuntu-18.04
strategy: strategy:
matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }} matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }}
fail-fast: false fail-fast: false
@ -388,14 +387,17 @@ jobs:
run: | run: |
# Should preserve paths so reports should still be in test/test-reports # Should preserve paths so reports should still be in test/test-reports
unzip -o 'test-reports-*.zip' unzip -o 'test-reports-*.zip'
- uses: actions/setup-python@v2
with:
python-version: 3.9
- name: Install dependencies - name: Install dependencies
# boto3 version copied from .circleci/docker/common/install_conda.sh # boto3 version copied from .circleci/docker/common/install_conda.sh
run: | run: |
pip3 install -r requirements.txt pip install -r requirements.txt
pip3 install boto3==1.16.34 junitparser rich pip install boto3==1.16.34 junitparser rich
- name: Output Test Results (Click Me) - name: Output Test Results (Click Me)
run: | run: |
python3 tools/render_junit.py test python tools/render_junit.py test
- name: Parse ref - name: Parse ref
id: parse-ref id: parse-ref
run: .github/scripts/parse_ref.py run: .github/scripts/parse_ref.py
@ -406,7 +408,6 @@ jobs:
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: us-east-1
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }} CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
JOB_BASE_NAME: pytorch-linux-xenial-cuda11.1-cudnn8-py3.6-gcc7-test JOB_BASE_NAME: pytorch-linux-xenial-cuda11.1-cudnn8-py3.6-gcc7-test
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }} CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
@ -414,4 +415,4 @@ jobs:
CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }} CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}' CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
run: | run: |
python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test

15
.github/workflows/pytorch-linux-xenial-py3.6-gcc5.4.yml vendored
View File

@ -152,7 +152,6 @@ jobs:
# temporary hack: set CIRCLE_* vars, until we update # temporary hack: set CIRCLE_* vars, until we update
# tools/stats/print_test_stats.py to natively support GitHub Actions # tools/stats/print_test_stats.py to natively support GitHub Actions
env: env:
AWS_DEFAULT_REGION: us-east-1
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }} CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }} CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
@ -370,7 +369,7 @@ jobs:
needs: needs:
- generate-test-matrix - generate-test-matrix
- test - test
runs-on: linux.2xlarge runs-on: ubuntu-18.04
strategy: strategy:
matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }} matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }}
fail-fast: false fail-fast: false
@ -389,14 +388,17 @@ jobs:
run: | run: |
# Should preserve paths so reports should still be in test/test-reports # Should preserve paths so reports should still be in test/test-reports
unzip -o 'test-reports-*.zip' unzip -o 'test-reports-*.zip'
- uses: actions/setup-python@v2
with:
python-version: 3.9
- name: Install dependencies - name: Install dependencies
# boto3 version copied from .circleci/docker/common/install_conda.sh # boto3 version copied from .circleci/docker/common/install_conda.sh
run: | run: |
pip3 install -r requirements.txt pip install -r requirements.txt
pip3 install boto3==1.16.34 junitparser rich pip install boto3==1.16.34 junitparser rich
- name: Output Test Results (Click Me) - name: Output Test Results (Click Me)
run: | run: |
python3 tools/render_junit.py test python tools/render_junit.py test
- name: Parse ref - name: Parse ref
id: parse-ref id: parse-ref
run: .github/scripts/parse_ref.py run: .github/scripts/parse_ref.py
@ -407,7 +409,6 @@ jobs:
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: us-east-1
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }} CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
JOB_BASE_NAME: pytorch-linux-xenial-py3.6-gcc5.4-test JOB_BASE_NAME: pytorch-linux-xenial-py3.6-gcc5.4-test
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }} CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
@ -415,7 +416,7 @@ jobs:
CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }} CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}' CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
run: | run: |
python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
pytorch_python_doc_build: pytorch_python_doc_build:
runs-on: linux.2xlarge runs-on: linux.2xlarge

15
.github/workflows/pytorch-linux-xenial-py3.6-gcc7-bazel-test.yml vendored
View File

@ -166,7 +166,6 @@ jobs:
# temporary hack: set CIRCLE_* vars, until we update # temporary hack: set CIRCLE_* vars, until we update
# tools/stats/print_test_stats.py to natively support GitHub Actions # tools/stats/print_test_stats.py to natively support GitHub Actions
env: env:
AWS_DEFAULT_REGION: us-east-1
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }} CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }} CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
@ -240,7 +239,7 @@ jobs:
if: always() if: always()
needs: needs:
- build-and-test - build-and-test
runs-on: linux.2xlarge runs-on: ubuntu-18.04
steps: steps:
- name: Checkout PyTorch - name: Checkout PyTorch
uses: actions/checkout@v2 uses: actions/checkout@v2
@ -256,14 +255,17 @@ jobs:
run: | run: |
# Should preserve paths so reports should still be in test/test-reports # Should preserve paths so reports should still be in test/test-reports
unzip -o 'test-reports-*.zip' unzip -o 'test-reports-*.zip'
- uses: actions/setup-python@v2
with:
python-version: 3.9
- name: Install dependencies - name: Install dependencies
# boto3 version copied from .circleci/docker/common/install_conda.sh # boto3 version copied from .circleci/docker/common/install_conda.sh
run: | run: |
pip3 install -r requirements.txt pip install -r requirements.txt
pip3 install boto3==1.16.34 junitparser rich pip install boto3==1.16.34 junitparser rich
- name: Output Test Results (Click Me) - name: Output Test Results (Click Me)
run: | run: |
python3 tools/render_junit.py test python tools/render_junit.py test
- name: Parse ref - name: Parse ref
id: parse-ref id: parse-ref
run: .github/scripts/parse_ref.py run: .github/scripts/parse_ref.py
@ -274,7 +276,6 @@ jobs:
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: us-east-1
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }} CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
JOB_BASE_NAME: pytorch-linux-xenial-py3.6-gcc7-bazel-test-test JOB_BASE_NAME: pytorch-linux-xenial-py3.6-gcc7-bazel-test-test
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }} CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
@ -282,4 +283,4 @@ jobs:
CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }} CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}' CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
run: | run: |
python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test

14
.github/workflows/pytorch-win-vs2019-cpu-py3.yml vendored
View File

@ -175,7 +175,7 @@ jobs:
needs: needs:
- generate-test-matrix - generate-test-matrix
- test - test
runs-on: linux.2xlarge runs-on: ubuntu-18.04
strategy: strategy:
matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }} matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }}
fail-fast: false fail-fast: false
@ -194,14 +194,17 @@ jobs:
- name: Unzip test reports - name: Unzip test reports
run: | run: |
unzip -o 'test-reports-*.zip' unzip -o 'test-reports-*.zip'
- uses: actions/setup-python@v2
with:
python-version: 3.9
- name: Install dependencies - name: Install dependencies
# boto3 version copied from .circleci/docker/common/install_conda.sh # boto3 version copied from .circleci/docker/common/install_conda.sh
run: | run: |
pip3 install -r requirements.txt pip install -r requirements.txt
pip3 install boto3==1.16.34 junitparser rich pip install boto3==1.16.34 junitparser rich
- name: Output Test Results (Click Me) - name: Output Test Results (Click Me)
run: | run: |
python3 tools/render_junit.py test python tools/render_junit.py test
- name: Parse ref - name: Parse ref
id: parse-ref id: parse-ref
run: .github/scripts/parse_ref.py run: .github/scripts/parse_ref.py
@ -212,7 +215,6 @@ jobs:
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: us-east-1
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }} CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
JOB_BASE_NAME: pytorch-win-vs2019-cpu-py3-test JOB_BASE_NAME: pytorch-win-vs2019-cpu-py3-test
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }} CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
@ -220,4 +222,4 @@ jobs:
CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }} CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}' CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
run: | run: |
python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test

14
.github/workflows/pytorch-win-vs2019-cuda10-cudnn7-py3.yml vendored
View File

@ -193,7 +193,7 @@ jobs:
needs: needs:
- generate-test-matrix - generate-test-matrix
- test - test
runs-on: linux.2xlarge runs-on: ubuntu-18.04
strategy: strategy:
matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }} matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }}
fail-fast: false fail-fast: false
@ -212,14 +212,17 @@ jobs:
- name: Unzip test reports - name: Unzip test reports
run: | run: |
unzip -o 'test-reports-*.zip' unzip -o 'test-reports-*.zip'
- uses: actions/setup-python@v2
with:
python-version: 3.9
- name: Install dependencies - name: Install dependencies
# boto3 version copied from .circleci/docker/common/install_conda.sh # boto3 version copied from .circleci/docker/common/install_conda.sh
run: | run: |
pip3 install -r requirements.txt pip install -r requirements.txt
pip3 install boto3==1.16.34 junitparser rich pip install boto3==1.16.34 junitparser rich
- name: Output Test Results (Click Me) - name: Output Test Results (Click Me)
run: | run: |
python3 tools/render_junit.py test python tools/render_junit.py test
- name: Parse ref - name: Parse ref
id: parse-ref id: parse-ref
run: .github/scripts/parse_ref.py run: .github/scripts/parse_ref.py
@ -230,7 +233,6 @@ jobs:
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: us-east-1
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }} CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
JOB_BASE_NAME: pytorch-win-vs2019-cuda10-cudnn7-py3-test JOB_BASE_NAME: pytorch-win-vs2019-cuda10-cudnn7-py3-test
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }} CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
@ -238,4 +240,4 @@ jobs:
CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }} CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}' CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
run: | run: |
python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test

14
.github/workflows/pytorch-win-vs2019-cuda11-cudnn8-py3.yml vendored
View File

@ -192,7 +192,7 @@ jobs:
needs: needs:
- generate-test-matrix - generate-test-matrix
- test - test
runs-on: linux.2xlarge runs-on: ubuntu-18.04
strategy: strategy:
matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }} matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }}
fail-fast: false fail-fast: false
@ -211,14 +211,17 @@ jobs:
- name: Unzip test reports - name: Unzip test reports
run: | run: |
unzip -o 'test-reports-*.zip' unzip -o 'test-reports-*.zip'
- uses: actions/setup-python@v2
with:
python-version: 3.9
- name: Install dependencies - name: Install dependencies
# boto3 version copied from .circleci/docker/common/install_conda.sh # boto3 version copied from .circleci/docker/common/install_conda.sh
run: | run: |
pip3 install -r requirements.txt pip install -r requirements.txt
pip3 install boto3==1.16.34 junitparser rich pip install boto3==1.16.34 junitparser rich
- name: Output Test Results (Click Me) - name: Output Test Results (Click Me)
run: | run: |
python3 tools/render_junit.py test python tools/render_junit.py test
- name: Parse ref - name: Parse ref
id: parse-ref id: parse-ref
run: .github/scripts/parse_ref.py run: .github/scripts/parse_ref.py
@ -229,7 +232,6 @@ jobs:
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: us-east-1
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }} CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
JOB_BASE_NAME: pytorch-win-vs2019-cuda11-cudnn8-py3-test JOB_BASE_NAME: pytorch-win-vs2019-cuda11-cudnn8-py3-test
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }} CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
@ -237,4 +239,4 @@ jobs:
CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }} CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}' CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
run: | run: |
python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test

36
benchmarks/upload_scribe.py
View File

@ -10,9 +10,9 @@ import argparse
import time import time
import json import json
import os import os
import requests
import subprocess import subprocess
from collections import defaultdict from collections import defaultdict
from tools.stats.scribe import send_to_scribe
class ScribeUploader: class ScribeUploader:
@ -30,6 +30,7 @@ class ScribeUploader:
elif field in self.schema['float']: elif field in self.schema['float']:
message['float'][field] = float(value) message['float'][field] = float(value)
else: else:
raise ValueError("Field {} is not currently used, " raise ValueError("Field {} is not currently used, "
"be intentional about adding new fields".format(field)) "be intentional about adding new fields".format(field))
return message return message
@ -43,19 +44,28 @@ class ScribeUploader:
def upload(self, messages): def upload(self, messages):
if os.environ.get('SCRIBE_INTERN'): if os.environ.get('SCRIBE_INTERN'):
return self._upload_intern(messages) return self._upload_intern(messages)
logs = json.dumps( access_token = os.environ.get("SCRIBE_GRAPHQL_ACCESS_TOKEN")
[ if not access_token:
{ raise ValueError("Can't find access token from environment variable")
"category": self.category, url = "https://graph.facebook.com/scribe_logs"
"message": json.dumps(message), r = requests.post(
"line_escape": False, url,
} data={
for message in messages "access_token": access_token,
] "logs": json.dumps(
[
{
"category": self.category,
"message": json.dumps(message),
"line_escape": False,
}
for message in messages
]
),
},
) )
res = send_to_scribe(logs) print(r.text)
print(res) r.raise_for_status()
class PytorchBenchmarkUploader(ScribeUploader): class PytorchBenchmarkUploader(ScribeUploader):
def __init__(self): def __init__(self):

52
tools/stats/print_test_stats.py
View File

@ -16,11 +16,12 @@ from typing import (Any, DefaultDict, Dict, Iterable, Iterator, List, Optional,
Set, Tuple, cast) Set, Tuple, cast)
from xml.dom import minidom from xml.dom import minidom
import requests
from typing_extensions import TypedDict from typing_extensions import TypedDict
from tools.stats.s3_stat_parser import (newify_case, get_S3_object_from_bucket, get_test_stats_summaries_for_job, from tools.stats.s3_stat_parser import (newify_case, get_S3_object_from_bucket, get_test_stats_summaries_for_job,
Report, Status, Commit, HAVE_BOTO3, Version2Case, VersionedReport, Report, Status, Commit, HAVE_BOTO3, Version2Case, VersionedReport,
Version1Report, Version2Report, ReportMetaMeta) Version1Report, Version2Report, ReportMetaMeta)
from tools.stats.scribe import send_to_scribe
SimplerSuite = Dict[str, Version2Case] SimplerSuite = Dict[str, Version2Case]
@ -609,8 +610,7 @@ class TestSuite:
sorted_tests = sorted(self.test_cases.values(), key=lambda x: x.time) sorted_tests = sorted(self.test_cases.values(), key=lambda x: x.time)
test_count = len(sorted_tests) test_count = len(sorted_tests)
print(f"class {self.name}:") print(f"class {self.name}:")
print( print(f" tests: {test_count} failed: {self.failed_count} skipped: {self.skipped_count} errored: {self.errored_count}")
f" tests: {test_count} failed: {self.failed_count} skipped: {self.skipped_count} errored: {self.errored_count}")
print(f" run_time: {self.total_time:.2f} seconds") print(f" run_time: {self.total_time:.2f} seconds")
print(f" avg_time: {self.total_time/test_count:.2f} seconds") print(f" avg_time: {self.total_time/test_count:.2f} seconds")
if test_count >= 2: if test_count >= 2:
@ -645,8 +645,7 @@ class TestFile:
self.test_suites[suite_name].update(test_case) self.test_suites[suite_name].update(test_case)
self.total_time += test_case.time self.total_time += test_case.time
else: else:
raise RuntimeWarning( raise RuntimeWarning(f'Duplicate test case {test_case.name} in suite {suite_name} called from {self.name}')
f'Duplicate test case {test_case.name} in suite {suite_name} called from {self.name}')
else: else:
self.test_suites[suite_name].append(test_case) self.test_suites[suite_name].append(test_case)
self.total_time += test_case.time self.total_time += test_case.time
@ -730,21 +729,33 @@ def build_message(
def send_report_to_scribe(reports: Dict[str, TestFile]) -> None: def send_report_to_scribe(reports: Dict[str, TestFile]) -> None:
access_token = os.environ.get("SCRIBE_GRAPHQL_ACCESS_TOKEN")
if not access_token:
print("No scribe access token provided, skip sending report!")
return
print("Scribe access token provided, sending report...")
url = "https://graph.facebook.com/scribe_logs"
meta_info = build_info() meta_info = build_info()
logs = json.dumps( r = requests.post(
[ url,
{ data={
"category": "perfpipe_pytorch_test_times", "access_token": access_token,
"message": json.dumps(build_message(test_file, test_suite, test_case, meta_info)), "logs": json.dumps(
"line_escape": False, [
} {
for test_file in reports.values() "category": "perfpipe_pytorch_test_times",
for test_suite in test_file.test_suites.values() "message": json.dumps(build_message(test_file, test_suite, test_case, meta_info)),
for test_case in test_suite.test_cases.values() "line_escape": False,
] }
for test_file in reports.values()
for test_suite in test_file.test_suites.values()
for test_case in test_suite.test_cases.values()
]
),
},
) )
res = send_to_scribe(logs) r.raise_for_status()
print(res)
def assemble_s3_object( def assemble_s3_object(
@ -756,7 +767,7 @@ def assemble_s3_object(
**build_info(), # type: ignore[misc] **build_info(), # type: ignore[misc]
'total_seconds': total_seconds, 'total_seconds': total_seconds,
'format_version': 2, 'format_version': 2,
'files': { 'files' : {
name: { name: {
'total_seconds': test_file.total_time, 'total_seconds': test_file.total_time,
'suites': { 'suites': {
@ -869,7 +880,6 @@ def reports_has_no_tests(reports: Dict[str, TestFile]) -> bool:
return False return False
return True return True
if __name__ == '__main__': if __name__ == '__main__':
import argparse import argparse
import sys import sys
@ -938,7 +948,7 @@ if __name__ == '__main__':
print(f"ERROR ENCOUNTERED WHEN UPLOADING TO SCRIBE: {e}") print(f"ERROR ENCOUNTERED WHEN UPLOADING TO SCRIBE: {e}")
# longest_tests can contain duplicates as the same tests can be spawned from different files # longest_tests can contain duplicates as the same tests can be spawned from different files
longest_tests: List[TestCase] = [] longest_tests : List[TestCase] = []
total_time = 0.0 total_time = 0.0
for filename, test_filename in reports_by_file.items(): for filename, test_filename in reports_by_file.items():
for suite_name, test_suite in test_filename.test_suites.items(): for suite_name, test_suite in test_filename.test_suites.items():

42
tools/stats/scribe.py
View File

@ -1,42 +0,0 @@
import base64
import bz2
import os
import json
def send_to_scribe(logs: str) -> str:
access_token = os.environ.get("SCRIBE_GRAPHQL_ACCESS_TOKEN", "")
# boto3 can be used when the runner has IAM roles setup
# currently it's used as a fallback when SCRIBE_GRAPHQL_ACCESS_TOKEN is empty
if access_token == "":
return _send_to_scribe_via_boto3(logs)
return _send_to_scribe_via_http(access_token, logs)
def _send_to_scribe_via_boto3(logs: str) -> str:
# lazy import so that we don't need to introduce extra dependencies
import boto3 # type: ignore[import]
print("Scribe access token not provided, sending report via boto3...")
event = {"base64_bz2_logs": base64.b64encode(bz2.compress(logs.encode())).decode()}
client = boto3.client("lambda")
res = client.invoke(FunctionName='gh-ci-scribe-proxy', Payload=json.dumps(event).encode())
payload = str(res['Payload'].read().decode())
if res['FunctionError']:
raise Exception(payload)
return payload
def _send_to_scribe_via_http(access_token: str, logs: str) -> str:
# lazy import so that we don't need to introduce extra dependencies
import requests # type: ignore[import]
print("Scribe access token provided, sending report via http...")
r = requests.post(
"https://graph.facebook.com/scribe_logs",
data={"access_token": access_token, "logs": logs},
)
r.raise_for_status()
return str(r.text)

35
tools/stats/upload_binary_size_to_scuba.py
View File

@ -9,8 +9,9 @@ import sys
import time import time
import zipfile import zipfile
import requests
from typing import Any, Dict, Generator, List from typing import Any, Dict, Generator, List
from tools.stats.scribe import send_to_scribe
def get_size(file_dir: str) -> int: def get_size(file_dir: str) -> int:
@ -51,18 +52,28 @@ def build_message(size: int) -> Dict[str, Any]:
def send_message(messages: List[Dict[str, Any]]) -> None: def send_message(messages: List[Dict[str, Any]]) -> None:
logs = json.dumps( access_token = os.environ.get("SCRIBE_GRAPHQL_ACCESS_TOKEN")
[ if not access_token:
{ raise ValueError("Can't find access token from environment variable")
"category": "perfpipe_pytorch_binary_size", url = "https://graph.facebook.com/scribe_logs"
"message": json.dumps(message), r = requests.post(
"line_escape": False, url,
} data={
for message in messages "access_token": access_token,
] "logs": json.dumps(
[
{
"category": "perfpipe_pytorch_binary_size",
"message": json.dumps(message),
"line_escape": False,
}
for message in messages
]
),
},
) )
res = send_to_scribe(logs) print(r.text)
print(res) r.raise_for_status()
def report_android_sizes(file_dir: str) -> None: def report_android_sizes(file_dir: str) -> None: