From 23fcc409d533cb8b5b6b2696be976e2eaf875eda Mon Sep 17 00:00:00 2001 From: Michael Suo Date: Fri, 15 Nov 2019 10:57:59 -0800 Subject: [PATCH] Revert "switch back to azure pipelines" (#29910) Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/29910 This reverts commit 6de1016f9dbf624f93f8c8d45feb56f8c222b7a6. Test Plan: Imported from OSS Differential Revision: D18532474 Pulled By: suo fbshipit-source-id: 852fdcf21bd4aa7ca94322d64e43aab5a822cabc --- .github/workflows/lint.yml | 210 ++++++++++++++++++++++++++++++++++ azure-pipelines.yml | 116 ------------------- tools/run-clang-tidy-in-ci.sh | 48 -------- 3 files changed, 210 insertions(+), 164 deletions(-) create mode 100644 .github/workflows/lint.yml delete mode 100644 azure-pipelines.yml delete mode 100755 tools/run-clang-tidy-in-ci.sh diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000000..cb20246c6a6 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,210 @@ +name: Lint + +on: + push: + branches: + - master + pull_request: + +jobs: + quick-checks: + runs-on: ubuntu-latest + steps: + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.x + architecture: x64 + - name: Checkout PyTorch + uses: actions/checkout@master + - name: Ensure consistent CircleCI YAML config + run: | + pip install -r requirements.txt + cd .circleci && ./ensure-consistency.py + - name: Ensure Docker version is correctly deployed + run: .circleci/validate-docker-version.py + - name: Shellcheck Jenkins scripts + run: | + sudo apt-get install -y shellcheck + .jenkins/run-shellcheck.sh + - name: Ensure no tabs + run: | + (! git grep -I -l $'\t' -- . ':(exclude)*.svg' ':(exclude)**Makefile' ':(exclude)**/contrib/**' ':(exclude)third_party' ':(exclude).gitattributes' ':(exclude).gitmodules' || (echo "The above files have tabs; please convert them to spaces"; false)) + - name: Ensure C++ source files are not executable + run: | + (! find . \( -path ./third_party -o -path ./.git -o -path ./torch/bin -o -path ./build \) -prune -o -type f -executable -regextype posix-egrep -not -regex '.+(\.(bash|sh|py|so)|git-pre-commit)$' -print | grep . || (echo 'The above files have executable permission; please remove their executable permission by using `chmod -x`'; false)) + - name: MyPy typecheck + run: | + pip install mypy mypy-extensions + mypy @mypy-files.txt + - name: C++ docs check + run: | + sudo apt-get install -y doxygen && pip install -r requirements.txt + cd docs/cpp/source && ./check-doxygen.sh + + flake8-py3: + runs-on: ubuntu-latest + steps: + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.x + architecture: x64 + - name: Fetch PyTorch + uses: actions/checkout@master + - name: Checkout PR tip + run: | + set -eux + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + # We are on a PR, so actions/checkout leaves us on a merge commit. + # Check out the actual tip of the branch. + git checkout ${{ github.event.pull_request.head.sha }} + fi + echo ::set-output name=commit_sha::$(git rev-parse HEAD) + id: get_pr_tip + - name: Run flake8 + run: | + set -eux + pip install flake8 + flake8 --exit-zero > ${GITHUB_WORKSPACE}/flake8-output.txt + cat ${GITHUB_WORKSPACE}/flake8-output.txt + - name: Add annotations + uses: pytorch/add-annotations-github-action@master + with: + check_name: 'flake8-py3' + linter_output_path: 'flake8-output.txt' + commit_sha: ${{ steps.get_pr_tip.outputs.commit_sha }} + regex: '^(?.*?):(?\d+):(?\d+): (?\w\d+) (?.*)' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + flake8-py2: + runs-on: ubuntu-latest + steps: + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 2.x + architecture: x64 + - name: Fetch PyTorch + uses: actions/checkout@master + - name: Checkout PR tip + run: | + set -eux + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + # We are on a PR, so actions/checkout leaves us on a merge commit. + # Check out the actual tip of the branch. + git checkout ${{ github.event.pull_request.head.sha }} + fi + echo ::set-output name=commit_sha::$(git rev-parse HEAD) + id: get_pr_tip + - name: Run flake8 + run: | + set -eux + pip install flake8 + rm -rf .circleci + flake8 --exit-zero > ${GITHUB_WORKSPACE}/flake8-output.txt + cat ${GITHUB_WORKSPACE}/flake8-output.txt + - name: Add annotations + uses: pytorch/add-annotations-github-action@master + with: + check_name: 'flake8-py2' + linter_output_path: 'flake8-output.txt' + commit_sha: ${{ steps.get_pr_tip.outputs.commit_sha }} + regex: '^(?.*?):(?\d+):(?\d+): (?\w\d+) (?.*)' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + + clang-tidy: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.x + architecture: x64 + - name: Checkout PyTorch + uses: actions/checkout@master + - name: Checkout PR tip + run: | + set -eux + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + # We are on a PR, so actions/checkout leaves us on a merge commit. + # Check out the actual tip of the branch. + git checkout ${{ github.event.pull_request.head.sha }} + fi + echo ::set-output name=commit_sha::$(git rev-parse HEAD) + id: get_pr_tip + - name: Install dependencies + run: | + set -eux + # Install CUDA + wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-ubuntu1804.pin + sudo mv cuda-ubuntu1804.pin /etc/apt/preferences.d/cuda-repository-pin-600 + sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub + sudo add-apt-repository "deb http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/ /" + sudo apt-get update + sudo apt-get --no-install-recommends -y install cuda + # Install dependencies + pip install pyyaml + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - + sudo apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main" + sudo apt-get update + sudo apt-get install -y clang-tidy-8 + sudo update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-8 1000 + - name: Run clang-tidy + run: | + set -eux + git remote add upstream https://github.com/pytorch/pytorch + git fetch upstream "$GITHUB_BASE_REF" + BASE_SHA=${{ github.event.pull_request.base.sha }} + HEAD_SHA=${{ github.event.pull_request.head.sha }} + MERGE_BASE=$(git merge-base $BASE_SHA $HEAD_SHA) + + if [[ ! -d build ]]; then + git submodule update --init --recursive + + export USE_NCCL=0 + # We really only need compile_commands.json, so no need to build! + time python setup.py --cmake-only build + + # Generate ATen files. + time python aten/src/ATen/gen.py \ + -s aten/src/ATen \ + -d build/aten/src/ATen \ + aten/src/ATen/Declarations.cwrap \ + aten/src/THNN/generic/THNN.h \ + aten/src/THCUNN/generic/THCUNN.h \ + aten/src/ATen/nn.yaml \ + aten/src/ATen/native/native_functions.yaml + + # Generate PyTorch files. + time python tools/setup_helpers/generate_code.py \ + --declarations-path build/aten/src/ATen/Declarations.yaml \ + --nn-path aten/src + fi + + # Run Clang-Tidy + # The negative filters below are to exclude files that include onnx_pb.h or + # caffe2_pb.h, otherwise we'd have to build protos as part of this CI job. + python tools/clang_tidy.py \ + --verbose \ + --paths torch/csrc/ \ + --diff "$MERGE_BASE" \ + -g"-torch/csrc/jit/export.cpp" \ + -g"-torch/csrc/jit/import.cpp" \ + -g"-torch/csrc/jit/netdef_converter.cpp" \ + "$@" > ${GITHUB_WORKSPACE}/clang-tidy-output.txt + + cat ${GITHUB_WORKSPACE}/clang-tidy-output.txt + - name: Add annotations + uses: suo/add-annotations-github-action@master + with: + check_name: 'clang-tidy' + linter_output_path: 'clang-tidy-output.txt' + commit_sha: ${{ steps.get_pr_tip.outputs.commit_sha }} + regex: '^(?.*?):(?\d+):(?\d+): (?.*?) \[(?.*)\]' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index d243a9527b5..00000000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,116 +0,0 @@ -trigger: -- master - -jobs: -- job: consistent_circleci - displayName: "Ensure consistent CircleCI YAML" - pool: - vmImage: 'Ubuntu 16.04' - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: "3.7" - - script: pip install -r requirements.txt - - script: cd .circleci && ./ensure-consistency.py - -- job: validate_docker_version - displayName: "Ensure Docker version is correctly deployed" - pool: - vmImage: 'Ubuntu 16.04' - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: "3.7" - - script: .circleci/validate-docker-version.py - -- job: shellcheck_jenkins - displayName: "Shellcheck Jenkins scripts" - pool: - vmImage: 'Ubuntu 16.04' - steps: - - script: sudo apt-get install -y shellcheck - - script: .jenkins/run-shellcheck.sh - -- job: ensure_no_tabs - displayName: "Ensure no tabs" - pool: - vmImage: 'Ubuntu 16.04' - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: "2.7" - - script: (! git grep -I -l $'\t' -- . ':(exclude)*.svg' ':(exclude)**Makefile' ':(exclude)**/contrib/**' ':(exclude)third_party' ':(exclude).gitattributes' ':(exclude).gitmodules' || (echo "The above files have tabs; please convert them to spaces"; false)) - -- job: ensure_not_executable - displayName: "Ensure C++ source files are not executable" - pool: - vmImage: 'Ubuntu 16.04' - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: "2.7" - - script: (! find . \( -path ./third_party -o -path ./.git -o -path ./torch/bin -o -path ./build \) -prune -o -type f -executable -regextype posix-egrep -not -regex '.+(\.(bash|sh|py|so)|git-pre-commit)$' -print | grep . || (echo 'The above files have executable permission; please remove their executable permission by using `chmod -x`'; false)) - -- job: python_27_lint - displayName: "Python 2.7 Lint" - pool: - vmImage: 'Ubuntu 16.04' - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: "2.7" - - script: pip install flake8 - - script: | - rm -rf .circleci - flake8 -- job: python_37_lint - displayName: "Python 3.7 Lint" - pool: - vmImage: 'Ubuntu 16.04' - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: "3.7" - - script: pip install flake8 - - script: flake8 - -- job: mypy_typecheck - displayName: "MyPy typecheck" - pool: - vmImage: 'Ubuntu 16.04' - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: "3.6" - - script: pip install mypy mypy-extensions - - script: mypy @mypy-files.txt - -- job: cpp_doc_check - displayName: "CPP doc check" - pool: - vmImage: 'Ubuntu 16.04' - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: "3.6" - - script: sudo apt-get install -y doxygen && pip install -r requirements.txt - - script: cd docs/cpp/source && ./check-doxygen.sh - -- job: clang_tidy - displayName: "clang tidy" - pool: - vmImage: 'Ubuntu 16.04' - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: "3.6" - - script: | - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - - sudo apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main" - sudo apt-get update - sudo apt-get install -y clang-tidy-8 - sudo update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-8 1000 - - script: git branch master origin/master - - script: pip install pyyaml - - script: tools/run-clang-tidy-in-ci.sh - diff --git a/tools/run-clang-tidy-in-ci.sh b/tools/run-clang-tidy-in-ci.sh deleted file mode 100755 index 331c94f5d90..00000000000 --- a/tools/run-clang-tidy-in-ci.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -set -ex - -BASE_BRANCH=master -# From https://docs.travis-ci.com/user/environment-variables -if [[ $TRAVIS ]]; then - git remote add upstream https://github.com/pytorch/pytorch - git fetch upstream "$TRAVIS_BRANCH" - BASE_BRANCH="upstream/$TRAVIS_BRANCH" -fi - -if [[ ! -d build ]]; then - git submodule update --init --recursive - - mkdir build - pushd build - # We really only need compile_commands.json, so no need to build! - time cmake .. - popd - - # Generate ATen files. - time python aten/src/ATen/gen.py \ - -s aten/src/ATen \ - -d build/aten/src/ATen \ - aten/src/ATen/Declarations.cwrap \ - aten/src/THNN/generic/THNN.h \ - aten/src/THCUNN/generic/THCUNN.h \ - aten/src/ATen/nn.yaml \ - aten/src/ATen/native/native_functions.yaml - - # Generate PyTorch files. - time python tools/setup_helpers/generate_code.py \ - --declarations-path build/aten/src/ATen/Declarations.yaml \ - --nn-path aten/src -fi - -# Run Clang-Tidy -# The negative filters below are to exclude files that include onnx_pb.h or -# caffe2_pb.h, otherwise we'd have to build protos as part of this CI job. -time python tools/clang_tidy.py \ - --verbose \ - --paths torch/csrc/ \ - --diff "$BASE_BRANCH" \ - -g"-torch/csrc/jit/export.cpp" \ - -g"-torch/csrc/jit/import.cpp" \ - -g"-torch/csrc/jit/netdef_converter.cpp" \ - "$@"