From dfa18bf3f39c5a90b48baf956e50fa7da4462d3d Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Mon, 29 Jul 2024 12:20:46 +0800 Subject: [PATCH] [CI] add new test config label `ci-test-showlocals` to control test log verbosity (#131981) Add a new label `ci-test-showlocals` and add it to test config filter. If the PR is labeled with `ci-test-showlocals` or "ci-test-showlocals" present in the PR comment, the test config filter will set a environment variable `TEST_SHOWLOCALS`. Then `pytest` will show local variables on failures for better debugging. Pull Request resolved: https://github.com/pytorch/pytorch/pull/131981 Approved by: https://github.com/malfet --- .github/actions/filter-test-configs/action.yml | 3 +++ .github/actions/linux-test/action.yml | 1 + .github/scripts/filter_test_configs.py | 3 +++ .github/scripts/test_filter_test_configs.py | 17 +++++++++++++++++ .github/workflows/_linux-test.yml | 2 ++ .github/workflows/_mac-test-mps.yml | 2 ++ .github/workflows/_mac-test.yml | 1 + .github/workflows/_rocm-test.yml | 2 ++ .github/workflows/_win-test.yml | 1 + .github/workflows/_xpu-test.yml | 2 ++ test/run_test.py | 4 ++-- torch/testing/_internal/common_utils.py | 4 ++-- 12 files changed, 38 insertions(+), 4 deletions(-) diff --git a/.github/actions/filter-test-configs/action.yml b/.github/actions/filter-test-configs/action.yml index e65448c99f4..57f09c038d7 100644 --- a/.github/actions/filter-test-configs/action.yml +++ b/.github/actions/filter-test-configs/action.yml @@ -41,6 +41,9 @@ outputs: ci-verbose-test-logs: description: True if ci-verbose-test-logs label was on PR or [ci-verbose-test-logs] in PR body. value: ${{ steps.filter.outputs.ci-verbose-test-logs }} + ci-test-showlocals: + description: True if ci-test-showlocals label was on PR or [ci-test-showlocals] in PR body. + value: ${{ steps.filter.outputs.ci-test-showlocals }} ci-no-test-timeout: description: True if ci-no-test-timeout label was on PR or [ci-no-test-timeout] in PR body. value: ${{ steps.filter.outputs.ci-no-test-timeout }} diff --git a/.github/actions/linux-test/action.yml b/.github/actions/linux-test/action.yml index 6c8e761444b..24cb6e0eb40 100644 --- a/.github/actions/linux-test/action.yml +++ b/.github/actions/linux-test/action.yml @@ -167,6 +167,7 @@ runs: REENABLED_ISSUES: ${{ steps.keep-going.outputs.reenabled-issues }} CONTINUE_THROUGH_ERROR: ${{ steps.keep-going.outputs.keep-going }} VERBOSE_TEST_LOGS: ${{ steps.keep-going.outputs.ci-verbose-test-logs }} + TEST_SHOWLOCALS: ${{ steps.keep-going.outputs.ci-test-showlocals }} NO_TEST_TIMEOUT: ${{ steps.keep-going.outputs.ci-no-test-timeout }} NO_TD: ${{ steps.keep-going.outputs.ci-no-td }} TD_DISTRIBUTED: ${{ steps.keep-going.outputs.ci-td-distributed }} diff --git a/.github/scripts/filter_test_configs.py b/.github/scripts/filter_test_configs.py index 3c2969b3523..12c577c7732 100755 --- a/.github/scripts/filter_test_configs.py +++ b/.github/scripts/filter_test_configs.py @@ -505,6 +505,9 @@ def perform_misc_tasks( "ci-verbose-test-logs", check_for_setting(labels, pr_body, "ci-verbose-test-logs"), ) + set_output( + "ci-test-showlocals", check_for_setting(labels, pr_body, "ci-test-showlocals") + ) set_output( "ci-no-test-timeout", check_for_setting(labels, pr_body, "ci-no-test-timeout") ) diff --git a/.github/scripts/test_filter_test_configs.py b/.github/scripts/test_filter_test_configs.py index 2f73d022c3d..421da22f7e4 100755 --- a/.github/scripts/test_filter_test_configs.py +++ b/.github/scripts/test_filter_test_configs.py @@ -683,6 +683,7 @@ class TestConfigFilter(TestCase): def _gen_expected_string( keep_going: bool = False, ci_verbose_test_logs: bool = False, + ci_test_showlocals: bool = False, ci_no_test_timeout: bool = False, ci_no_td: bool = False, ci_td_distributed: bool = False, @@ -692,6 +693,7 @@ class TestConfigFilter(TestCase): return ( f"keep-going={keep_going}\n" f"ci-verbose-test-logs={ci_verbose_test_logs}\n" + f"ci-test-showlocals={ci_test_showlocals}\n" f"ci-no-test-timeout={ci_no_test_timeout}\n" f"ci-no-td={ci_no_td}\n" f"ci-td-distributed={ci_td_distributed}\n" @@ -733,6 +735,21 @@ class TestConfigFilter(TestCase): ), "description": "No pipe logs label and no test timeout in PR body", }, + { + "labels": {"ci-test-showlocals"}, + "test_matrix": '{include: [{config: "default"}]}', + "job_name": "A job name", + "expected": _gen_expected_string(ci_test_showlocals=True), + "description": "Has ci-test-showlocals", + }, + { + "labels": {}, + "test_matrix": '{include: [{config: "default"}]}', + "job_name": "A job name", + "pr_body": "[ci-test-showlocals]", + "expected": _gen_expected_string(ci_test_showlocals=True), + "description": "ci-test-showlocals in body", + }, { "labels": {"ci-no-test-timeout"}, "test_matrix": '{include: [{config: "default"}]}', diff --git a/.github/workflows/_linux-test.yml b/.github/workflows/_linux-test.yml index 5f3f290dd31..fa4ae11a3bf 100644 --- a/.github/workflows/_linux-test.yml +++ b/.github/workflows/_linux-test.yml @@ -198,6 +198,7 @@ jobs: REENABLED_ISSUES: ${{ steps.keep-going.outputs.reenabled-issues }} CONTINUE_THROUGH_ERROR: ${{ steps.keep-going.outputs.keep-going }} VERBOSE_TEST_LOGS: ${{ steps.keep-going.outputs.ci-verbose-test-logs }} + TEST_SHOWLOCALS: ${{ steps.keep-going.outputs.ci-test-showlocals }} NO_TEST_TIMEOUT: ${{ steps.keep-going.outputs.ci-no-test-timeout }} NO_TD: ${{ steps.keep-going.outputs.ci-no-td }} TD_DISTRIBUTED: ${{ steps.keep-going.outputs.ci-td-distributed }} @@ -251,6 +252,7 @@ jobs: -e REENABLED_ISSUES \ -e CONTINUE_THROUGH_ERROR \ -e VERBOSE_TEST_LOGS \ + -e TEST_SHOWLOCALS \ -e NO_TEST_TIMEOUT \ -e NO_TD \ -e TD_DISTRIBUTED \ diff --git a/.github/workflows/_mac-test-mps.yml b/.github/workflows/_mac-test-mps.yml index 2c0da2f8afd..9ad6b6291f3 100644 --- a/.github/workflows/_mac-test-mps.yml +++ b/.github/workflows/_mac-test-mps.yml @@ -35,6 +35,7 @@ jobs: is-test-matrix-empty: ${{ steps.filter.outputs.is-test-matrix-empty }} keep-going: ${{ steps.filter.outputs.keep-going }} ci-verbose-test-logs: ${{ steps.filter.outputs.ci-verbose-test-logs }} + ci-test-showlocals: ${{ steps.filter.outputs.ci-test-showlocals }} ci-no-test-timeout: ${{ steps.filter.outputs.ci-no-test-timeout }} ci-no-td: ${{ steps.filter.outputs.ci-no-td }} reenabled-issues: ${{ steps.filter.outputs.reenabled-issues }} @@ -98,6 +99,7 @@ jobs: PR_BODY: ${{ github.event.pull_request.body }} CONTINUE_THROUGH_ERROR: ${{ needs.filter.outputs.keep-going }} VERBOSE_TEST_LOGS: ${{ needs.filter.outputs.ci-verbose-test-logs }} + TEST_SHOWLOCALS: ${{ needs.filter.outputs.ci-test-showlocals }} NO_TEST_TIMEOUT: ${{ needs.filter.outputs.ci-no-test-timeout }} NO_TD: ${{ needs.filter.outputs.ci-no-td }} PIP_REQUIREMENTS_FILE: .github/requirements/pip-requirements-${{ runner.os }}.txt diff --git a/.github/workflows/_mac-test.yml b/.github/workflows/_mac-test.yml index 3e82194ff34..adf136226ba 100644 --- a/.github/workflows/_mac-test.yml +++ b/.github/workflows/_mac-test.yml @@ -144,6 +144,7 @@ jobs: PYTORCH_TEST_RERUN_DISABLED_TESTS: ${{ matrix.rerun_disabled_tests && '1' || '0' }} CONTINUE_THROUGH_ERROR: ${{ steps.keep-going.outputs.keep-going }} VERBOSE_TEST_LOGS: ${{ steps.keep-going.outputs.ci-verbose-test-logs }} + TEST_SHOWLOCALS: ${{ steps.keep-going.outputs.ci-test-showlocals }} NO_TEST_TIMEOUT: ${{ steps.keep-going.outputs.ci-no-test-timeout }} NO_TD: ${{ steps.keep-going.outputs.ci-no-td }} PIP_REQUIREMENTS_FILE: .github/requirements/pip-requirements-${{ runner.os }}.txt diff --git a/.github/workflows/_rocm-test.yml b/.github/workflows/_rocm-test.yml index 1f2d86273ee..790cd9d403d 100644 --- a/.github/workflows/_rocm-test.yml +++ b/.github/workflows/_rocm-test.yml @@ -154,6 +154,7 @@ jobs: SHA1: ${{ github.event.pull_request.head.sha || github.sha }} CONTINUE_THROUGH_ERROR: ${{ steps.keep-going.outputs.keep-going }} VERBOSE_TEST_LOGS: ${{ steps.keep-going.outputs.ci-verbose-test-logs }} + TEST_SHOWLOCALS: ${{ steps.keep-going.outputs.ci-test-showlocals }} NO_TEST_TIMEOUT: ${{ steps.keep-going.outputs.ci-no-test-timeout }} NO_TD: ${{ steps.keep-going.outputs.ci-no-td }} TEST_CONFIG: ${{ matrix.config }} @@ -205,6 +206,7 @@ jobs: -e REENABLED_ISSUES \ -e CONTINUE_THROUGH_ERROR \ -e VERBOSE_TEST_LOGS \ + -e TEST_SHOWLOCALS \ -e NO_TEST_TIMEOUT \ -e NO_TD \ -e MAX_JOBS="$(nproc --ignore=2)" \ diff --git a/.github/workflows/_win-test.yml b/.github/workflows/_win-test.yml index 347355651f4..3519ad8d81d 100644 --- a/.github/workflows/_win-test.yml +++ b/.github/workflows/_win-test.yml @@ -157,6 +157,7 @@ jobs: PYTHON_VERSION: 3.8 CONTINUE_THROUGH_ERROR: ${{ steps.keep-going.outputs.keep-going }} VERBOSE_TEST_LOGS: ${{ steps.keep-going.outputs.ci-verbose-test-logs }} + TEST_SHOWLOCALS: ${{ steps.keep-going.outputs.ci-test-showlocals }} NO_TEST_TIMEOUT: ${{ steps.keep-going.outputs.ci-no-test-timeout }} NO_TD: ${{ steps.keep-going.outputs.ci-no-td }} VC_PRODUCT: "BuildTools" diff --git a/.github/workflows/_xpu-test.yml b/.github/workflows/_xpu-test.yml index d7af711f8ad..036a2c8eeca 100644 --- a/.github/workflows/_xpu-test.yml +++ b/.github/workflows/_xpu-test.yml @@ -143,6 +143,7 @@ jobs: PYTORCH_RETRY_TEST_CASES: 1 PYTORCH_OVERRIDE_FLAKY_SIGNAL: 1 CONTINUE_THROUGH_ERROR: ${{ steps.keep-going.outputs.keep-going }} + TEST_SHOWLOCALS: ${{ steps.keep-going.outputs.ci-test-showlocals }} VERBOSE_TEST_LOGS: ${{ steps.keep-going.outputs.ci-verbose-test-logs }} NO_TEST_TIMEOUT: ${{ steps.keep-going.outputs.ci-no-test-timeout }} NO_TD: ${{ steps.keep-going.outputs.ci-no-td }} @@ -189,6 +190,7 @@ jobs: -e PYTORCH_OVERRIDE_FLAKY_SIGNAL \ -e CONTINUE_THROUGH_ERROR \ -e VERBOSE_TEST_LOGS \ + -e TEST_SHOWLOCALS \ -e NO_TEST_TIMEOUT \ -e NO_TD \ -e MAX_JOBS="$(nproc --ignore=2)" \ diff --git a/test/run_test.py b/test/run_test.py index 8e9782f2f62..58ce4ec6e66 100755 --- a/test/run_test.py +++ b/test/run_test.py @@ -1129,14 +1129,14 @@ def parse_args(): parser.add_argument( "--showlocals", action=argparse.BooleanOptionalAction, - default=(not IS_CI), + default=strtobool(os.environ.get("TEST_SHOWLOCALS", str(not IS_CI))), help="Show local variables in tracebacks (default: True)", ) else: parser.add_argument( "--showlocals", action="store_true", - default=(not IS_CI), + default=strtobool(os.environ.get("TEST_SHOWLOCALS", str(not IS_CI))), help="Show local variables in tracebacks (default: True)", ) parser.add_argument("--no-showlocals", dest="showlocals", action="store_false") diff --git a/torch/testing/_internal/common_utils.py b/torch/testing/_internal/common_utils.py index c4be7f1fb46..b27bb8c7612 100644 --- a/torch/testing/_internal/common_utils.py +++ b/torch/testing/_internal/common_utils.py @@ -899,7 +899,7 @@ TEST_IN_SUBPROCESS = args.subprocess TEST_SAVE_XML = args.save_xml REPEAT_COUNT = args.repeat SEED = args.seed -SHOW_LOCALS = args.showlocals +SHOWLOCALS = args.showlocals if not getattr(expecttest, "ACCEPT", False): expecttest.ACCEPT = args.accept UNITTEST_ARGS = [sys.argv[0]] + remaining @@ -1137,7 +1137,7 @@ def run_tests(argv=UNITTEST_ARGS): if not lint_test_case_extension(suite): sys.exit(1) - if SHOW_LOCALS: + if SHOWLOCALS: argv = [ argv[0], *(["--showlocals", "--tb=long", "--color=yes"] if USE_PYTEST else ["--locals"]),