From 4fe6a5dc349f97a57708a666f4d8c7525f7471e0 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Wed, 7 Aug 2024 18:42:53 +0000 Subject: [PATCH] Move slow tests to be in repo (#132379) Move the slow test json to be in the pytorch/pytorch repo and make a job that will update it weekly. The job uses the same environment as the commit hash. It uses similar code to the hash updates, but the hash update contains a lot of code that is specific to the hash update, so I chose to pick out the parts that are relevant Remove references to the old file and set up testing to read from the new file instead The old update cadence was every day, the new one is every week The auto slow test infra + the lack of pinning between pytorch and test-infra makes it really hard to tell if a test started failing because of a change or because of the slow test json changing. While this can have benefits, like disable test issues being effective everywhere immediately, it can also be very confusing, especially since we don't have the same insight into slow tests like we do for disable issues. Example PR made: https://github.com/pytorch/pytorch/pull/132383 (with all the changes from this PR because it was working on top of this) We should just get rid of this at some point in favor of the slowTest decorator, but there are some tests that take 5+ minutes to run and I don't want to track them down right now Pull Request resolved: https://github.com/pytorch/pytorch/pull/132379 Approved by: https://github.com/huydhn --- .github/workflows/weekly.yml | 28 +++ .gitignore | 1 - scripts/release/apply-release-changes.sh | 2 - test/slow_tests.json | 302 +++++++++++++++++++++++ tools/stats/import_test_stats.py | 12 - tools/testing/test_selections.py | 3 +- tools/testing/update_slow_tests.py | 208 ++++++++++++++++ torch/testing/_internal/common_utils.py | 2 +- 8 files changed, 540 insertions(+), 18 deletions(-) create mode 100644 test/slow_tests.json create mode 100644 tools/testing/update_slow_tests.py diff --git a/.github/workflows/weekly.yml b/.github/workflows/weekly.yml index f097b146c21..7975dd5cb52 100644 --- a/.github/workflows/weekly.yml +++ b/.github/workflows/weekly.yml @@ -39,3 +39,31 @@ jobs: test-infra-ref: main updatebot-token: ${{ secrets.UPDATEBOT_TOKEN }} pytorchbot-token: ${{ secrets.GH_PYTORCHBOT_TOKEN }} + + update-slow-tests: + runs-on: ubuntu-latest + environment: update-commit-hash + steps: + - name: Checkout repo + uses: actions/checkout@v3 + with: + fetch-depth: 0 + token: ${{ secrets.UPDATEBOT_TOKEN }} + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: '3.9' + - name: Install requirements + shell: bash + run: | + pip install rockset==1.0.3 requests==2.32.2 + - name: Update slow test file + shell: bash + env: + ROCKSET_API_KEY: ${{ secrets.ROCKSET_API_KEY }} + PYTORCHBOT_TOKEN: ${{ secrets.GH_PYTORCHBOT_TOKEN }} + UPDATEBOT_TOKEN: ${{ secrets.UPDATEBOT_TOKEN }} + run: | + git config --global user.name "PyTorch UpdateBot" + git config --global user.email "pytorchupdatebot@users.noreply.github.com" + python tools/testing/update_slow_tests.py diff --git a/.gitignore b/.gitignore index 8a1d99053a1..366463ca0ae 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,6 @@ coverage.xml /.extracted_scripts/ **/.pytorch_specified_test_cases.csv **/.pytorch-disabled-tests.json -**/.pytorch-slow-tests.json */*.pyc */*.so* */**/__pycache__ diff --git a/scripts/release/apply-release-changes.sh b/scripts/release/apply-release-changes.sh index 9c5c96831ce..c1f37a40292 100755 --- a/scripts/release/apply-release-changes.sh +++ b/scripts/release/apply-release-changes.sh @@ -50,8 +50,6 @@ SLOW_VER=$(aws s3api list-object-versions --bucket ossci-metrics --prefix slow-t DISABLED_TESTS_VER=$(aws s3api list-object-versions --bucket ossci-metrics --prefix disabled-tests-condensed.json --query 'Versions[?IsLatest].[VersionId]' --output text) sed -i -e s#unstable-jobs.json#"unstable-jobs.json?versionId=${UNSTABLE_VER}"# .github/scripts/filter_test_configs.py sed -i -e s#disabled-jobs.json#"disabled-jobs.json?versionId=${DISABLED_VER}"# .github/scripts/filter_test_configs.py -# please note we want to match slow-tests.json not .pytorch-slow-tests.json hence "/" is needed here -sed -i -e s#/slow-tests.json#"/slow-tests.json?versionId=${SLOW_VER}"# tools/stats/import_test_stats.py sed -i -e s#disabled-tests-condensed.json#"disabled-tests-condensed.json?versionId=${DISABLED_TESTS_VER}"# tools/stats/import_test_stats.py # Optional git commit -m "[RELEASE-ONLY CHANGES] Branch Cut for Release {RELEASE_VERSION}" diff --git a/test/slow_tests.json b/test/slow_tests.json new file mode 100644 index 00000000000..b198bdcf2d4 --- /dev/null +++ b/test/slow_tests.json @@ -0,0 +1,302 @@ +{ + "test_AllenaiLongformerBase_repro_cpu (__main__.CpuHalideTests)": 211.949, + "test_adaptive_max_pool2d1_cpu (__main__.CpuHalideTests)": 111.929, + "test_alexnet_prefix_cpu (__main__.CpuHalideTests)": 185.141, + "test_aot_autograd_symbolic_exhaustive_linalg_svd_cpu_float32 (__main__.TestEagerFusionOpInfoCPU)": 67.44693333333333, + "test_aot_autograd_symbolic_exhaustive_nn_functional_max_pool1d_cpu_float32 (__main__.TestEagerFusionOpInfoCPU)": 103.4952, + "test_aot_autograd_symbolic_exhaustive_nn_functional_max_pool2d_cpu_float32 (__main__.TestEagerFusionOpInfoCPU)": 215.06906666666666, + "test_aot_autograd_symbolic_exhaustive_nn_functional_max_pool3d_cpu_float32 (__main__.TestEagerFusionOpInfoCPU)": 126.95360000000001, + "test_aot_autograd_symbolic_exhaustive_svd_cpu_float32 (__main__.TestEagerFusionOpInfoCPU)": 69.75275, + "test_aot_autograd_symbolic_module_exhaustive_nn_TransformerDecoderLayer_cpu_float32 (__main__.TestEagerFusionModuleInfoCPU)": 110.57966666666667, + "test_aot_export_joint_simple_repro_dynamic_shapes (__main__.DynamicShapesAotAutogradFallbackTests)": 345.20975000000004, + "test_aoti_eager_override_registration_cpu (__main__.CpuTests)": 81.80724000000001, + "test_aoti_eager_override_registration_cuda (__main__.GPUTests)": 81.5502857142857, + "test_aoti_eager_override_registration_dynamic_shapes_cpu (__main__.DynamicShapesCodegenCpuTests)": 80.72995238095237, + "test_aoti_eager_override_registration_dynamic_shapes_cpu (__main__.DynamicShapesCpuTests)": 79.88047619047619, + "test_aoti_eager_override_registration_dynamic_shapes_cuda (__main__.DynamicShapesCodegenGPUTests)": 75.02325, + "test_aoti_eager_override_registration_dynamic_shapes_cuda (__main__.DynamicShapesGPUTests)": 74.38550000000001, + "test_aoti_eager_with_scalar_cpu (__main__.CpuTests)": 87.54754166666667, + "test_aoti_eager_with_scalar_cuda (__main__.GPUTests)": 85.06014285714285, + "test_aoti_eager_with_scalar_dynamic_shapes_cpu (__main__.DynamicShapesCodegenCpuTests)": 87.73019047619047, + "test_aoti_eager_with_scalar_dynamic_shapes_cpu (__main__.DynamicShapesCpuTests)": 87.14119047619049, + "test_aoti_eager_with_scalar_dynamic_shapes_cuda (__main__.DynamicShapesCodegenGPUTests)": 83.3455, + "test_aoti_eager_with_scalar_dynamic_shapes_cuda (__main__.DynamicShapesGPUTests)": 82.4865, + "test_avg_pool3d_backward2_cuda (__main__.GPUTests)": 94.67914285714285, + "test_avg_pool3d_backward2_dynamic_shapes_cpu (__main__.DynamicShapesCodegenCpuTests)": 70.80028571428572, + "test_avg_pool3d_backward2_dynamic_shapes_cpu (__main__.DynamicShapesCpuTests)": 66.188125, + "test_avg_pool3d_backward2_dynamic_shapes_cuda (__main__.DynamicShapesCodegenGPUTests)": 66.239, + "test_avg_pool3d_backward2_dynamic_shapes_cuda (__main__.DynamicShapesGPUTests)": 135.30899999999997, + "test_avg_pool3d_backward_cpu (__main__.CpuHalideTests)": 61.719, + "test_backward_nn_functional_multi_head_attention_forward_cpu_float32 (__main__.TestCompositeComplianceCPU)": 78.70693333333334, + "test_backward_nn_functional_multi_head_attention_forward_cuda_float32 (__main__.TestCompositeComplianceCUDA)": 96.261, + "test_basic_cpu (__main__.EfficientConvBNEvalCpuTests)": 237.76485714285712, + "test_basic_cuda (__main__.EfficientConvBNEvalCudaTests)": 92.503, + "test_captured_score_mod_aot_eager_gradcheck_score_mod_name__head_offset_mode_eager (__main__.TestFlexAttention)": 139.8705, + "test_checkpoint_cast (__main__.TestFxToOnnx)": 136.983, + "test_comprehensive_constant_pad_nd_cpu_float16 (__main__.TestInductorOpInfoCPU)": 60.31935294117646, + "test_comprehensive_diff_cpu_bool (__main__.TestInductorOpInfoCPU)": 92.7407, + "test_comprehensive_diff_cpu_float32 (__main__.TestInductorOpInfoCPU)": 92.67049999999999, + "test_comprehensive_diff_cpu_float64 (__main__.TestInductorOpInfoCPU)": 91.261, + "test_comprehensive_diff_cpu_int32 (__main__.TestInductorOpInfoCPU)": 92.00640000000001, + "test_comprehensive_diff_cpu_int64 (__main__.TestInductorOpInfoCPU)": 88.7649, + "test_comprehensive_diff_cuda_complex128 (__main__.TestDecompCUDA)": 120.986, + "test_comprehensive_diff_cuda_complex64 (__main__.TestDecompCUDA)": 105.15944444444446, + "test_comprehensive_diff_cuda_float32 (__main__.TestDecompCUDA)": 87.45349999999999, + "test_comprehensive_diff_cuda_float64 (__main__.TestDecompCUDA)": 88.94550000000001, + "test_comprehensive_dist_cpu_float16 (__main__.TestInductorOpInfoCPU)": 72.8767, + "test_comprehensive_dist_cpu_float32 (__main__.TestInductorOpInfoCPU)": 71.64869999999999, + "test_comprehensive_dist_cpu_float64 (__main__.TestInductorOpInfoCPU)": 70.62299999999999, + "test_comprehensive_eye_cpu_bool (__main__.TestInductorOpInfoCPU)": 112.79639999999999, + "test_comprehensive_eye_cpu_float16 (__main__.TestInductorOpInfoCPU)": 110.69359999999999, + "test_comprehensive_eye_cpu_float32 (__main__.TestInductorOpInfoCPU)": 111.8332, + "test_comprehensive_eye_cpu_float64 (__main__.TestInductorOpInfoCPU)": 113.01580000000001, + "test_comprehensive_eye_cpu_int32 (__main__.TestInductorOpInfoCPU)": 110.6647, + "test_comprehensive_eye_cpu_int64 (__main__.TestInductorOpInfoCPU)": 113.61270000000002, + "test_comprehensive_grid_sampler_2d_cpu_float32 (__main__.TestDecompCPU)": 337.5013, + "test_comprehensive_grid_sampler_2d_cpu_float32 (__main__.TestInductorOpInfoCPU)": 112.65060000000001, + "test_comprehensive_grid_sampler_2d_cpu_float64 (__main__.TestDecompCPU)": 352.82779999999997, + "test_comprehensive_grid_sampler_2d_cpu_float64 (__main__.TestInductorOpInfoCPU)": 67.2527, + "test_comprehensive_grid_sampler_2d_cuda_bfloat16 (__main__.TestDecompCUDA)": 277.8468888888889, + "test_comprehensive_grid_sampler_2d_cuda_float16 (__main__.TestDecompCUDA)": 261.31533333333334, + "test_comprehensive_grid_sampler_2d_cuda_float32 (__main__.TestDecompCUDA)": 1217.510111111111, + "test_comprehensive_grid_sampler_2d_cuda_float32 (__main__.TestInductorOpInfoCUDA)": 73.16566666666667, + "test_comprehensive_grid_sampler_2d_cuda_float64 (__main__.TestDecompCUDA)": 1187.5324999999998, + "test_comprehensive_grid_sampler_2d_cuda_float64 (__main__.TestInductorOpInfoCUDA)": 81.23666666666666, + "test_comprehensive_linalg_svd_cuda_complex128 (__main__.TestDecompCUDA)": 83.36449999999999, + "test_comprehensive_linalg_svd_cuda_complex64 (__main__.TestDecompCUDA)": 85.197, + "test_comprehensive_linalg_vector_norm_cpu_float16 (__main__.TestInductorOpInfoCPU)": 176.8523, + "test_comprehensive_linalg_vector_norm_cpu_float32 (__main__.TestInductorOpInfoCPU)": 176.5644, + "test_comprehensive_linalg_vector_norm_cpu_float64 (__main__.TestInductorOpInfoCPU)": 176.33440000000002, + "test_comprehensive_logspace_cpu_float32 (__main__.TestInductorOpInfoCPU)": 379.27200000000005, + "test_comprehensive_logspace_cpu_float64 (__main__.TestInductorOpInfoCPU)": 382.0692, + "test_comprehensive_logspace_cpu_int32 (__main__.TestInductorOpInfoCPU)": 365.48, + "test_comprehensive_logspace_cpu_int64 (__main__.TestInductorOpInfoCPU)": 366.99120000000005, + "test_comprehensive_masked_amax_cpu_float16 (__main__.TestInductorOpInfoCPU)": 85.73589999999999, + "test_comprehensive_masked_amax_cpu_float32 (__main__.TestInductorOpInfoCPU)": 84.76559999999999, + "test_comprehensive_masked_amax_cpu_float64 (__main__.TestInductorOpInfoCPU)": 83.74539999999999, + "test_comprehensive_masked_amax_cpu_int32 (__main__.TestInductorOpInfoCPU)": 81.6752, + "test_comprehensive_masked_amax_cpu_int64 (__main__.TestInductorOpInfoCPU)": 80.1269, + "test_comprehensive_masked_amin_cpu_float16 (__main__.TestInductorOpInfoCPU)": 85.1681, + "test_comprehensive_masked_amin_cpu_float32 (__main__.TestInductorOpInfoCPU)": 87.01599999999999, + "test_comprehensive_masked_amin_cpu_float64 (__main__.TestInductorOpInfoCPU)": 85.30009999999999, + "test_comprehensive_masked_amin_cpu_int32 (__main__.TestInductorOpInfoCPU)": 81.06280000000001, + "test_comprehensive_masked_amin_cpu_int64 (__main__.TestInductorOpInfoCPU)": 84.49640000000001, + "test_comprehensive_masked_mean_cpu_bool (__main__.TestInductorOpInfoCPU)": 82.6498, + "test_comprehensive_masked_mean_cpu_float16 (__main__.TestInductorOpInfoCPU)": 85.0721, + "test_comprehensive_masked_mean_cpu_float32 (__main__.TestInductorOpInfoCPU)": 86.45490000000002, + "test_comprehensive_masked_mean_cpu_float64 (__main__.TestInductorOpInfoCPU)": 84.9486, + "test_comprehensive_masked_mean_cpu_int32 (__main__.TestInductorOpInfoCPU)": 85.1464, + "test_comprehensive_masked_mean_cpu_int64 (__main__.TestInductorOpInfoCPU)": 83.1313, + "test_comprehensive_masked_norm_cpu_float16 (__main__.TestInductorOpInfoCPU)": 422.03270000000003, + "test_comprehensive_masked_norm_cpu_float32 (__main__.TestInductorOpInfoCPU)": 419.49539999999996, + "test_comprehensive_masked_norm_cpu_float64 (__main__.TestInductorOpInfoCPU)": 409.55060000000003, + "test_comprehensive_masked_norm_cuda_float16 (__main__.TestInductorOpInfoCUDA)": 93.67716666666666, + "test_comprehensive_masked_norm_cuda_float32 (__main__.TestInductorOpInfoCUDA)": 88.622, + "test_comprehensive_masked_norm_cuda_float64 (__main__.TestInductorOpInfoCUDA)": 91.79166666666667, + "test_comprehensive_masked_prod_cpu_bool (__main__.TestInductorOpInfoCPU)": 81.3001, + "test_comprehensive_masked_prod_cpu_float16 (__main__.TestInductorOpInfoCPU)": 86.5596, + "test_comprehensive_masked_prod_cpu_float32 (__main__.TestInductorOpInfoCPU)": 85.2926, + "test_comprehensive_masked_prod_cpu_float64 (__main__.TestInductorOpInfoCPU)": 84.71660000000001, + "test_comprehensive_masked_prod_cpu_int32 (__main__.TestInductorOpInfoCPU)": 84.0162, + "test_comprehensive_masked_prod_cpu_int64 (__main__.TestInductorOpInfoCPU)": 81.37209999999999, + "test_comprehensive_masked_sum_cpu_bool (__main__.TestInductorOpInfoCPU)": 81.57050000000001, + "test_comprehensive_masked_sum_cpu_float16 (__main__.TestInductorOpInfoCPU)": 82.18870000000001, + "test_comprehensive_masked_sum_cpu_float32 (__main__.TestInductorOpInfoCPU)": 82.77929999999999, + "test_comprehensive_masked_sum_cpu_float64 (__main__.TestInductorOpInfoCPU)": 81.9615, + "test_comprehensive_masked_sum_cpu_int32 (__main__.TestInductorOpInfoCPU)": 82.8871, + "test_comprehensive_masked_sum_cpu_int64 (__main__.TestInductorOpInfoCPU)": 83.2116, + "test_comprehensive_nn_functional_conv_transpose3d_cuda_complex128 (__main__.TestDecompCUDA)": 62.840444444444444, + "test_comprehensive_nn_functional_conv_transpose3d_cuda_complex64 (__main__.TestDecompCUDA)": 63.12155555555556, + "test_comprehensive_nn_functional_gaussian_nll_loss_cuda_float32 (__main__.TestDecompCUDA)": 115.99399999999999, + "test_comprehensive_nn_functional_gaussian_nll_loss_cuda_float64 (__main__.TestDecompCUDA)": 112.42922222222222, + "test_comprehensive_nn_functional_glu_cpu_float16 (__main__.TestInductorOpInfoCPU)": 66.2836, + "test_comprehensive_nn_functional_glu_cpu_float32 (__main__.TestInductorOpInfoCPU)": 63.87760000000001, + "test_comprehensive_nn_functional_glu_cpu_float64 (__main__.TestInductorOpInfoCPU)": 61.07164705882354, + "test_comprehensive_nn_functional_grid_sample_cpu_float32 (__main__.TestDecompCPU)": 93.46609090909091, + "test_comprehensive_nn_functional_grid_sample_cpu_float64 (__main__.TestDecompCPU)": 92.66881818181818, + "test_comprehensive_nn_functional_grid_sample_cuda_bfloat16 (__main__.TestDecompCUDA)": 72.35, + "test_comprehensive_nn_functional_grid_sample_cuda_float16 (__main__.TestDecompCUDA)": 64.90466666666666, + "test_comprehensive_nn_functional_grid_sample_cuda_float32 (__main__.TestDecompCUDA)": 265.2443333333333, + "test_comprehensive_nn_functional_grid_sample_cuda_float64 (__main__.TestDecompCUDA)": 250.08033333333333, + "test_comprehensive_nn_functional_interpolate_bicubic_cuda_float32 (__main__.TestDecompCUDA)": 61.85044444444444, + "test_comprehensive_nn_functional_interpolate_bicubic_cuda_float64 (__main__.TestDecompCUDA)": 63.002444444444436, + "test_comprehensive_nn_functional_interpolate_trilinear_cuda_float32 (__main__.TestDecompCUDA)": 102.0025, + "test_comprehensive_nn_functional_interpolate_trilinear_cuda_float64 (__main__.TestDecompCUDA)": 104.59100000000001, + "test_comprehensive_nn_functional_max_pool1d_cpu_float16 (__main__.TestInductorOpInfoCPU)": 152.2596, + "test_comprehensive_nn_functional_max_pool1d_cpu_float32 (__main__.TestInductorOpInfoCPU)": 140.01214285714286, + "test_comprehensive_nn_functional_max_pool1d_cpu_float64 (__main__.TestInductorOpInfoCPU)": 140.58085714285716, + "test_comprehensive_nn_functional_max_pool2d_cpu_float16 (__main__.TestInductorOpInfoCPU)": 710.7855714285714, + "test_comprehensive_nn_functional_max_pool2d_cpu_float32 (__main__.TestInductorOpInfoCPU)": 697.3474285714285, + "test_comprehensive_nn_functional_max_pool2d_cpu_float64 (__main__.TestInductorOpInfoCPU)": 678.1218571428572, + "test_comprehensive_nn_functional_max_pool2d_cpu_int32 (__main__.TestInductorOpInfoCPU)": 641.9231428571428, + "test_comprehensive_nn_functional_max_pool2d_cpu_int64 (__main__.TestInductorOpInfoCPU)": 655.1732857142857, + "test_comprehensive_nn_functional_max_pool2d_cuda_float16 (__main__.TestInductorOpInfoCUDA)": 775.9625, + "test_comprehensive_nn_functional_max_pool2d_cuda_float32 (__main__.TestInductorOpInfoCUDA)": 767.9121666666666, + "test_comprehensive_nn_functional_max_pool2d_cuda_float64 (__main__.TestInductorOpInfoCUDA)": 778.5028333333333, + "test_comprehensive_nn_functional_pad_constant_cpu_float16 (__main__.TestInductorOpInfoCPU)": 60.259235294117644, + "test_comprehensive_nn_functional_pad_constant_cpu_float32 (__main__.TestInductorOpInfoCPU)": 60.22264705882352, + "test_comprehensive_nn_functional_pad_constant_cpu_float64 (__main__.TestInductorOpInfoCPU)": 60.483411764705885, + "test_comprehensive_nn_functional_poisson_nll_loss_cpu_float16 (__main__.TestInductorOpInfoCPU)": 94.4827142857143, + "test_comprehensive_nn_functional_poisson_nll_loss_cpu_float32 (__main__.TestInductorOpInfoCPU)": 96.45214285714285, + "test_comprehensive_nn_functional_poisson_nll_loss_cpu_float64 (__main__.TestInductorOpInfoCPU)": 91.70985714285715, + "test_comprehensive_nn_functional_poisson_nll_loss_cpu_int32 (__main__.TestInductorOpInfoCPU)": 95.28557142857143, + "test_comprehensive_nn_functional_poisson_nll_loss_cpu_int64 (__main__.TestInductorOpInfoCPU)": 92.5167142857143, + "test_comprehensive_nn_functional_unfold_cpu_float16 (__main__.TestInductorOpInfoCPU)": 189.38628571428572, + "test_comprehensive_nn_functional_unfold_cpu_float32 (__main__.TestInductorOpInfoCPU)": 183.38171428571428, + "test_comprehensive_nn_functional_unfold_cpu_float64 (__main__.TestInductorOpInfoCPU)": 184.58571428571423, + "test_comprehensive_ormqr_cuda_complex128 (__main__.TestDecompCUDA)": 137.61211111111112, + "test_comprehensive_ormqr_cuda_complex64 (__main__.TestDecompCUDA)": 139.59522222222222, + "test_comprehensive_ormqr_cuda_float32 (__main__.TestDecompCUDA)": 88.364, + "test_comprehensive_ormqr_cuda_float32 (__main__.TestInductorOpInfoCUDA)": 83.6305, + "test_comprehensive_ormqr_cuda_float64 (__main__.TestDecompCUDA)": 87.9585, + "test_comprehensive_pca_lowrank_cuda_complex128 (__main__.TestDecompCUDA)": 68.5215, + "test_comprehensive_pca_lowrank_cuda_complex64 (__main__.TestDecompCUDA)": 62.06933333333333, + "test_comprehensive_svd_cuda_complex128 (__main__.TestDecompCUDA)": 94.2525, + "test_comprehensive_svd_cuda_complex64 (__main__.TestDecompCUDA)": 94.307, + "test_comprehensive_svd_lowrank_cuda_complex128 (__main__.TestDecompCUDA)": 68.14222222222222, + "test_comprehensive_svd_lowrank_cuda_complex64 (__main__.TestDecompCUDA)": 72.507, + "test_cond_autograd_nested (__main__.TestControlFlow)": 162.97220000000002, + "test_constructor_autograd_SparseBSC_cuda (__main__.TestSparseAnyCUDA)": 110.65333333333334, + "test_constructor_autograd_SparseBSR_cuda (__main__.TestSparseAnyCUDA)": 119.97566666666665, + "test_constructor_autograd_SparseCSC_cuda (__main__.TestSparseAnyCUDA)": 86.57166666666667, + "test_constructor_autograd_SparseCSR_cuda (__main__.TestSparseAnyCUDA)": 88.60611111111112, + "test_conv1d_basic (__main__.TestXNNPACKConv1dTransformPass)": 93.19026470588237, + "test_conv2d_binary_inplace_fusion_failed_cpu_cpp_wrapper (__main__.TestCppWrapper)": 75.7442, + "test_conv3d_binary_dynamic_shapes (__main__.TestDynamicPatternMatcher)": 113.2255238095238, + "test_conv3d_unary_dynamic_shapes (__main__.TestDynamicPatternMatcher)": 71.51290476190476, + "test_conv_freezing_non_abi_compatible_cuda (__main__.AOTInductorTestNonABICompatibleCuda)": 73.27433333333333, + "test_conv_transpose2d_packed_cpu_dynamic_shapes_cpp_wrapper (__main__.DynamicShapesCppWrapperCpuTests)": 62.88960000000001, + "test_correctness_NAdam_use_closure_True_cuda_float32 (__main__.CompiledOptimizerParityTestsCUDA)": 66.9665, + "test_cusparse_multiple_threads_same_device (__main__.TestCuda)": 93.28726315789474, + "test_custom_module_lstm (__main__.TestQuantizedOps)": 81.82516, + "test_ddp_model_diff_shape_across_ranks (__main__.TestDistBackendWithSpawn)": 91.468, + "test_ddp_uneven_inputs (__main__.TestDistBackendWithSpawn)": 538.063, + "test_diff_hyperparams_sharding_strategy_str_full_shard (__main__.TestFSDPUseOrigParamsMultipleParamGroups)": 70.84433333333332, + "test_diff_hyperparams_sharding_strategy_str_no_shard (__main__.TestFSDPUseOrigParamsMultipleParamGroups)": 69.47433333333333, + "test_diff_hyperparams_sharding_strategy_str_shard_grad_op (__main__.TestFSDPUseOrigParamsMultipleParamGroups)": 67.11266666666667, + "test_dispatch_symbolic_meta_outplace_all_strides_nn_functional_gaussian_nll_loss_cuda_float32 (__main__.TestMetaCUDA)": 113.79488888888889, + "test_dtypeview_cpu (__main__.CpuTests)": 84.04108000000001, + "test_dtypeview_cuda_cuda_wrapper (__main__.TestCudaWrapper)": 279.51483333333334, + "test_dtypeview_cuda_dynamic_shapes_cuda_wrapper (__main__.DynamicShapesCudaWrapperCudaTests)": 283.54316666666665, + "test_dtypeview_dynamic_shapes_cpu (__main__.DynamicShapesCodegenCpuTests)": 96.16909523809525, + "test_dtypeview_dynamic_shapes_cpu (__main__.DynamicShapesCpuTests)": 97.33604761904763, + "test_fail_creation_ops.py (__main__.TestTyping)": 65.19961538461537, + "test_fail_random.py (__main__.TestTyping)": 63.64657575757575, + "test_fake_tensor_mode_huggingface_databricks_dolly_v2_3b (__main__.TORCH_EXPORT_EXPORTEDPROGRAM)": 96.72200000000001, + "test_fake_tensor_mode_huggingface_google_t5 (__main__.TORCH_EXPORT_EXPORTEDPROGRAM)": 108.259, + "test_fake_tensor_mode_huggingface_google_t5 (__main__.TORCH_NN_MODULE)": 209.738, + "test_fn_fwgrad_bwgrad_cumprod_cuda_complex128 (__main__.TestFwdGradientsCUDA)": 100.12588888888891, + "test_fn_gradgrad_cumprod_cuda_complex128 (__main__.TestBwdGradientsCUDA)": 108.84245454545454, + "test_fn_gradgrad_map_nested_cpu_float64 (__main__.TestBwdGradientsCPU)": 81.5349411764706, + "test_fn_gradgrad_map_nested_cuda_float64 (__main__.TestBwdGradientsCUDA)": 78.576, + "test_fn_gradgrad_map_triple_nested_cpu_float64 (__main__.TestBwdGradientsCPU)": 514.4126666666667, + "test_fn_gradgrad_map_triple_nested_cuda_float64 (__main__.TestBwdGradientsCUDA)": 398.1390909090909, + "test_fn_gradgrad_ormqr_cuda_complex128 (__main__.TestBwdGradientsCUDA)": 70.3155, + "test_fuse_large_params_cpu (__main__.CpuTests)": 101.38141666666665, + "test_fuse_large_params_dynamic_shapes_cpu (__main__.DynamicShapesCodegenCpuTests)": 142.97099999999998, + "test_fuse_large_params_dynamic_shapes_cpu (__main__.DynamicShapesCpuTests)": 144.0665238095238, + "test_fuse_large_params_dynamic_shapes_cuda (__main__.DynamicShapesCodegenGPUTests)": 73.9485, + "test_grad_nn_Transformer_cuda_float64 (__main__.TestModuleCUDA)": 97.20400000000001, + "test_gradgrad_nn_TransformerDecoderLayer_cuda_float64 (__main__.TestModuleCUDA)": 230.74563636363638, + "test_gradgrad_nn_TransformerEncoder_eval_mode_cuda_float64 (__main__.TestModuleCUDA)": 130.09063636363638, + "test_gradgrad_nn_TransformerEncoder_train_mode_cuda_float64 (__main__.TestModuleCUDA)": 152.54545454545453, + "test_grid_sampler_2d_cpu (__main__.CpuHalideTests)": 191.693, + "test_inplace_gradgrad_cumprod_cuda_complex128 (__main__.TestBwdGradientsCUDA)": 105.473, + "test_jit_cuda_archflags (__main__.TestCppExtensionJIT)": 136.84644444444444, + "test_linear_binary_cpp_wrapper (__main__.TestCppWrapper)": 382.9056, + "test_linear_binary_dynamic_shapes_cpp_wrapper (__main__.DynamicShapesCppWrapperCpuTests)": 403.69230000000005, + "test_linear_dynamic_shapes_batch_size_1000_in_features_1000_out_features_1024_bias_False_input_3d_False_cpu_float16 (__main__.TestSelectAlgorithmDynamicShapesCPU)": 66.2571, + "test_linear_dynamic_shapes_batch_size_1000_in_features_1000_out_features_1024_bias_False_input_3d_True_cpu_float16 (__main__.TestSelectAlgorithmDynamicShapesCPU)": 131.267, + "test_linear_dynamic_shapes_batch_size_1000_in_features_1000_out_features_1024_bias_True_input_3d_False_cpu_float16 (__main__.TestSelectAlgorithmDynamicShapesCPU)": 69.4666, + "test_linear_dynamic_shapes_batch_size_1000_in_features_1000_out_features_1024_bias_True_input_3d_True_cpu_float16 (__main__.TestSelectAlgorithmDynamicShapesCPU)": 134.5207, + "test_linear_packed_cpp_wrapper (__main__.TestCppWrapper)": 198.969, + "test_linear_packed_dynamic_shapes_cpp_wrapper (__main__.DynamicShapesCppWrapperCpuTests)": 206.356, + "test_linear_static_shapes_batch_size_1000_in_features_1000_out_features_1024_bias_False_input_3d_False_cpu_float16 (__main__.TestSelectAlgorithmCPU)": 67.953, + "test_linear_static_shapes_batch_size_1000_in_features_1000_out_features_1024_bias_False_input_3d_True_cpu_float16 (__main__.TestSelectAlgorithmCPU)": 128.8496, + "test_linear_static_shapes_batch_size_1000_in_features_1000_out_features_1024_bias_True_input_3d_False_cpu_float16 (__main__.TestSelectAlgorithmCPU)": 71.0599, + "test_linear_static_shapes_batch_size_1000_in_features_1000_out_features_1024_bias_True_input_3d_True_cpu_float16 (__main__.TestSelectAlgorithmCPU)": 132.49540000000005, + "test_lstm_cpu (__main__.TestMkldnnCPU)": 94.47895454545456, + "test_lstm_packed_change_input_sizes_cpu_cpp_wrapper (__main__.TestCppWrapper)": 62.3839, + "test_lstm_packed_change_input_sizes_cpu_dynamic_shapes_cpp_wrapper (__main__.DynamicShapesCppWrapperCpuTests)": 60.9987, + "test_max_autotune_cutlass_backend_addmm_dynamic_False_max_autotune_gemm_backends_ATen,Triton,CUTLASS (__main__.TestCutlassBackend)": 81.91425, + "test_missing_cubin_non_abi_compatible_cuda (__main__.AOTInductorTestNonABICompatibleCuda)": 76.22216666666667, + "test_pipeline_order_flex_and_zero_bubble_ScheduleClass0 (__main__.TestSchedulePlan)": 76.18842857142859, + "test_python_ref__refs_special_zeta_cuda_float64 (__main__.TestCommonCUDA)": 64.07560000000001, + "test_python_ref_executor__refs_special_zeta_executor_aten_cuda_float64 (__main__.TestCommonCUDA)": 111.9, + "test_python_ref_torch_fallback__refs_special_zeta_cuda_float64 (__main__.TestCommonCUDA)": 62.56008333333333, + "test_qat_conv2d_unary (__main__.TestQuantizePT2EX86Inductor)": 165.02692, + "test_qat_conv_bn_fusion_cuda (__main__.TestQuantizePT2EQAT_ConvBn1d)": 64.07754545454546, + "test_qat_conv_bn_fusion_cuda (__main__.TestQuantizePT2EQAT_ConvBn2d)": 63.76154545454545, + "test_qat_conv_bn_fusion_no_conv_bias (__main__.TestQuantizePT2EQAT_ConvBn1d)": 75.56493617021276, + "test_qat_conv_bn_fusion_no_conv_bias (__main__.TestQuantizePT2EQAT_ConvBn2d)": 74.55095744680851, + "test_qat_conv_bn_relu_fusion_cuda (__main__.TestQuantizePT2EQAT_ConvBn1d)": 62.93418181818183, + "test_qat_conv_bn_relu_fusion_cuda (__main__.TestQuantizePT2EQAT_ConvBn2d)": 64.67954545454546, + "test_qat_mobilenet_v2 (__main__.TestQuantizePT2EQATModels)": 206.24339393939394, + "test_qat_resnet18 (__main__.TestQuantizePT2EQATModels)": 68.29046153846154, + "test_qlinear_add_cpu (__main__.TestPatternMatcher)": 70.38814285714287, + "test_qlinear_add_cpu_cpp_wrapper (__main__.TestCppWrapper)": 504.9821999999999, + "test_qlinear_add_cpu_dynamic_shapes_cpp_wrapper (__main__.DynamicShapesCppWrapperCpuTests)": 528.0255, + "test_qlinear_add_int8_mixed_bf16 (__main__.TestPatternMatcher)": 153.60283333333334, + "test_qlinear_add_relu_cpu (__main__.TestPatternMatcher)": 71.7899523809524, + "test_qlinear_add_relu_cpu_cpp_wrapper (__main__.TestCppWrapper)": 512.7648999999999, + "test_qlinear_add_relu_cpu_dynamic_shapes_cpp_wrapper (__main__.DynamicShapesCppWrapperCpuTests)": 530.9839, + "test_qlinear_add_relu_int8_mixed_bf16 (__main__.TestPatternMatcher)": 157.79833333333332, + "test_qlinear_gelu_cpu_cpp_wrapper (__main__.TestCppWrapper)": 61.369600000000005, + "test_qlinear_gelu_cpu_dynamic_shapes_cpp_wrapper (__main__.DynamicShapesCppWrapperCpuTests)": 61.51380000000001, + "test_quick_core_backward__unsafe_masked_index_cpu_float64 (__main__.TestDecompCPU)": 389.8819, + "test_quick_core_backward__unsafe_masked_index_cuda_float64 (__main__.TestDecompCUDA)": 888.4723333333333, + "test_quick_core_backward__unsafe_masked_index_put_accumulate_cpu_float64 (__main__.TestDecompCPU)": 578.2715000000001, + "test_quick_core_backward__unsafe_masked_index_put_accumulate_cuda_float64 (__main__.TestDecompCUDA)": 1291.462111111111, + "test_quick_core_backward_expand_copy_cuda_float64 (__main__.TestDecompCUDA)": 92.35, + "test_quick_core_backward_roll_cpu_float64 (__main__.TestDecompCPU)": 97.31772727272727, + "test_quick_core_backward_roll_cuda_float64 (__main__.TestDecompCUDA)": 212.4562222222222, + "test_quick_core_backward_select_scatter_cpu_float64 (__main__.TestDecompCPU)": 65.85936363636364, + "test_quick_core_backward_select_scatter_cuda_float64 (__main__.TestDecompCUDA)": 151.2542222222222, + "test_quick_core_backward_split_cuda_float64 (__main__.TestDecompCUDA)": 79.787, + "test_quick_core_backward_split_with_sizes_copy_cpu_float64 (__main__.TestDecompCPU)": 76.91172727272728, + "test_quick_core_backward_split_with_sizes_copy_cuda_float64 (__main__.TestDecompCUDA)": 173.51111111111112, + "test_quick_core_backward_std_cuda_float64 (__main__.TestDecompCUDA)": 117.01122222222222, + "test_rnn_decomp_module_nn_LSTM_train_mode_cuda_float32 (__main__.TestDecompCUDA)": 72.3970909090909, + "test_save_load_large_string_attribute (__main__.TestSaveLoad)": 128.59333333333333, + "test_shuffler_iterdatapipe (__main__.IntegrationTestDataLoaderDataPipe)": 111.14317647058824, + "test_sum_all_cpu_float64 (__main__.TestReductionsCPU)": 161.41700000000003, + "test_svd_lowrank_cuda_complex128 (__main__.TestLinalgCUDA)": 306.7471111111111, + "test_svd_lowrank_cuda_float64 (__main__.TestLinalgCUDA)": 70.03777777777779, + "test_terminate_handler_on_crash (__main__.TestTorch)": 68.968, + "test_terminate_signal (__main__.ForkTest)": 144.6801515151515, + "test_terminate_signal (__main__.SpawnTest)": 135.16911764705878, + "test_train_parity_multi_group (__main__.TestFullyShard1DTrainingCore)": 88.26866666666666, + "test_transpose_copy (__main__.CPUReproTests)": 62.54614285714285, + "test_triton_bsr_scatter_mm_blocksize_32_cuda_bfloat16 (__main__.TestSparseCompressedTritonKernelsCUDA)": 119.48249999999999, + "test_triton_bsr_scatter_mm_blocksize_64_cuda_bfloat16 (__main__.TestSparseCompressedTritonKernelsCUDA)": 71.85900000000001, + "test_triton_bsr_scatter_mm_blocksize_64_cuda_float32 (__main__.TestSparseCompressedTritonKernelsCUDA)": 87.0915, + "test_unary_ops (__main__.TestTEFuserDynamic)": 119.30365384615385, + "test_unary_ops (__main__.TestTEFuserStatic)": 90.27661538461538, + "test_unspec_inputs_cuda_cuda_wrapper (__main__.TestCudaWrapper)": 84.24216666666666, + "test_unspec_inputs_cuda_dynamic_shapes_cuda_wrapper (__main__.DynamicShapesCudaWrapperCudaTests)": 83.43050000000001, + "test_upsample_bicubic2d_cpu (__main__.CpuHalideTests)": 96.144, + "test_variant_consistency_jit_nn_functional_max_pool2d_cpu_float32 (__main__.TestJitCPU)": 106.12053333333334, + "test_variant_consistency_jit_nn_functional_max_pool2d_cuda_float32 (__main__.TestJitCUDA)": 105.8185, + "test_vec_compare_op_cpu_only (__main__.CPUReproTests)": 71.327, + "test_verify_model_across_rank_with_logger (__main__.TestDistBackendWithSpawn)": 61.44333333333333, + "test_verify_model_across_rank_without_logger (__main__.TestDistBackendWithSpawn)": 61.16233333333334, + "test_vmapjvpvjp_diff_cuda_float32 (__main__.TestOperatorsCUDA)": 81.9345, + "test_vmapjvpvjp_linalg_lu_solve_cpu_float32 (__main__.TestOperatorsCPU)": 62.15947826086957, + "test_vmapjvpvjp_linalg_lu_solve_cuda_float32 (__main__.TestOperatorsCUDA)": 86.87155555555556, + "test_vmapjvpvjp_linalg_multi_dot_cuda_float32 (__main__.TestOperatorsCUDA)": 84.2345, + "test_vmapjvpvjp_linalg_solve_triangular_cuda_float32 (__main__.TestOperatorsCUDA)": 83.042, + "test_vmapjvpvjp_linalg_svd_cuda_float32 (__main__.TestOperatorsCUDA)": 91.31800000000001, + "test_vmapjvpvjp_max_pool2d_with_indices_backward_cpu_float32 (__main__.TestOperatorsCPU)": 84.47900000000003, + "test_vmapjvpvjp_max_pool2d_with_indices_backward_cuda_float32 (__main__.TestOperatorsCUDA)": 111.041, + "test_vmapjvpvjp_nn_functional_conv2d_cpu_float32 (__main__.TestOperatorsCPU)": 70.18206666666667, + "test_vmapjvpvjp_nn_functional_conv2d_cuda_float32 (__main__.TestOperatorsCUDA)": 71.4435, + "test_vmapjvpvjp_nn_functional_max_pool1d_cuda_float32 (__main__.TestOperatorsCUDA)": 65.864, + "test_vmapjvpvjp_nn_functional_max_pool2d_cpu_float32 (__main__.TestOperatorsCPU)": 78.47160000000001, + "test_vmapjvpvjp_nn_functional_max_pool2d_cuda_float32 (__main__.TestOperatorsCUDA)": 108.5055, + "test_vmapjvpvjp_svd_cuda_float32 (__main__.TestOperatorsCUDA)": 78.44033333333334, + "test_vmapjvpvjp_unbind_cpu_float32 (__main__.TestOperatorsCPU)": 61.437625000000004, + "test_vmapjvpvjp_unbind_cuda_float32 (__main__.TestOperatorsCUDA)": 113.4555, + "test_vmapvjpvjp_meshgrid_list_of_tensors_cuda_float32 (__main__.TestOperatorsCUDA)": 111.5335, + "test_vmapvjpvjp_meshgrid_variadic_tensors_cuda_float32 (__main__.TestOperatorsCUDA)": 117.1695, + "test_vmapvjpvjp_nn_functional_bilinear_cuda_float32 (__main__.TestOperatorsCUDA)": 146.6571111111111 +} \ No newline at end of file diff --git a/tools/stats/import_test_stats.py b/tools/stats/import_test_stats.py index 013fb7ddeba..57ceb9b8b49 100644 --- a/tools/stats/import_test_stats.py +++ b/tools/stats/import_test_stats.py @@ -21,7 +21,6 @@ def get_disabled_issues() -> list[str]: return issue_numbers -SLOW_TESTS_FILE = ".pytorch-slow-tests.json" DISABLED_TESTS_FILE = ".pytorch-disabled-tests.json" ADDITIONAL_CI_FILES_FOLDER = Path(".additional_ci_files") TEST_TIMES_FILE = "test-times.json" @@ -77,17 +76,6 @@ def fetch_and_cache( return {} -def get_slow_tests( - dirpath: str, filename: str = SLOW_TESTS_FILE -) -> dict[str, float] | None: - url = "https://ossci-metrics.s3.amazonaws.com/slow-tests.json" - try: - return fetch_and_cache(dirpath, filename, url, lambda x: x) - except Exception: - print("Couldn't download slow test set, leaving all tests enabled...") - return {} - - def get_test_times() -> dict[str, dict[str, float]]: return get_from_test_infra_generated_stats( "test-times.json", diff --git a/tools/testing/test_selections.py b/tools/testing/test_selections.py index 5aff1120c60..d59b653655f 100644 --- a/tools/testing/test_selections.py +++ b/tools/testing/test_selections.py @@ -6,7 +6,7 @@ import subprocess from pathlib import Path from typing import Callable, Sequence -from tools.stats.import_test_stats import get_disabled_tests, get_slow_tests +from tools.stats.import_test_stats import get_disabled_tests from tools.testing.test_run import ShardedTest, TestRun @@ -258,5 +258,4 @@ def calculate_shards( def get_test_case_configs(dirpath: str) -> None: - get_slow_tests(dirpath=dirpath) get_disabled_tests(dirpath=dirpath) diff --git a/tools/testing/update_slow_tests.py b/tools/testing/update_slow_tests.py new file mode 100644 index 00000000000..ad00f28f23a --- /dev/null +++ b/tools/testing/update_slow_tests.py @@ -0,0 +1,208 @@ +import json +import os +import subprocess +import time +from pathlib import Path +from typing import Any, cast, Dict, Optional, Tuple + +import requests +import rockset # type: ignore[import] + + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent +QUERY = """ +WITH most_recent_strict_commits AS ( + SELECT + push.head_commit.id as sha, + FROM + commons.push + WHERE + push.ref = 'refs/heads/viable/strict' + AND push.repository.full_name = 'pytorch/pytorch' + ORDER BY + push._event_time DESC + LIMIT + 3 +), workflows AS ( + SELECT + id + FROM + commons.workflow_run w + INNER JOIN most_recent_strict_commits c on w.head_sha = c.sha + WHERE + w.name != 'periodic' +), +job AS ( + SELECT + j.id + FROM + commons.workflow_job j + INNER JOIN workflows w on w.id = j.run_id + WHERE + j.name NOT LIKE '%asan%' +), +duration_per_job AS ( + SELECT + test_run.classname, + test_run.name, + job.id, + SUM(time) as time + FROM + commons.test_run_s3 test_run + /* `test_run` is ginormous and `job` is small, so lookup join is essential */ + INNER JOIN job ON test_run.job_id = job.id HINT(join_strategy = lookup) + WHERE + /* cpp tests do not populate `file` for some reason. */ + /* Exclude them as we don't include them in our slow test infra */ + test_run.file IS NOT NULL + /* do some more filtering to cut down on the test_run size */ + AND test_run.skipped IS NULL + AND test_run.failure IS NULL + AND test_run.error IS NULL + GROUP BY + test_run.classname, + test_run.name, + job.id +) +SELECT + CONCAT( + name, + ' (__main__.', + classname, + ')' + ) as test_name, + AVG(time) as avg_duration_sec +FROM + duration_per_job +GROUP BY + CONCAT( + name, + ' (__main__.', + classname, + ')' + ) +HAVING + AVG(time) > 60.0 +ORDER BY + test_name +""" + + +UPDATEBOT_TOKEN = os.environ["UPDATEBOT_TOKEN"] +PYTORCHBOT_TOKEN = os.environ["PYTORCHBOT_TOKEN"] + + +def git_api( + url: str, params: Dict[str, str], type: str = "get", token: str = UPDATEBOT_TOKEN +) -> Any: + headers = { + "Accept": "application/vnd.github.v3+json", + "Authorization": f"token {token}", + } + if type == "post": + return requests.post( + f"https://api.github.com{url}", + data=json.dumps(params), + headers=headers, + ).json() + elif type == "patch": + return requests.patch( + f"https://api.github.com{url}", + data=json.dumps(params), + headers=headers, + ).json() + else: + return requests.get( + f"https://api.github.com{url}", + params=params, + headers=headers, + ).json() + + +def make_pr(source_repo: str, params: Dict[str, Any]) -> int: + response = git_api(f"/repos/{source_repo}/pulls", params, type="post") + print(f"made pr {response['html_url']}") + return cast(int, response["number"]) + + +def approve_pr(source_repo: str, pr_number: int) -> None: + params = {"event": "APPROVE"} + # use pytorchbot to approve the pr + git_api( + f"/repos/{source_repo}/pulls/{pr_number}/reviews", + params, + type="post", + token=PYTORCHBOT_TOKEN, + ) + + +def make_comment(source_repo: str, pr_number: int, msg: str) -> None: + params = {"body": msg} + # comment with pytorchbot because pytorchmergebot gets ignored + git_api( + f"/repos/{source_repo}/issues/{pr_number}/comments", + params, + type="post", + token=PYTORCHBOT_TOKEN, + ) + + +def search_for_open_pr( + source_repo: str, search_string: str +) -> Optional[Tuple[int, str]]: + params = { + "q": f"is:pr is:open in:title author:pytorchupdatebot repo:{source_repo} {search_string}", + "sort": "created", + } + response = git_api("/search/issues", params) + if response["total_count"] != 0: + # pr does exist + pr_num = response["items"][0]["number"] + link = response["items"][0]["html_url"] + response = git_api(f"/repos/{source_repo}/pulls/{pr_num}", {}) + branch_name = response["head"]["ref"] + print( + f"pr does exist, number is {pr_num}, branch name is {branch_name}, link is {link}" + ) + return pr_num, branch_name + return None + + +if __name__ == "__main__": + rs_client = rockset.RocksetClient( + host="api.usw2a1.rockset.com", api_key=os.environ["ROCKSET_API_KEY"] + ) + + results = rs_client.sql(QUERY).results + slow_tests = {row["test_name"]: row["avg_duration_sec"] for row in results} + + with open(REPO_ROOT / "test" / "slow_tests.json", "w") as f: + json.dump(slow_tests, f, indent=2) + + branch_name = f"update_slow_tests_{int(time.time())}" + pr_num = None + + open_pr = search_for_open_pr("pytorch/pytorch", "Update slow tests") + if open_pr is not None: + pr_num, branch_name = open_pr + + subprocess.run(["git", "checkout", "-b", branch_name], cwd=REPO_ROOT) + subprocess.run(["git", "add", "test/slow_tests.json"], cwd=REPO_ROOT) + subprocess.run(["git", "commit", "-m", "Update slow tests"], cwd=REPO_ROOT) + subprocess.run( + f"git push --set-upstream origin {branch_name} -f".split(), cwd=REPO_ROOT + ) + + params = { + "title": "Update slow tests", + "head": branch_name, + "base": "main", + "body": "This PR is auto-generated weekly by [this action](https://github.com/pytorch/pytorch/blob/main/" + + ".github/workflows/weeekly.yml).\nUpdate the list of slow tests.", + } + if pr_num is None: + # no existing pr, so make a new one and approve it + pr_num = make_pr("pytorch/pytorch", params) + time.sleep(5) + approve_pr("pytorch/pytorch", pr_num) + make_comment("pytorch/pytorch", pr_num, "@pytorchbot merge") diff --git a/torch/testing/_internal/common_utils.py b/torch/testing/_internal/common_utils.py index 0010b117eba..059c6acb13c 100644 --- a/torch/testing/_internal/common_utils.py +++ b/torch/testing/_internal/common_utils.py @@ -276,7 +276,7 @@ OPINFO_SAMPLE_INPUT_INDEX: Optional[int] = TestEnvironment.def_setting( ) DEFAULT_DISABLED_TESTS_FILE = '.pytorch-disabled-tests.json' -DEFAULT_SLOW_TESTS_FILE = '.pytorch-slow-tests.json' +DEFAULT_SLOW_TESTS_FILE = 'slow_tests.json' disabled_tests_dict = {} slow_tests_dict = {}