mirror of
https://github.com/zebrajr/pytorch.git
synced 2025-12-07 12:21:27 +01:00
73f6601cfc
* Move memory heavy tests from `test_pytorch_onnx_onnxruntime.py` to `test_models_onnxruntime.py`. The former is run in parallel in CI, while the latter is not. A change is that the moved tests are now only covered in default opset export. * Refactor and create base class for tests that export model to ONNX and verify with ONNX Runtime. The new base class are parameterized with `opset_version` and `is_script`. Further work can be done to refactor existing test classes in `test_pytorch_onnx_onnxruntime.py`. See #75630 * Reduce unnecessarily large tensor size in `test_pytorch_onnx_onnxruntime.py` to further reduce memory usage and test time. After this PR, the running time for `test_pytorch_onnx_onnxruntime.py` is reduced from `1338.82s (0:22:18)` to `225.07s (0:03:45)`, benchmarked on 10900x with `-n 10`. Fixes #79179 Pull Request resolved: https://github.com/pytorch/pytorch/pull/79640 Approved by: https://github.com/justinchuby, https://github.com/garymm
94 lines
2.8 KiB
Bash
Executable File
94 lines
2.8 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
set -ex
|
|
|
|
UNKNOWN=()
|
|
|
|
# defaults
|
|
PARALLEL=1
|
|
|
|
while [[ $# -gt 0 ]]
|
|
do
|
|
arg="$1"
|
|
case $arg in
|
|
-p|--parallel)
|
|
PARALLEL=1
|
|
shift # past argument
|
|
;;
|
|
*) # unknown option
|
|
UNKNOWN+=("$1") # save it in an array for later
|
|
shift # past argument
|
|
;;
|
|
esac
|
|
done
|
|
set -- "${UNKNOWN[@]}" # leave UNKNOWN
|
|
|
|
if [[ $PARALLEL == 1 ]]; then
|
|
pip install pytest-xdist
|
|
fi
|
|
|
|
# pytest, scipy, hypothesis: these may not be necessary
|
|
# pytest-cov: installing since `coverage run -m pytest ..` doesn't work
|
|
# parameterized: parameterizing test class
|
|
pip install pytest scipy hypothesis pytest-cov parameterized
|
|
pip install -e tools/coverage_plugins_package # allows coverage to run w/o failing due to a missing plug-in
|
|
|
|
# realpath might not be available on MacOS
|
|
script_path=$(python -c "import os; import sys; print(os.path.realpath(sys.argv[1]))" "${BASH_SOURCE[0]}")
|
|
top_dir=$(dirname $(dirname $(dirname "$script_path")))
|
|
test_paths=(
|
|
"$top_dir/test/onnx"
|
|
)
|
|
|
|
args=()
|
|
args+=("-v")
|
|
args+=("--cov")
|
|
args+=("--cov-report")
|
|
args+=("xml:test/coverage.xml")
|
|
args+=("--cov-append")
|
|
|
|
args_parallel=()
|
|
if [[ $PARALLEL == 1 ]]; then
|
|
args_parallel+=("-n")
|
|
args_parallel+=("auto")
|
|
fi
|
|
|
|
# onnxruntime only support py3
|
|
# "Python.h" not found in py2, needed by TorchScript custom op compilation.
|
|
if [[ "${SHARD_NUMBER}" == "1" ]]; then
|
|
# These exclusions are for tests that take a long time / a lot of GPU
|
|
# memory to run; they should be passing (and you will test them if you
|
|
# run them locally
|
|
pytest "${args[@]}" "${args_parallel[@]}" \
|
|
--ignore "$top_dir/test/onnx/test_pytorch_onnx_onnxruntime.py" \
|
|
--ignore "$top_dir/test/onnx/test_models_onnxruntime.py" \
|
|
--ignore "$top_dir/test/onnx/test_pytorch_onnx_onnxruntime_cuda.py" \
|
|
--ignore "$top_dir/test/onnx/test_custom_ops.py" \
|
|
--ignore "$top_dir/test/onnx/test_utility_funs.py" \
|
|
--ignore "$top_dir/test/onnx/test_models.py" \
|
|
"${test_paths[@]}"
|
|
|
|
# Heavy memory usage tests that cannot run in parallel.
|
|
pytest "${args[@]}" \
|
|
"$top_dir/test/onnx/test_models_onnxruntime.py" \
|
|
"$top_dir/test/onnx/test_custom_ops.py" \
|
|
"$top_dir/test/onnx/test_utility_funs.py" \
|
|
"$top_dir/test/onnx/test_models_onnxruntime.py" "-k" "not TestModelsONNXRuntime"
|
|
fi
|
|
|
|
if [[ "${SHARD_NUMBER}" == "2" ]]; then
|
|
# Heavy memory usage tests that cannot run in parallel.
|
|
# TODO(#79802): Parameterize test_models.py
|
|
pytest "${args[@]}" \
|
|
"$top_dir/test/onnx/test_models.py" \
|
|
"$top_dir/test/onnx/test_models_onnxruntime.py" "-k" "TestModelsONNXRuntime"
|
|
|
|
pytest "${args[@]}" "${args_parallel[@]}" \
|
|
"$top_dir/test/onnx/test_pytorch_onnx_onnxruntime.py"
|
|
fi
|
|
|
|
# Our CI expects both coverage.xml and .coverage to be within test/
|
|
if [ -d .coverage ]; then
|
|
mv .coverage test/.coverage
|
|
fi
|