mirror of
https://github.com/zebrajr/pytorch.git
synced 2025-12-07 12:21:27 +01:00
634555d981
This change introduces a mechanism to test onnx export based on sample inputs registered in OpInfo, similar to how MPS and other components of pytorch are tested. It provides test coverage on ops and dtypes previously unattainable with manually created test models. This is the best way for us to discover gaps in the exporter support, especially for ops with partial existing support. This test is adapted from https://github.com/pytorch/pytorch/blob/master/test/test_mps.py This PR also - Update sqrt to support integer inputs to match pytorch behavior - Add pytest-subtests for unittest subtests support in the new test file I only enabled very few ops: `t`, `ceil` and `sqrt` because otherwise too many things will fail due to (1) unsupported dtypes in the exporter (2) unimplemented dtype support in onnxruntime (3) unexpected input to verification.verify. Subsequent PRs should improve `verification.verify` first for it to accept any legal input to a pytorch model, then incrementally fix the symbolic functions to enable more test cases. Fixes #85363 Design #88118 Pull Request resolved: https://github.com/pytorch/pytorch/pull/86182 Approved by: https://github.com/BowenBao
98 lines
3.0 KiB
Bash
Executable File
98 lines
3.0 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
set -ex
|
|
|
|
UNKNOWN=()
|
|
|
|
# defaults
|
|
PARALLEL=1
|
|
export TORCH_ONNX_EXPERIMENTAL_RUNTIME_TYPE_CHECK=ERRORS
|
|
|
|
while [[ $# -gt 0 ]]
|
|
do
|
|
arg="$1"
|
|
case $arg in
|
|
-p|--parallel)
|
|
PARALLEL=1
|
|
shift # past argument
|
|
;;
|
|
*) # unknown option
|
|
UNKNOWN+=("$1") # save it in an array for later
|
|
shift # past argument
|
|
;;
|
|
esac
|
|
done
|
|
set -- "${UNKNOWN[@]}" # leave UNKNOWN
|
|
|
|
if [[ $PARALLEL == 1 ]]; then
|
|
pip install pytest-xdist
|
|
fi
|
|
|
|
# pytest, scipy, hypothesis: these may not be necessary
|
|
# pytest-cov: installing since `coverage run -m pytest ..` doesn't work
|
|
# pytest-subtests: unittest subtests support for pytest
|
|
# parameterized: parameterizing test class
|
|
pip install pytest scipy hypothesis pytest-cov pytest-subtests parameterized
|
|
# allows coverage to run w/o failing due to a missing plug-in
|
|
pip install -e tools/coverage_plugins_package
|
|
|
|
# realpath might not be available on MacOS
|
|
script_path=$(python -c "import os; import sys; print(os.path.realpath(sys.argv[1]))" "${BASH_SOURCE[0]}")
|
|
top_dir=$(dirname $(dirname $(dirname "$script_path")))
|
|
test_paths=(
|
|
"$top_dir/test/onnx"
|
|
)
|
|
|
|
args=()
|
|
args+=("-v")
|
|
args+=("--cov")
|
|
args+=("--cov-report")
|
|
args+=("xml:test/coverage.xml")
|
|
args+=("--cov-append")
|
|
|
|
args_parallel=()
|
|
if [[ $PARALLEL == 1 ]]; then
|
|
args_parallel+=("-n")
|
|
args_parallel+=("auto")
|
|
fi
|
|
|
|
# onnxruntime only support py3
|
|
# "Python.h" not found in py2, needed by TorchScript custom op compilation.
|
|
if [[ "${SHARD_NUMBER}" == "1" ]]; then
|
|
# These exclusions are for tests that take a long time / a lot of GPU
|
|
# memory to run; they should be passing (and you will test them if you
|
|
# run them locally
|
|
pytest "${args[@]}" "${args_parallel[@]}" \
|
|
--ignore "$top_dir/test/onnx/test_pytorch_onnx_onnxruntime.py" \
|
|
--ignore "$top_dir/test/onnx/test_models_onnxruntime.py" \
|
|
--ignore "$top_dir/test/onnx/test_pytorch_onnx_onnxruntime_cuda.py" \
|
|
--ignore "$top_dir/test/onnx/test_custom_ops.py" \
|
|
--ignore "$top_dir/test/onnx/test_utility_funs.py" \
|
|
--ignore "$top_dir/test/onnx/test_models.py" \
|
|
--ignore "$top_dir/test/onnx/test_models_quantized_onnxruntime.py" \
|
|
"${test_paths[@]}"
|
|
|
|
# Heavy memory usage tests that cannot run in parallel.
|
|
pytest "${args[@]}" \
|
|
"$top_dir/test/onnx/test_custom_ops.py" \
|
|
"$top_dir/test/onnx/test_utility_funs.py" \
|
|
"$top_dir/test/onnx/test_models_onnxruntime.py" "-k" "not TestModelsONNXRuntime"
|
|
fi
|
|
|
|
if [[ "${SHARD_NUMBER}" == "2" ]]; then
|
|
# Heavy memory usage tests that cannot run in parallel.
|
|
# TODO(#79802): Parameterize test_models.py
|
|
pytest "${args[@]}" \
|
|
"$top_dir/test/onnx/test_models.py" \
|
|
"$top_dir/test/onnx/test_models_quantized_onnxruntime.py" \
|
|
"$top_dir/test/onnx/test_models_onnxruntime.py" "-k" "TestModelsONNXRuntime"
|
|
|
|
pytest "${args[@]}" "${args_parallel[@]}" \
|
|
"$top_dir/test/onnx/test_pytorch_onnx_onnxruntime.py"
|
|
fi
|
|
|
|
# Our CI expects both coverage.xml and .coverage to be within test/
|
|
if [ -d .coverage ]; then
|
|
mv .coverage test/.coverage
|
|
fi
|