OSSForks/pytorch
1
0
Fork 0
mirror of https://github.com/zebrajr/pytorch.git synced 2025-12-06 12:20:52 +01:00
Code Issues Actions 38 Packages Projects Releases Wiki Activity

Introduce checks to validate public API tests (#131390)

Browse Source 
This PR introduces a new sanity check for the public API tests in `.ci/pytorch/test.sh`.
* Validates two public API tests:
    1. Ensures `test_correct_module_names` fails when a new file OR an existing file adds an invalid public API function (e.g. one whose `__module__` is unset).
    2. Ensures `test_modules_can_be_imported` fails when a module underneath `torch/` cannot be imported.
* Runs this in CI as part just before the pre-existing FC / BC checks.

I've verified that re-introducing the bug that #131386 fixed causes the new check to fail:
![public_api_failure](https://github.com/user-attachments/assets/376ddef3-d14a-41f6-93e2-f935deb6555a)
Pull Request resolved: https://github.com/pytorch/pytorch/pull/131390
Approved by: https://github.com/albanD
This commit is contained in:
Joel Schlosser 2024-08-02 15:09:34 -04:00 committed by PyTorch MergeBot
parent fcef6cc6d1
commit 64720f3b89
1 changed files with 102 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
.ci/pytorch/test.sh
View File

@ -1040,11 +1040,113 @@ test_xla() {
assert_git_not_dirty
}
function check_public_api_test_fails {
test_name=$1
invalid_item_name=$2
invalid_item_desc=$3
echo "Running public API test '${test_name}'..."
test_output=$(python test/test_public_bindings.py -k "${test_name}" 2>&1) && ret=$? || ret=$?
# Ensure test fails correctly.
if [ "$ret" -eq 0 ]; then
cat << EOF
Expected the public API test '${test_name}' to fail after introducing
${invalid_item_desc}, but it succeeded! Check test/test_public_bindings.py
for any changes that may have broken the test.
EOF
return 1
fi
# Ensure invalid item is in the test output.
echo "${test_output}" | grep -q "${invalid_item_name}" && ret=$? || ret=$?
if [ $ret -ne 0 ]; then
cat << EOF
Expected the public API test '${test_name}' to identify ${invalid_item_desc}, but
it didn't! It's possible the test may not have run. Check test/test_public_bindings.py
for any changes that may have broken the test.
EOF
return 1
fi
echo "Success! '${test_name}' identified ${invalid_item_desc} ${invalid_item_name}."
return 0
}
# Do NOT run this test before any other tests, like test_python_shard, etc.
# Because this function uninstalls the torch built from branch and installs
# the torch built on its base commit.
test_forward_backward_compatibility() {
set -x
# First, validate public API tests in the torch built from branch.
# Step 1. Make sure the public API test "test_correct_module_names" fails when a new file
# introduces an invalid public API function.
new_filename=$(mktemp XXXXXXXX.py -p "${TORCH_INSTALL_DIR}")
BAD_PUBLIC_FUNC=$(
cat << 'EOF'
def new_public_func():
pass
# valid public API functions have __module__ set correctly
new_public_func.__module__ = None
EOF
)
echo "${BAD_PUBLIC_FUNC}" >> "${new_filename}"
invalid_api="torch.$(basename -s '.py' "${new_filename}").new_public_func"
echo "Created an invalid public API function ${invalid_api}..."
check_public_api_test_fails \
"test_correct_module_names" \
"${invalid_api}" \
"an invalid public API function" && ret=$? || ret=$?
rm -v "${new_filename}"
if [ "$ret" -ne 0 ]; then
exit 1
fi
# Step 2. Make sure that the public API test "test_correct_module_names" fails when an existing
# file is modified to introduce an invalid public API function.
EXISTING_FILEPATH="${TORCH_INSTALL_DIR}/nn/parameter.py"
cp -v "${EXISTING_FILEPATH}" "${EXISTING_FILEPATH}.orig"
echo "${BAD_PUBLIC_FUNC}" >> "${EXISTING_FILEPATH}"
invalid_api="torch.nn.parameter.new_public_func"
echo "Appended an invalid public API function to existing file ${EXISTING_FILEPATH}..."
check_public_api_test_fails \
"test_correct_module_names" \
"${invalid_api}" \
"an invalid public API function" && ret=$? || ret=$?
mv -v "${EXISTING_FILEPATH}.orig" "${EXISTING_FILEPATH}"
if [ "$ret" -ne 0 ]; then
exit 1
fi
# Step 3. Make sure that the public API test "test_modules_can_be_imported" fails when a module
# cannot be imported.
new_module_dir=$(mktemp XXXXXXXX -d -p "${TORCH_INSTALL_DIR}")
echo "invalid syntax garbage" > "${new_module_dir}/__init__.py"
invalid_module_name="torch.$(basename "${new_module_dir}")"
check_public_api_test_fails \
"test_modules_can_be_imported" \
"${invalid_module_name}" \
"a non-importable module" && ret=$? || ret=$?
rm -rv "${new_module_dir}"
if [ "$ret" -ne 0 ]; then
exit 1
fi
# Next, build torch from the merge base.
REPO_DIR=$(pwd)
if [[ "${BASE_SHA}" == "${SHA1}" ]]; then
echo "On trunk, we should compare schemas with torch built from the parent commit"