Add a CI Job to Check BC Changes in Function Schemas (#26329)

Summary:
Ready for review, and results, please check https://circleci.com/gh/pytorch/pytorch/2827354?utm_campaign=vcs-integration-link&utm_medium=referral&utm_source=github-build-link

Also, an experimental PR stacked on this have caught bc-breaking changes introduced by it.
https://github.com/pytorch/pytorch/pull/26398
Pull Request resolved: https://github.com/pytorch/pytorch/pull/26329

Reviewed By: hl475

Differential Revision: D17485668

Pulled By: houseroad

fbshipit-source-id: b10682f1785a20ea04521992e0973b1380b4dd3b

.circleci/cimodel/data/pytorch_build_definitions.py

@@ -186,6 +186,7 @@ def instantiate_configs():
 
         distro_name = fc.find_prop("distro_name")
         compiler_name = fc.find_prop("compiler_name")
+        compiler_version = fc.find_prop("compiler_version")
         is_xla = fc.find_prop("is_xla") or False
         parms_list_ignored_for_docker_image = []
 
@@ -246,6 +247,20 @@ def instantiate_configs():
         if cuda_version == "9" and python_version == "3.6":
             c.dependent_tests = gen_dependent_configs(c)
 
+        if (compiler_name == "gcc"
+                and compiler_version == "5.4"
+                and not is_namedtensor):
+            bc_breaking_check = Conf(
+                "backward-compatibility-check",
+                [],
+                is_xla=False,
+                restrict_phases=["test"],
+                is_namedtensor=False,
+                is_important=True,
+                parent_build=c,
+            )
+            c.dependent_tests.append(bc_breaking_check)
+
         config_list.append(c)
 
     return config_list

.circleci/config.yml

@@ -1594,6 +1594,14 @@ workflows:
           build_environment: "pytorch-linux-xenial-py3.6-gcc5.4-test"
           docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:347"
           resource_class: large
+      - pytorch_linux_test:
+          name: pytorch_linux_backward_compatibility_check_test
+          requires:
+            - setup
+            - pytorch_linux_xenial_py3_6_gcc5_4_build
+          build_environment: "pytorch-linux-backward-compatibility-check-test"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:347"
+          resource_class: large
       - pytorch_linux_build:
           name: pytorch_namedtensor_linux_xenial_py3_6_gcc5_4_build
           requires:

.circleci/scripts/should_run_job.py

@@ -54,6 +54,8 @@ default_set = set([
     'pytorch-ios-10.2.1-x86_64_build',
     'pytorch-ios-10.2.1-arm64_build',
 
+    # Pytorch backward compatibility check
+    'pytorch-linux-backward-compatibility-check-test',
 
     # XLA
     'pytorch-xla-linux-xenial-py3.6-clang7',

.gitignore

@@ -37,6 +37,7 @@ test/cpp/api/mnist
 test/custom_operator/model.pt
 test/data/legacy_modules.t7
 test/data/*.pt
+test/backward_compatibility/new_schemas.txt
 dropout_model.pt
 test/generated_type_hints_smoketest.py
 test/htmlcov

.jenkins/pytorch/common.sh

@@ -161,6 +161,11 @@ function pip_install() {
   pip install --progress-bar off "$@" || pip install --progress-bar off "$@" || pip install --progress-bar off "$@"
 }
 
+function pip_uninstall() {
+  # uninstall 2 times
+  pip uninstall -y "$@" || pip uninstall -y "$@"
+}
+
 function get_exit_code() {
   set +e
   "$@"

.jenkins/pytorch/test.sh

@@ -188,10 +188,28 @@ test_xla() {
   assert_git_not_dirty
 }
 
+# Do NOT run this test before any other tests, like test_python_nn, etc.
+# Because this function uninstalls the torch built from branch, and install
+# nightly version.
+test_backward_compatibility() {
+  set -x
+  pushd test/backward_compatibility
+  python dump_all_function_schemas.py --filename new_schemas.txt
+  pip_uninstall torch
+  pip_install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
+  python check_backward_compatibility.py --new-schemas new_schemas.txt
+  popd
+  set +x
+  assert_git_not_dirty
+}
+
 (cd test && python -c "import torch; print(torch.__config__.show())")
 (cd test && python -c "import torch; print(torch.__config__.parallel_info())")
 
-if [[ "${BUILD_ENVIRONMENT}" == *xla* || "${JOB_BASE_NAME}" == *xla* ]]; then
+if [[ "${BUILD_ENVIRONMENT}" == *backward* ]]; then
+  test_backward_compatibility
+  # Do NOT add tests after bc check tests, see its comment.
+elif [[ "${BUILD_ENVIRONMENT}" == *xla* || "${JOB_BASE_NAME}" == *xla* ]]; then
   test_torchvision
   test_xla
 elif [[ "${BUILD_ENVIRONMENT}" == *-test1 || "${JOB_BASE_NAME}" == *-test1 ]]; then

test/backward_compatibility/check_backward_compatibility.py

@@ -0,0 +1,53 @@
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import argparse
+import sys
+import torch
+from torch._C import parse_schema
+
+
+def check_bc(new_schema_dict):
+    existing_schemas = torch._C._jit_get_all_schemas()
+    for existing_schema in existing_schemas:
+        print("processing existing schema: ", str(existing_schema))
+        new_schemas = new_schema_dict.get(existing_schema.name, [])
+        found = False
+        for new_schema in new_schemas:
+            if new_schema.is_backward_compatible_with(existing_schema):
+                found = True
+                break
+        if not found:
+            print('Can NOT find backward compatible schemas after changes '
+                  'for schema {} from the following candidates:\n[\n{}\n]'
+                  .format(
+                      str(existing_schema),
+                      "\n\t".join(str(s) for s in new_schemas)))
+            print('The PR is introducing backward incompatible changes to the '
+                  'operator library. Please contact PyTorch team to confirm '
+                  'whether this change is wanted or not.')
+            # TODO Print out more details about why candidates don't match.
+            return False
+    print('Found backward compatible schemas for all existing schemas')
+    return True
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Process some integers.')
+    parser.add_argument(
+        '--new-schemas',
+        help='filename to load new schemas',
+        type=str,
+        default='schemas.txt')
+    args = parser.parse_args()
+    new_schema_dict = dict()
+    with open(args.new_schemas, 'r') as f:
+        line = f.readline()
+        while line:
+            s = parse_schema(line.strip())
+            line = f.readline()
+            slist = new_schema_dict.get(s.name, [])
+            slist.append(s)
+            new_schema_dict[s.name] = slist
+
+    if not check_bc(new_schema_dict):
+        sys.exit(1)

test/backward_compatibility/dump_all_function_schemas.py

@@ -0,0 +1,24 @@
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import argparse
+import torch
+
+
+def dump(filename):
+    schemas = torch._C._jit_get_all_schemas()
+    with open(filename, 'w') as f:
+        for s in schemas:
+            f.write(str(s))
+            f.write('\n')
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Process some integers.')
+    parser.add_argument(
+        '-f',
+        '--filename',
+        help='filename to dump the schemas',
+        type=str,
+        default='schemas.txt')
+    args = parser.parse_args()
+    dump(args.filename)