Test PyTorch using python-3.8 + GCC-9 on Bionic (Reland) (#39121)

Summary:
Enable new test config in .circleci/config.yml
Skip scanning several 3rd-party packages to work around https://bugs.python.org/issue40350
Remove pre python-3.5 checks from `test.sh` and update `scikit-learn` to python-3.8 compatible version

This is a reland of https://github.com/pytorch/pytorch/pull/39030
Pull Request resolved: https://github.com/pytorch/pytorch/pull/39121

Differential Revision: D21820375

Pulled By: malfet

fbshipit-source-id: d0be79b7d204cf692e055d42b9be42402dc4c1c0

.circleci/cimodel/data/pytorch_build_data.py

@@ -60,11 +60,7 @@ CONFIG_TREE_DATA = [
             ]),
         ]),
         ("gcc", [
-            ("9", [
-                ("3.8", [
-                    ("build_only", [XImportant(True)]),
-                ]),
-            ]),
+            ("9", [XImportant("3.8")]),
         ]),
     ]),
 ]

.circleci/config.yml

@@ -2689,6 +2689,13 @@ workflows:
           name: pytorch_linux_bionic_py3_8_gcc9_build
           build_environment: "pytorch-linux-bionic-py3.8-gcc9-build"
           docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-bionic-py3.8-gcc9:209062ef-ab58-422a-b295-36c4eed6e906"
+      - pytorch_linux_test:
+          name: pytorch_linux_bionic_py3_8_gcc9_test
+          requires:
+            - pytorch_linux_bionic_py3_8_gcc9_build
+          build_environment: "pytorch-linux-bionic-py3.8-gcc9-test"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-bionic-py3.8-gcc9:209062ef-ab58-422a-b295-36c4eed6e906"
+          resource_class: large
       - pytorch_macos_10_13_py3_build:
           name: pytorch_macos_10_13_py3_build
       - pytorch_macos_10_13_py3_test:

.jenkins/pytorch/test.sh

@@ -53,25 +53,14 @@ if [[ "$BUILD_ENVIRONMENT" != *ppc64le* ]] && [[ "$BUILD_ENVIRONMENT" != *-bazel
   # TODO: Please move this to Docker
   # The version is fixed to avoid flakiness: https://github.com/pytorch/pytorch/issues/31136
   pip_install --user "hypothesis==4.53.2"
-
-  # TODO: move this to Docker
-  PYTHON_VERSION=$(python -c 'import platform; print(platform.python_version())'|cut -c1)
-  echo $PYTHON_VERSION
-  # if [[ $PYTHON_VERSION == "2" ]]; then
-  #   pip_install --user https://s3.amazonaws.com/ossci-linux/wheels/tensorboard-1.14.0a0-py2-none-any.whl
-  # else
-  #   pip_install --user https://s3.amazonaws.com/ossci-linux/wheels/tensorboard-1.14.0a0-py3-none-any.whl
-  # fi
-  pip_install --user tb-nightly
-  # mypy will fail to install on Python <3.4.  In that case,
-  # we just won't run these tests.
   # Pin MyPy version because new errors are likely to appear with each release
-  pip_install --user "mypy==0.770" || true
+  pip_install --user "mypy==0.770"
+  # Update scikit-learn to a python-3.8 compatible version
+  if [[ $(python -c "import sys; print(int(sys.version_info >= (3, 8)))") == "1" ]]; then
+    pip_install -U scikit-learn
   fi
 
-# faulthandler become built-in since 3.3
-if [[ ! $(python -c "import sys; print(int(sys.version_info >= (3, 3)))") == "1" ]]; then
-  pip_install --user faulthandler
+  pip_install --user tb-nightly
 fi
 
 # DANGER WILL ROBINSON.  The LD_PRELOAD here could cause you problems

aten/tools/valgrind.sup

@@ -32,3 +32,11 @@
    obj:*/lib*/libomp.so*
    ...
 }
+
+{
+   ignore_memmove_source_destination_overlap
+   Memcheck:Overlap
+   fun:__memcpy_chk
+   fun:cpuinfo_linux_parse_cpulist
+   ...
+}

test/run_test.py

@@ -573,6 +573,13 @@ def get_dep_modules(test):
             'urllib',
             'json',
             'collections',
+            # Modules below are excluded because they are hitting https://bugs.python.org/issue40350
+            # Trigger AttributeError: 'NoneType' object has no attribute 'is_package'
+            'mpl_toolkits',
+            'google',
+            'onnx',
+            # Triggers RecursionError
+            'mypy'
         ],
     )
     # HACK: some platforms default to ascii, so we can't just run_script :(