Migrate all the Caffe2 Centos builds to explicity use devltoolset (#28465)

Summary:
Continues https://github.com/pytorch/pytorch/pull/28431 with a new branch name that can trigger all the CI

https://github.com/pytorch/pytorch/issues/28059

pytorch/ossci-job-dsl@b2c823a
Pull Request resolved: https://github.com/pytorch/pytorch/pull/28465

Differential Revision: D18104647

Pulled By: bddppq

fbshipit-source-id: 24decf44bdf73bd8a9c64d5fcaf34eec7a356f6e

.circleci/cimodel/data/caffe2_build_data.py

@@ -4,29 +4,29 @@ from cimodel.lib.conf_tree import Ver
 
 CONFIG_TREE_DATA = [
     (Ver("ubuntu", "16.04"), [
-        (Ver("cuda", "9.0"), [
+        ([Ver("cuda", "9.0")], [
             # TODO make explicit that this is a "secret TensorRT build"
             #  (see https://github.com/pytorch/pytorch/pull/17323#discussion_r259446749)
             # TODO Uh oh, were we supposed to make this one important?!
             X("py2"),
             XImportant("cmake"),
         ]),
-        (Ver("cuda", "10.1"), [XImportant("py3.5")]),  # TensorRT 6 build
+        ([Ver("cuda", "10.1")], [XImportant("py3.5")]),  # TensorRT 6 build
-        (Ver("mkl"), [XImportant("py2")]),
+        ([Ver("mkl")], [XImportant("py2")]),
-        (Ver("gcc", "5"), [XImportant("onnx_py2")]),
+        ([Ver("gcc", "5")], [XImportant("onnx_py2")]),
-        (Ver("clang", "3.8"), [X("py2")]),
+        ([Ver("clang", "3.8")], [X("py2")]),
-        (Ver("clang", "3.9"), [X("py2")]),
+        ([Ver("clang", "3.9")], [X("py2")]),
-        (Ver("clang", "7"), [XImportant("py2"), XImportant("onnx_py3.6")]),
+        ([Ver("clang", "7")], [XImportant("py2"), XImportant("onnx_py3.6")]),
-        (Ver("android"), [XImportant("py2")]),
+        ([Ver("android")], [XImportant("py2")]),
     ]),
     (Ver("centos", "7"), [
-        (Ver("cuda", "9.0"), [X("py2")]),
+        ([Ver("devtoolset", "7"), Ver("cuda", "9.2")], [X("py3.6")]),
     ]),
     (Ver("macos", "10.13"), [
         # TODO ios and system aren't related. system qualifies where the python comes
         #  from (use the system python instead of homebrew or anaconda)
-        (Ver("ios"), [X("py2")]),
+        ([Ver("ios")], [X("py2")]),
-        (Ver("system"), [XImportant("py2")]),
+        ([Ver("system")], [XImportant("py2")]),
     ]),
 ]
 
@@ -50,12 +50,12 @@ class TreeConfigNode(ConfigNode):
     def is_build_only(self):
         if str(self.find_prop("language_version")) == "onnx_py3.6":
             return False
-        return str(self.find_prop("compiler_version")) in [
+        return set(str(c) for c in self.find_prop("compiler_version")).intersection({
             "clang3.8",
             "clang3.9",
             "clang7",
             "android",
-        ] or self.find_prop("distro_version").name == "macos"
+        }) or self.find_prop("distro_version").name == "macos"
 
 
 class TopLevelNode(TreeConfigNode):

.circleci/cimodel/data/caffe2_build_definitions.py

@@ -12,23 +12,29 @@ from dataclasses import dataclass
 
 DOCKER_IMAGE_PATH_BASE = "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/"
 
-DOCKER_IMAGE_VERSION = 325
+DOCKER_IMAGE_VERSION = 336
 
 
 @dataclass
 class Conf:
     language: str
     distro: Ver
-    compiler: Ver
+    # There could be multiple compiler versions configured (e.g. nvcc
+    # for gpu files and host compiler (gcc/clang) for cpu files)
+    compilers: [Ver]
     build_only: bool
     is_important: bool
 
+    @property
+    def compiler_names(self):
+        return [c.name for c in self.compilers]
+
     # TODO: Eventually we can probably just remove the cudnn7 everywhere.
     def get_cudnn_insertion(self):
 
         omit = self.language == "onnx_py2" \
             or self.language == "onnx_py3.6" \
-            or self.compiler.name in ["android", "mkl", "clang"] \
+            or set(self.compiler_names).intersection({"android", "mkl", "clang"}) \
             or str(self.distro) in ["ubuntu14.04", "macos10.13"]
 
         return [] if omit else ["cudnn7"]
@@ -40,7 +46,7 @@ class Conf:
         ] + self.get_build_name_middle_parts()
 
     def get_build_name_middle_parts(self):
-        return [str(self.compiler)] + self.get_cudnn_insertion() + [str(self.distro)]
+        return [str(c) for c in self.compilers] + self.get_cudnn_insertion() + [str(self.distro)]
 
     def construct_phase_name(self, phase):
         root_parts = self.get_build_name_root_parts()
@@ -80,11 +86,11 @@ class Conf:
 
         build_env_name = "-".join(parts)
         parameters["build_environment"] = miniutils.quote(build_env_name)
-        if self.compiler.name == "ios":
+        if "ios" in self.compiler_names:
             parameters["build_ios"] = miniutils.quote("1")
         if phase == "test":
             # TODO cuda should not be considered a compiler
-            if self.compiler.name == "cuda":
+            if "cuda" in self.compiler_names:
                 parameters["use_cuda_docker_runtime"] = miniutils.quote("1")
 
         if self.distro.name != "macos":
@@ -92,7 +98,7 @@ class Conf:
             if self.build_only:
                 parameters["build_only"] = miniutils.quote("1")
         if phase == "test":
-            resource_class = "large" if self.compiler.name != "cuda" else "gpu.medium"
+            resource_class = "large" if "cuda" not in self.compiler_names else "gpu.medium"
             parameters["resource_class"] = resource_class
 
         return parameters
@@ -125,11 +131,10 @@ def instantiate_configs():
     root = get_root()
     found_configs = conf_tree.dfs(root)
     for fc in found_configs:
-
         c = Conf(
             language=fc.find_prop("language_version"),
             distro=fc.find_prop("distro_version"),
-            compiler=fc.find_prop("compiler_version"),
+            compilers=fc.find_prop("compiler_version"),
             build_only=fc.find_prop("build_only"),
             is_important=fc.find_prop("important"),
         )
@@ -143,12 +148,8 @@ def get_workflow_jobs():
 
     configs = instantiate_configs()
 
-    # TODO Why don't we build this config?
-    # See https://github.com/pytorch/pytorch/pull/17323#discussion_r259450540
-    filtered_configs = filter(lambda x: not (str(x.distro) == "ubuntu14.04" and str(x.compiler) == "gcc4.9"), configs)
-
     x = []
-    for conf_options in filtered_configs:
+    for conf_options in configs:
 
         phases = ["build"]
         if not conf_options.build_only:

.circleci/config.yml

@@ -1930,7 +1930,7 @@ workflows:
                 - master
                 - /ci-all\/.*/
           build_environment: "caffe2-py2-cuda9.0-cudnn7-ubuntu16.04-build"
-          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-ubuntu16.04:325"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-ubuntu16.04:336"
       - caffe2_linux_test:
           name: caffe2_py2_cuda9_0_cudnn7_ubuntu16_04_test
           requires:
@@ -1943,14 +1943,14 @@ workflows:
                 - /ci-all\/.*/
           build_environment: "caffe2-py2-cuda9.0-cudnn7-ubuntu16.04-test"
           use_cuda_docker_runtime: "1"
-          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-ubuntu16.04:325"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-ubuntu16.04:336"
           resource_class: gpu.medium
       - caffe2_linux_build:
           name: caffe2_cmake_cuda9_0_cudnn7_ubuntu16_04_build
           requires:
             - setup
           build_environment: "caffe2-cmake-cuda9.0-cudnn7-ubuntu16.04-build"
-          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-ubuntu16.04:325"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-ubuntu16.04:336"
       - caffe2_linux_test:
           name: caffe2_cmake_cuda9_0_cudnn7_ubuntu16_04_test
           requires:
@@ -1958,14 +1958,14 @@ workflows:
             - caffe2_cmake_cuda9_0_cudnn7_ubuntu16_04_build
           build_environment: "caffe2-cmake-cuda9.0-cudnn7-ubuntu16.04-test"
           use_cuda_docker_runtime: "1"
-          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-ubuntu16.04:325"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-ubuntu16.04:336"
           resource_class: gpu.medium
       - caffe2_linux_build:
           name: caffe2_py3_5_cuda10_1_cudnn7_ubuntu16_04_build
           requires:
             - setup
           build_environment: "caffe2-py3.5-cuda10.1-cudnn7-ubuntu16.04-build"
-          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py3.5-cuda10.1-cudnn7-ubuntu16.04:325"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py3.5-cuda10.1-cudnn7-ubuntu16.04:336"
       - caffe2_linux_test:
           name: caffe2_py3_5_cuda10_1_cudnn7_ubuntu16_04_test
           requires:
@@ -1973,35 +1973,35 @@ workflows:
             - caffe2_py3_5_cuda10_1_cudnn7_ubuntu16_04_build
           build_environment: "caffe2-py3.5-cuda10.1-cudnn7-ubuntu16.04-test"
           use_cuda_docker_runtime: "1"
-          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py3.5-cuda10.1-cudnn7-ubuntu16.04:325"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py3.5-cuda10.1-cudnn7-ubuntu16.04:336"
           resource_class: gpu.medium
       - caffe2_linux_build:
           name: caffe2_py2_mkl_ubuntu16_04_build
           requires:
             - setup
           build_environment: "caffe2-py2-mkl-ubuntu16.04-build"
-          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-mkl-ubuntu16.04:325"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-mkl-ubuntu16.04:336"
       - caffe2_linux_test:
           name: caffe2_py2_mkl_ubuntu16_04_test
           requires:
             - setup
             - caffe2_py2_mkl_ubuntu16_04_build
           build_environment: "caffe2-py2-mkl-ubuntu16.04-test"
-          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-mkl-ubuntu16.04:325"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-mkl-ubuntu16.04:336"
           resource_class: large
       - caffe2_linux_build:
           name: caffe2_onnx_py2_gcc5_ubuntu16_04_build
           requires:
             - setup
           build_environment: "caffe2-onnx-py2-gcc5-ubuntu16.04-build"
-          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-gcc5-ubuntu16.04:325"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-gcc5-ubuntu16.04:336"
       - caffe2_linux_test:
           name: caffe2_onnx_py2_gcc5_ubuntu16_04_test
           requires:
             - setup
             - caffe2_onnx_py2_gcc5_ubuntu16_04_build
           build_environment: "caffe2-onnx-py2-gcc5-ubuntu16.04-test"
-          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-gcc5-ubuntu16.04:325"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-gcc5-ubuntu16.04:336"
           resource_class: large
       - caffe2_linux_build:
           name: caffe2_py2_clang3_8_ubuntu16_04_build
@@ -2013,7 +2013,7 @@ workflows:
                 - master
                 - /ci-all\/.*/
           build_environment: "caffe2-py2-clang3.8-ubuntu16.04-build"
-          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang3.8-ubuntu16.04:325"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang3.8-ubuntu16.04:336"
           build_only: "1"
       - caffe2_linux_build:
           name: caffe2_py2_clang3_9_ubuntu16_04_build
@@ -2025,38 +2025,38 @@ workflows:
                 - master
                 - /ci-all\/.*/
           build_environment: "caffe2-py2-clang3.9-ubuntu16.04-build"
-          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang3.9-ubuntu16.04:325"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang3.9-ubuntu16.04:336"
           build_only: "1"
       - caffe2_linux_build:
           name: caffe2_py2_clang7_ubuntu16_04_build
           requires:
             - setup
           build_environment: "caffe2-py2-clang7-ubuntu16.04-build"
-          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang7-ubuntu16.04:325"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-clang7-ubuntu16.04:336"
           build_only: "1"
       - caffe2_linux_build:
           name: caffe2_onnx_py3_6_clang7_ubuntu16_04_build
           requires:
             - setup
           build_environment: "caffe2-onnx-py3.6-clang7-ubuntu16.04-build"
-          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py3.6-clang7-ubuntu16.04:325"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py3.6-clang7-ubuntu16.04:336"
       - caffe2_linux_test:
           name: caffe2_onnx_py3_6_clang7_ubuntu16_04_test
           requires:
             - setup
             - caffe2_onnx_py3_6_clang7_ubuntu16_04_build
           build_environment: "caffe2-onnx-py3.6-clang7-ubuntu16.04-test"
-          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py3.6-clang7-ubuntu16.04:325"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py3.6-clang7-ubuntu16.04:336"
           resource_class: large
       - caffe2_linux_build:
           name: caffe2_py2_android_ubuntu16_04_build
           requires:
             - setup
           build_environment: "caffe2-py2-android-ubuntu16.04-build"
-          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-android-ubuntu16.04:325"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-android-ubuntu16.04:336"
           build_only: "1"
       - caffe2_linux_build:
-          name: caffe2_py2_cuda9_0_cudnn7_centos7_build
+          name: caffe2_py3_6_devtoolset7_cuda9_2_cudnn7_centos7_build
           requires:
             - setup
           filters:
@@ -2064,21 +2064,21 @@ workflows:
               only:
                 - master
                 - /ci-all\/.*/
-          build_environment: "caffe2-py2-cuda9.0-cudnn7-centos7-build"
+          build_environment: "caffe2-py3.6-devtoolset7-cuda9.2-cudnn7-centos7-build"
-          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-centos7:325"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py3.6-devtoolset7-cuda9.2-cudnn7-centos7:336"
       - caffe2_linux_test:
-          name: caffe2_py2_cuda9_0_cudnn7_centos7_test
+          name: caffe2_py3_6_devtoolset7_cuda9_2_cudnn7_centos7_test
           requires:
             - setup
-            - caffe2_py2_cuda9_0_cudnn7_centos7_build
+            - caffe2_py3_6_devtoolset7_cuda9_2_cudnn7_centos7_build
           filters:
             branches:
               only:
                 - master
                 - /ci-all\/.*/
-          build_environment: "caffe2-py2-cuda9.0-cudnn7-centos7-test"
+          build_environment: "caffe2-py3.6-devtoolset7-cuda9.2-cudnn7-centos7-test"
           use_cuda_docker_runtime: "1"
-          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py2-cuda9.0-cudnn7-centos7:325"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/py3.6-devtoolset7-cuda9.2-cudnn7-centos7:336"
           resource_class: gpu.medium
       - caffe2_macos_build:
           name: caffe2_py2_ios_macos10_13_build

.circleci/scripts/should_run_job.py

@@ -32,6 +32,8 @@ default_set = set([
     'caffe2-py2-clang7-ubuntu16.04',
     # Caffe2 CMake
     'caffe2-cmake-cuda9.0-cudnn7-ubuntu16.04',
+    # Caffe2 CentOS
+    'caffe2-py3.6-devtoolset7-cuda9.0-cudnn7-centos7',
 
     # Binaries
     'manywheel 2.7mu cpu devtoolset7',

.jenkins/caffe2/build.sh

@@ -4,16 +4,6 @@ set -ex
 
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
-if [[ "$BUILD_ENVIRONMENT" == *centos7* ]]; then
-  # CentOS has gcc 4 but we need a newer compiler. Upgrade it.
-  sudo yum install -y centos-release-scl
-  sudo yum install -y devtoolset-4\*
-  sudo yum install -y scl-utils
-
-  source scl_source enable devtoolset-4 || yes
-  gcc --version
-fi
-
 # CMAKE_ARGS are only passed to 'cmake' and the -Dfoo=bar does not work with
 # setup.py, so we build a list of foo=bars and then either convert it to
 # -Dfoo=bars or export them before running setup.py

.jenkins/caffe2/test.sh

@@ -104,8 +104,13 @@ fi
 echo "Running Python tests.."
 if [[ "$BUILD_ENVIRONMENT" == *py3* ]]; then
   # locale setting is required by click package with py3
-  export LC_ALL=C.UTF-8
+  for loc in "en_US.utf8" "C.UTF-8"; do
-  export LANG=C.UTF-8
+    if locale -a | grep "$loc" >/dev/null 2>&1; then
+      export LC_ALL="$loc"
+      export LANG="$loc"
+      break;
+    fi
+  done
 fi
 
 pip install --user pytest-sugar

docker/caffe2/jenkins/centos-cuda/Dockerfile

@@ -8,6 +8,13 @@ ARG EC2
 ADD ./install_base.sh install_base.sh
 RUN bash ./install_base.sh && rm install_base.sh
 
+# Install devtoolset
+ARG DEVTOOLSET_VERSION
+ADD ./install_devtoolset.sh install_devtoolset.sh
+RUN bash ./install_devtoolset.sh
+RUN rm install_devtoolset.sh
+ENV BASH_ENV "/etc/profile"
+
 # Compile/install ccache for faster builds
 ADD ./install_ccache.sh install_ccache.sh
 RUN bash ./install_ccache.sh && rm install_ccache.sh

docker/caffe2/jenkins/centos/Dockerfile

@@ -6,6 +6,13 @@ ARG EC2
 ADD ./install_base.sh install_base.sh
 RUN bash ./install_base.sh && rm install_base.sh
 
+# Install devtoolset
+ARG DEVTOOLSET_VERSION
+ADD ./install_devtoolset.sh install_devtoolset.sh
+RUN bash ./install_devtoolset.sh
+RUN rm install_devtoolset.sh
+ENV BASH_ENV "/etc/profile"
+
 # Compile/install ccache for faster builds
 ADD ./install_ccache.sh install_ccache.sh
 RUN bash ./install_ccache.sh && rm install_ccache.sh

docker/caffe2/jenkins/common/install_ccache.sh

@@ -2,19 +2,6 @@
 
 set -ex
 
-# Install ccache from source.
-# Needs specific branch to work with nvcc (ccache/ccache#145)
-# Also pulls in a commit that disables documentation generation,
-# as this requires asciidoc to be installed (which pulls in a LOT of deps).
-pushd /tmp
-git clone https://github.com/pietern/ccache -b ccbin
-pushd ccache
-./autogen.sh
-./configure --prefix=/usr/local
-make "-j$(nproc)" install
-popd
-popd
-
 # Install sccache from pre-compiled binary.
 curl https://s3.amazonaws.com/ossci-linux/sccache -o /usr/local/bin/sccache
 chmod a+x /usr/local/bin/sccache