diff --git a/.ci/pytorch/build.sh b/.ci/pytorch/build.sh
index 05d74153eef..3ba2d36624e 100755
--- a/.ci/pytorch/build.sh
+++ b/.ci/pytorch/build.sh
@@ -257,6 +257,7 @@ if [[ "$BUILD_ENVIRONMENT" == *-bazel-* ]]; then
   set -e -o pipefail
 
   get_bazel
+  python3 tools/optional_submodules.py checkout_eigen
 
   # Leave 1 CPU free and use only up to 80% of memory to reduce the change of crashing
   # the runner
diff --git a/.gitmodules b/.gitmodules
index bf1fc38c679..b22e1026d6a 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -2,10 +2,6 @@
     ignore = dirty
     path = third_party/pybind11
     url = https://github.com/pybind/pybind11.git
-[submodule "third_party/eigen"]
-    ignore = dirty
-    path = third_party/eigen
-    url = https://gitlab.com/libeigen/eigen.git
 [submodule "third_party/googletest"]
     ignore = dirty
     path = third_party/googletest
diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake
index 5e31f60a4b1..55055f44482 100644
--- a/cmake/Dependencies.cmake
+++ b/cmake/Dependencies.cmake
@@ -778,25 +778,6 @@ elseif(NOT TARGET fp16 AND USE_SYSTEM_FP16)
 endif()
 list(APPEND Caffe2_DEPENDENCY_LIBS fp16)
 
-# ---[ EIGEN
-# Due to license considerations, we will only use the MPL2 parts of Eigen.
-set(EIGEN_MPL2_ONLY 1)
-if(USE_SYSTEM_EIGEN_INSTALL)
-  find_package(Eigen3)
-  if(EIGEN3_FOUND)
-    message(STATUS "Found system Eigen at " ${EIGEN3_INCLUDE_DIR})
-  else()
-    message(STATUS "Did not find system Eigen. Using third party subdirectory.")
-    set(EIGEN3_INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR}/../third_party/eigen)
-    caffe2_update_option(USE_SYSTEM_EIGEN_INSTALL OFF)
-  endif()
-else()
-  message(STATUS "Using third party subdirectory Eigen.")
-  set(EIGEN3_INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR}/../third_party/eigen)
-endif()
-include_directories(SYSTEM ${EIGEN3_INCLUDE_DIR})
-
-
 # ---[ Python Interpreter
 # If not given a Python installation, then use the current active Python
 if(NOT Python_EXECUTABLE)
@@ -810,6 +791,29 @@ if(NOT Python_EXECUTABLE)
   endif()
 endif()
 
+
+# ---[ EIGEN
+# Due to license considerations, we will only use the MPL2 parts of Eigen.
+set(EIGEN_MPL2_ONLY 1)
+if(USE_SYSTEM_EIGEN_INSTALL)
+  find_package(Eigen3)
+  if(EIGEN3_FOUND)
+    message(STATUS "Found system Eigen at " ${EIGEN3_INCLUDE_DIR})
+  else()
+    message(STATUS "Did not find system Eigen. Using third party subdirectory.")
+    execute_process(COMMAND ${Python_EXECUTABLE} ../tools/optional_modules.py checkout_eigen
+                    WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR})
+
+    set(EIGEN3_INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR}/../third_party/eigen)
+    caffe2_update_option(USE_SYSTEM_EIGEN_INSTALL OFF)
+  endif()
+else()
+  message(STATUS "Using third party subdirectory Eigen.")
+  set(EIGEN3_INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR}/../third_party/eigen)
+endif()
+include_directories(SYSTEM ${EIGEN3_INCLUDE_DIR})
+
+
 if(BUILD_PYTHON)
   set(PYTHON_COMPONENTS Development.Module)
   if(USE_NUMPY)
diff --git a/third_party/eigen b/third_party/eigen
deleted file mode 160000
index 3147391d946..00000000000
--- a/third_party/eigen
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 3147391d946bb4b6c68edd901f2add6ac1f31f8c
diff --git a/third_party/eigen_pin.txt b/third_party/eigen_pin.txt
new file mode 100644
index 00000000000..18091983f59
--- /dev/null
+++ b/third_party/eigen_pin.txt
@@ -0,0 +1 @@
+3.4.0
diff --git a/tools/optional_submodules.py b/tools/optional_submodules.py
index b8fa1b8e816..1e7589edf2f 100644
--- a/tools/optional_submodules.py
+++ b/tools/optional_submodules.py
@@ -43,5 +43,21 @@ def checkout_nccl() -> None:
         _checkout_by_tag("https://github.com/NVIDIA/nccl", release_tag)
 
 
+def checkout_eigen() -> None:
+    eigen_tag = _read_file(third_party_path / "eigen_pin.txt")
+    print(f"-- Checkout Eigen release tag: {eigen_tag}")
+    eigen_basedir = third_party_path / "eigen"
+    if not eigen_basedir.exists():
+        _checkout_by_tag("https://gitlab.com/libeigen/eigen", eigen_tag)
+
+
 if __name__ == "__main__":
-    checkout_nccl()
+    import sys
+
+    if len(sys.argv) == 1:
+        # If no arguments are given checkout all optional dependency
+        checkout_nccl()
+        checkout_eigen()
+    else:
+        # Otherwise just call top-level function of choice
+        globals()[sys.argv[1]]()