diff --git a/BUILD.bazel b/BUILD.bazel
index 25148db0229..f079c76a7f7 100644
--- a/BUILD.bazel
+++ b/BUILD.bazel
@@ -551,7 +551,6 @@ cc_library(
         ":caffe2_headers",
         ":caffe2_perfkernels_avx",
         ":caffe2_perfkernels_avx2",
-        ":caffe2_perfkernels_avx512",
         "//third_party/miniz-2.1.0:miniz",
         "@com_google_protobuf//:protobuf",
         "@eigen",
@@ -651,28 +650,6 @@ cc_library(
     alwayslink = True,
 )
 
-cc_library(
-    name = "caffe2_perfkernels_avx512",
-    srcs = [
-        "caffe2/perfkernels/common_avx512.cc",
-    ],
-    hdrs = PERF_HEADERS,
-    copts = PERF_COPTS + [
-        "-mavx512f",
-        "-mavx512dq",
-        "-mavx512vl",
-        "-mavx2",
-        "-mfma",
-        "-mavx",
-    ],
-    visibility = ["//visibility:public"],
-    deps = [
-        ":caffe2_headers",
-        "//c10",
-    ],
-    alwayslink = True,
-)
-
 # torch
 torch_cuda_headers = glob(["torch/csrc/cuda/*.h"])
 
diff --git a/build.bzl b/build.bzl
index 8fd15f4e9c4..9ada64dc302 100644
--- a/build.bzl
+++ b/build.bzl
@@ -19,7 +19,6 @@ def define_targets(rules):
             "CAFFE2_BUILD_SHARED_LIBS",
             "CAFFE2_PERF_WITH_AVX",
             "CAFFE2_PERF_WITH_AVX2",
-            "CAFFE2_PERF_WITH_AVX512",
             "CAFFE2_USE_EXCEPTION_PTR",
             "CAFFE2_USE_CUDNN",
             "USE_MKLDNN",
diff --git a/caffe2/core/macros.h.in b/caffe2/core/macros.h.in
index 2497effd863..2929f105b31 100644
--- a/caffe2/core/macros.h.in
+++ b/caffe2/core/macros.h.in
@@ -11,7 +11,6 @@
 #cmakedefine CAFFE2_HAS_MKL_SGEMM_PACK
 #cmakedefine CAFFE2_PERF_WITH_AVX
 #cmakedefine CAFFE2_PERF_WITH_AVX2
-#cmakedefine CAFFE2_PERF_WITH_AVX512
 #cmakedefine CAFFE2_THREADPOOL_MAIN_IMBALANCE
 #cmakedefine CAFFE2_THREADPOOL_STATS
 #cmakedefine CAFFE2_USE_EXCEPTION_PTR
@@ -57,7 +56,6 @@
   {"HAS_MKL_SGEMM_PACK", "${CAFFE2_HAS_MKL_SGEMM_PACK}"}, \
   {"PERF_WITH_AVX", "${CAFFE2_PERF_WITH_AVX}"}, \
   {"PERF_WITH_AVX2", "${CAFFE2_PERF_WITH_AVX2}"}, \
-  {"PERF_WITH_AVX512", "${CAFFE2_PERF_WITH_AVX512}"}, \
   {"USE_EXCEPTION_PTR", "${CAFFE2_USE_EXCEPTION_PTR}"}, \
   {"USE_ACCELERATE", "${CAFFE2_USE_ACCELERATE}"}, \
   {"USE_EIGEN_FOR_BLAS", "${CAFFE2_USE_EIGEN_FOR_BLAS}"}, \
diff --git a/caffe2/perfkernels/CMakeLists.txt b/caffe2/perfkernels/CMakeLists.txt
index 21f0b10ae13..83e4a5f915d 100644
--- a/caffe2/perfkernels/CMakeLists.txt
+++ b/caffe2/perfkernels/CMakeLists.txt
@@ -10,11 +10,9 @@ endif()
 file(GLOB common_srcs *.cc)
 file(GLOB avx_srcs *_avx.cc)
 file(GLOB avx2_srcs *_avx2.cc)
-file(GLOB avx512_srcs *_avx512.cc)
-# exclude avx, avx2, and avx512 srcs from common_srcs
+# exclude avx and avx2 srcs from common_srcs
 exclude(common_srcs "${common_srcs}" ${avx_srcs})
 exclude(common_srcs "${common_srcs}" ${avx2_srcs})
-exclude(common_srcs "${common_srcs}" ${avx512_srcs})
 
 # We will always build common srcs.
 set(Caffe2_CPU_SRCS ${Caffe2_CPU_SRCS} ${common_srcs})
diff --git a/caffe2/perfkernels/common_avx512.cc b/caffe2/perfkernels/common_avx512.cc
deleted file mode 100644
index 273e14699b9..00000000000
--- a/caffe2/perfkernels/common_avx512.cc
+++ /dev/null
@@ -1,23 +0,0 @@
-// This file is here merely to check that the flags are not mixed up: for
-// example, if your compiler did not specify -mavx512f, -mavx512dq, and
-// -mavx512vl you should not provide the CAFFE2_PERF_WITH_AVX512 macro.
-
-#include "caffe2/core/common.h"
-
-#ifdef CAFFE2_PERF_WITH_AVX512
-#if !defined(__AVX512F__) || !defined(__AVX512DQ__) || !defined(__AVX512VL__)
-#error( \
-    "You found a build system error: CAFFE2_PERF_WITH_AVX512 is defined" \
-    "but __AVX512F__, __AVX512DQ__, or __AVX512VL is not defined" \
-    "(via e.g. -mavx512f, -mavx512dq, and -mavx512vl).");
-#endif
-#endif // CAFFE2_PERF_WITH_AVX512
-
-#if defined(__AVX512F__) && defined(__AVX512DQ__) && defined(__AVX512VL__)
-#ifndef CAFFE2_PERF_WITH_AVX512
-#error( \
-    "You found a build system error: __AVX512F__, __AVX512DQ__, __AVX512VL__ " \
-    "is defined (via e.g. -mavx512f, -mavx512dq, and -mavx512vl) " \
-    "but CAFFE2_PERF_WITH_AVX512 is not defined.");
-#endif // CAFFE2_PERF_WITH_AVX512
-#endif
diff --git a/cmake/MiscCheck.cmake b/cmake/MiscCheck.cmake
index 433d96ebfd2..ee8746b298a 100644
--- a/cmake/MiscCheck.cmake
+++ b/cmake/MiscCheck.cmake
@@ -70,8 +70,6 @@ CHECK_CXX_SOURCE_COMPILES(
      }" CAFFE2_COMPILER_SUPPORTS_AVX512_EXTENSIONS)
 if(CAFFE2_COMPILER_SUPPORTS_AVX512_EXTENSIONS)
   message(STATUS "Current compiler supports avx512f extension. Will build fbgemm.")
-  # Also see CMakeLists.txt under caffe2/perfkernels.
-  set(CAFFE2_PERF_WITH_AVX512 1)
 endif()
 cmake_pop_check_state()