diff --git a/.github/actions/checkout-pytorch/action.yml b/.github/actions/checkout-pytorch/action.yml
index 055404c6947..293c35b47e2 100644
--- a/.github/actions/checkout-pytorch/action.yml
+++ b/.github/actions/checkout-pytorch/action.yml
@@ -43,6 +43,8 @@ runs:
             sudo git clean -ffdx
           else
             git clean -ffdx
+            # Do the same for submodules, should cleanup stale submodule dirs
+            git submodule foreach --recursive git clean -ffdx
           fi
         fi