#include #include namespace c10d { std::vector getTensorShapes( const std::vector& tensors) { std::vector shapeTensors; shapeTensors.reserve(tensors.size()); for (const auto& tensor : tensors) { // Use `at::tensor()` to copy the data underlying `sizes()` since it may be // released elsewhere. at::Tensor shapesTensor = at::tensor(tensor.sizes(), at::TensorOptions().dtype(at::kLong)); shapeTensors.emplace_back(std::move(shapesTensor)); } return shapeTensors; } size_t getTensorsNumel(const std::vector& tensors) { size_t numel = 0; for (auto& tensor : tensors) { numel += tensor.numel(); } return numel; } void getGlobalRankStartAndStride( const std::vector& globalRanksInGroup, int& globalRankStart, int& globalRankStride) { if (globalRanksInGroup.empty()) { globalRankStart = 0; } else { globalRankStart = static_cast(globalRanksInGroup[0]); } if (globalRanksInGroup.empty()) { globalRankStride = 1; } else if (globalRanksInGroup.size() == 1) { globalRankStride = 0; } else { bool ranksAreStrided = true; auto startRank = globalRanksInGroup[0]; auto stride = globalRanksInGroup[1] - globalRanksInGroup[0]; for (std::vector::size_type i = 0; i < globalRanksInGroup.size(); i++) { if (globalRanksInGroup[i] != startRank + i * stride) { ranksAreStrided = false; break; } } if (ranksAreStrided) { globalRankStride = static_cast(globalRanksInGroup[1] - globalRanksInGroup[0]); } else { globalRankStride = -1; } } } } // namespace c10d