mirror of
https://github.com/zebrajr/pytorch.git
synced 2025-12-06 12:20:52 +01:00
134bce7cd0
Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/47875 Implementing several unary operators for _foreach_ APIs. ### Planned list of ops - [x] abs - [x] acos - [x] asin - [x] atan - [x] ceil - [x] cos - [x] cosh - [x] erf - [x] erfc - [x] exp - [x] expm1 - [x] floor - [x] log - [x] log10 - [x] log1p - [x] log2 - [ ] frac - [x] neg - [ ] reciprocal - [x] round - [ ] rsqrt - [ ] sigmoid - [x] sin - [x] sinh - [x] sqrt - [x] tan - [x] tanh - [ ] trunc - [x] lgamma - [ ] digamma - [ ] erfinv - [ ] sign - [ ] mvlgamma - [ ] clamp - [ ] clamp_min - [ ] clamp_max ### Perf results ``` ----------------- OP: sin ----------------- Median: 998.79 us 300.84 us ----------------- OP: abs ----------------- Median: 1.19 ms 294.97 us ----------------- OP: acos ----------------- Median: 982.30 us 299.40 us ----------------- OP: asin ----------------- Median: 1.16 ms 298.09 us ----------------- OP: atan ----------------- Median: 986.92 us 295.64 us ----------------- OP: ceil ----------------- Median: 1.17 ms 297.25 us ----------------- OP: cos ----------------- Median: 972.72 us 294.41 us ----------------- OP: cosh ----------------- Median: 1.17 ms 294.97 us ----------------- OP: erf ----------------- Median: 1.17 ms 297.02 us ----------------- OP: erfc ----------------- Median: 1.14 ms 299.23 us ----------------- OP: exp ----------------- Median: 1.15 ms 298.79 us ----------------- OP: expm1 ----------------- Median: 1.17 ms 291.79 us ----------------- OP: floor ----------------- Median: 1.17 ms 293.51 us ----------------- OP: log ----------------- Median: 1.13 ms 318.01 us ----------------- OP: log10 ----------------- Median: 987.17 us 295.57 us ----------------- OP: log1p ----------------- Median: 1.13 ms 297.15 us ----------------- OP: log2 ----------------- Median: 974.21 us 295.01 us ----------------- OP: frac ----------------- Median: 1.15 ms 296.01 us ----------------- OP: neg ----------------- Median: 1.13 ms 294.98 us ----------------- OP: reciprocal ----------------- Median: 1.16 ms 293.69 us ----------------- OP: round ----------------- Median: 1.12 ms 297.48 us ----------------- OP: sigmoid ----------------- Median: 1.13 ms 296.53 us ----------------- OP: sin ----------------- Median: 991.02 us 295.78 us ----------------- OP: sinh ----------------- Median: 1.15 ms 295.70 us ----------------- OP: sqrt ----------------- Median: 1.17 ms 297.75 us ----------------- OP: tan ----------------- 978.20 us 297.99 us ----------------- OP: tanh ----------------- Median: 967.84 us 297.29 us ----------------- OP: trunc ----------------- Median: 1.14 ms 298.72 us ----------------- OP: lgamma ----------------- Median: 1.14 ms 317.53 us ``` ### Script ``` import torch import torch.optim as optim import torch.nn as nn import torchvision import torch.utils.benchmark as benchmark_utils inputs = [torch.rand(3, 200, 200, device="cuda") for _ in range(100)] def main(): for op in [ "sin", "abs", "acos", "asin", "atan", "ceil", "cos", "cosh", "erf", "erfc", "exp", "expm1", "floor", "log", "log10", "log1p", "log2", "frac", "neg", "reciprocal", "round", "sigmoid", "sin", "sinh", "sqrt", "tan", "tanh", "trunc", "lgamma" ]: print("\n\n----------------- OP: ", op, " -----------------") stmt = "[torch.{op}(t) for t in inputs]" timer = benchmark_utils.Timer( stmt=stmt.format(op = op), globals=globals(), label="str(optimizer)", ) print(f"autorange:\n{timer.blocked_autorange()}\n\n") stmt = "torch._foreach_{op}(inputs)" timer_mta = benchmark_utils.Timer( stmt=stmt.format(op = op), globals=globals(), label="str(optimizer_mta)", ) print(f"autorange:\n{timer_mta.blocked_autorange()}\n\n") if __name__ == "__main__": main() ``` Test Plan: Imported from OSS Reviewed By: nikithamalgifb Differential Revision: D24948801 Pulled By: izdeby fbshipit-source-id: defec3c0394d6816d9a8b05a42a057348f1b4d96
918 lines
48 KiB
Python
918 lines
48 KiB
Python
import torch
|
|
import unittest
|
|
from torch.testing._internal.common_utils import TestCase, run_tests, TEST_WITH_ROCM, TEST_WITH_SLOW
|
|
from torch.testing._internal.common_device_type import instantiate_device_type_tests, dtypes, skipCUDAIfRocm
|
|
from torch._six import inf, nan
|
|
|
|
N_values = [20] if not TEST_WITH_SLOW else [30, 300]
|
|
|
|
class TestForeach(TestCase):
|
|
foreach_bin_ops = [
|
|
torch._foreach_add,
|
|
torch._foreach_sub,
|
|
torch._foreach_mul,
|
|
torch._foreach_div,
|
|
]
|
|
|
|
foreach_bin_ops_ = [
|
|
torch._foreach_add_,
|
|
torch._foreach_sub_,
|
|
torch._foreach_mul_,
|
|
torch._foreach_div_,
|
|
]
|
|
|
|
torch_bin_ops = [
|
|
torch.add,
|
|
torch.sub,
|
|
torch.mul,
|
|
torch.div,
|
|
]
|
|
|
|
unary_ops = [
|
|
# foreach_op, foreach_op_, torch_op, bf16, complex64/128
|
|
(torch._foreach_sqrt, torch._foreach_sqrt_, torch.sqrt, True , True),
|
|
(torch._foreach_exp, torch._foreach_exp_, torch.exp, True, True),
|
|
(torch._foreach_acos, torch._foreach_acos_, torch.acos, False, True),
|
|
(torch._foreach_asin, torch._foreach_asin_, torch.asin, False, True),
|
|
(torch._foreach_atan, torch._foreach_atan_, torch.atan, False, True),
|
|
(torch._foreach_cos, torch._foreach_cos_, torch.cos, True, True),
|
|
(torch._foreach_cosh, torch._foreach_cosh_, torch.cosh, False, True),
|
|
(torch._foreach_log, torch._foreach_log_, torch.log, True, True),
|
|
(torch._foreach_log10, torch._foreach_log10_, torch.log10, True, True),
|
|
(torch._foreach_log2, torch._foreach_log2_, torch.log2, True, True),
|
|
(torch._foreach_neg, torch._foreach_neg_, torch.neg, True, True),
|
|
(torch._foreach_tan, torch._foreach_tan_, torch.tan, False, True),
|
|
(torch._foreach_tanh, torch._foreach_tanh_, torch.tanh, True, True),
|
|
(torch._foreach_sin, torch._foreach_sin_, torch.sin, False, True),
|
|
(torch._foreach_sinh, torch._foreach_sinh_, torch.sinh, False, True),
|
|
(torch._foreach_ceil, torch._foreach_ceil_, torch.ceil, False, False),
|
|
(torch._foreach_erf, torch._foreach_erf_, torch.erf, True, False),
|
|
(torch._foreach_erfc, torch._foreach_erfc_, torch.erfc, False, False),
|
|
(torch._foreach_expm1, torch._foreach_expm1_, torch.expm1, False, False),
|
|
(torch._foreach_floor, torch._foreach_floor_, torch.floor, False, False),
|
|
(torch._foreach_log1p, torch._foreach_log1p_, torch.log1p, True, False),
|
|
(torch._foreach_round, torch._foreach_round_, torch.round, False, False),
|
|
|
|
# See test_abs
|
|
# (torch._foreach_abs, torch._foreach_abs_, torch.abs, True, True),
|
|
]
|
|
|
|
def _get_test_data(self, device, dtype, N):
|
|
if dtype in [torch.bfloat16, torch.bool, torch.float16]:
|
|
tensors = [torch.randn(N, N, device=device).to(dtype) for _ in range(N)]
|
|
elif dtype in torch.testing.get_all_int_dtypes():
|
|
tensors = [torch.randint(1, 100, (N, N), device=device, dtype=dtype) for _ in range(N)]
|
|
else:
|
|
tensors = [torch.randn(N, N, device=device, dtype=dtype) for _ in range(N)]
|
|
|
|
return tensors
|
|
|
|
def _test_bin_op_list(self, device, dtype, foreach_op, foreach_op_, torch_op):
|
|
for N in N_values:
|
|
tensors1 = self._get_test_data(device, dtype, N)
|
|
tensors2 = self._get_test_data(device, dtype, N)
|
|
|
|
# Mimics cuda kernel dtype flow. With fp16/bf16 input, runs in fp32 and casts output back to fp16/bf16.
|
|
control_dtype = torch.float32 if (self.device_type == 'cuda' and
|
|
(dtype is torch.float16 or dtype is torch.bfloat16)) else dtype
|
|
expected = [torch_op(tensors1[i].to(dtype=control_dtype),
|
|
tensors2[i].to(dtype=control_dtype)).to(dtype=dtype) for i in range(N)]
|
|
res = foreach_op(tensors1, tensors2)
|
|
foreach_op_(tensors1, tensors2)
|
|
self.assertEqual(res, tensors1)
|
|
if (dtype is torch.float16 or dtype is torch.bfloat16) and TEST_WITH_ROCM:
|
|
self.assertEqual(tensors1, expected, atol=1.e-3, rtol=self.dtype_precisions[dtype][0])
|
|
else:
|
|
self.assertEqual(tensors1, expected)
|
|
|
|
def _test_pointwise_op(self, device, dtype, foreach_op, foreach_op_, torch_op):
|
|
for N in N_values:
|
|
values = [2 + i for i in range(N)]
|
|
for vals in [values[0], values]:
|
|
tensors = self._get_test_data(device, dtype, N)
|
|
tensors1 = self._get_test_data(device, dtype, N)
|
|
tensors2 = self._get_test_data(device, dtype, N)
|
|
|
|
# Mimics cuda kernel dtype flow. With fp16/bf16 input, runs in fp32 and casts output back to fp16/bf16.
|
|
control_dtype = torch.float32 if (self.device_type == 'cuda' and
|
|
(dtype is torch.float16 or dtype is torch.bfloat16)) else dtype
|
|
|
|
if not isinstance(vals, list):
|
|
expected = [torch_op(tensors[i].to(dtype=control_dtype),
|
|
tensors1[i].to(dtype=control_dtype),
|
|
tensors2[i].to(dtype=control_dtype),
|
|
value=values[0]).to(dtype=dtype) for i in range(N)]
|
|
else:
|
|
expected = [torch_op(tensors[i].to(dtype=control_dtype),
|
|
tensors1[i].to(dtype=control_dtype),
|
|
tensors2[i].to(dtype=control_dtype),
|
|
value=values[i]).to(dtype=dtype) for i in range(N)]
|
|
|
|
res = foreach_op(tensors, tensors1, tensors2, vals)
|
|
foreach_op_(tensors, tensors1, tensors2, vals)
|
|
self.assertEqual(res, tensors)
|
|
|
|
if (dtype is torch.float16 or dtype is torch.bfloat16) and TEST_WITH_ROCM:
|
|
self.assertEqual(tensors, expected, atol=1.e-3, rtol=self.dtype_precisions[dtype][0])
|
|
else:
|
|
self.assertEqual(tensors, expected)
|
|
|
|
# test error cases
|
|
for op in [torch._foreach_addcmul, torch._foreach_addcmul_, torch._foreach_addcdiv, torch._foreach_addcdiv_]:
|
|
tensors = self._get_test_data(device, dtype, N)
|
|
tensors1 = self._get_test_data(device, dtype, N)
|
|
tensors2 = self._get_test_data(device, dtype, N)
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "Tensor list must have same number of elements as scalar list."):
|
|
op(tensors, tensors1, tensors2, [2 for _ in range(N + 1)])
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "Tensor list must have same number of elements as scalar list."):
|
|
op(tensors, tensors1, tensors2, [2 for _ in range(N - 1)])
|
|
|
|
tensors = self._get_test_data(device, dtype, N + 1)
|
|
with self.assertRaisesRegex(RuntimeError, "Tensor lists must have the same number of tensors, got 21 and 20"):
|
|
op(tensors, tensors1, tensors2, [2 for _ in range(N)])
|
|
|
|
tensors1 = self._get_test_data(device, dtype, N + 1)
|
|
with self.assertRaisesRegex(RuntimeError, "Tensor lists must have the same number of tensors, got 21 and 20"):
|
|
op(tensors, tensors1, tensors2, [2 for _ in range(N)])
|
|
|
|
def _test_bin_op_list_alpha(self, device, dtype, foreach_op, foreach_op_, torch_op):
|
|
for N in [30, 300]:
|
|
tensors1 = self._get_test_data(device, dtype, N)
|
|
tensors2 = self._get_test_data(device, dtype, N)
|
|
alpha = 2
|
|
|
|
# Mimics cuda kernel dtype flow. With fp16/bf16 input, runs in fp32 and casts output back to fp16/bf16.
|
|
control_dtype = torch.float32 if (self.device_type == 'cuda' and
|
|
(dtype is torch.float16 or dtype is torch.bfloat16)) else dtype
|
|
expected = [torch_op(tensors1[i].to(dtype=control_dtype),
|
|
torch.mul(tensors2[i].to(dtype=control_dtype),
|
|
alpha)).to(dtype=dtype) for i in range(N)]
|
|
res = foreach_op(tensors1, tensors2, alpha=alpha)
|
|
foreach_op_(tensors1, tensors2, alpha=alpha)
|
|
self.assertEqual(res, tensors1)
|
|
|
|
if dtype == torch.bool:
|
|
expected = [e.to(torch.bool) for e in expected]
|
|
if (dtype is torch.float16 or dtype is torch.bfloat16) and TEST_WITH_ROCM:
|
|
self.assertEqual(tensors1, expected, atol=1.e-3, rtol=self.dtype_precisions[dtype][0])
|
|
else:
|
|
self.assertEqual(tensors1, expected)
|
|
|
|
#
|
|
# Unary ops
|
|
#
|
|
@dtypes(*(torch.testing.floating_and_complex_types_and(torch.bfloat16, torch.half)))
|
|
def test_unary_ops(self, device, dtype):
|
|
for fe_op, fe_op_, torch_op, support_bfloat16, support_complex in self.unary_ops:
|
|
for N in N_values:
|
|
tensors1 = self._get_test_data(device, dtype, N)
|
|
# Mimics cuda kernel dtype flow. With fp16/bf16 input, runs in fp32 and casts output back to fp16/bf16.
|
|
control_dtype = torch.float32 if (self.device_type == 'cuda' and
|
|
(dtype is torch.float16 or dtype is torch.bfloat16)) else dtype
|
|
|
|
if self.device_type == 'cpu' and dtype == torch.half and torch_op != torch.neg:
|
|
with self.assertRaisesRegex(RuntimeError, r"not implemented for \'Half\'"):
|
|
expected = [torch_op(tensors1[i]) for i in range(N)]
|
|
|
|
with self.assertRaisesRegex(RuntimeError, r"not implemented for \'Half\'"):
|
|
res = fe_op(tensors1)
|
|
break
|
|
|
|
if dtype == torch.bfloat16 and not support_bfloat16:
|
|
if self.device_type == 'cuda' or torch_op in [torch.sinh, torch.cosh]:
|
|
with self.assertRaisesRegex(RuntimeError, r"not implemented for \'BFloat16\'"):
|
|
expected = [torch_op(tensors1[i]) for i in range(N)]
|
|
|
|
with self.assertRaisesRegex(RuntimeError, r"not implemented for \'BFloat16\'"):
|
|
res = fe_op(tensors1)
|
|
break
|
|
|
|
if dtype in [torch.complex64, torch.complex128] and not support_complex:
|
|
# not using assertRaisesRegex due to different error messages
|
|
with self.assertRaises(RuntimeError):
|
|
expected = [torch_op(tensors1[i]) for i in range(N)]
|
|
|
|
with self.assertRaises(RuntimeError):
|
|
res = fe_op(tensors1)
|
|
break
|
|
|
|
expected = [torch_op(tensors1[i].to(dtype=control_dtype)).to(dtype=dtype) for i in range(N)]
|
|
res = fe_op(tensors1)
|
|
if (dtype is torch.float16 or dtype is torch.bfloat16) and TEST_WITH_ROCM:
|
|
self.assertEqual(res, expected, atol=1.e-3, rtol=self.dtype_precisions[dtype][0])
|
|
|
|
fe_op_(tensors1)
|
|
self.assertEqual(res, tensors1)
|
|
else:
|
|
self.assertEqual(res, expected)
|
|
|
|
fe_op_(tensors1)
|
|
self.assertEqual(res, tensors1)
|
|
|
|
# Separate test for abs due to a lot of special cases
|
|
# Absolute value of a complex number a + bj is defined as sqrt(a^2 + b^2), i.e. a floating point
|
|
@dtypes(*(torch.testing.floating_and_complex_types_and(torch.bfloat16, torch.half)))
|
|
def test_abs(self, device, dtype):
|
|
for N in N_values:
|
|
tensors1 = self._get_test_data(device, dtype, N)
|
|
# Mimics cuda kernel dtype flow. With fp16/bf16 input, runs in fp32 and casts output back to fp16/bf16.
|
|
control_dtype = torch.float32 if (self.device_type == 'cuda' and
|
|
(dtype is torch.float16 or dtype is torch.bfloat16)) else dtype
|
|
|
|
expected = [torch.abs(tensors1[i].to(dtype=control_dtype)).to(dtype=dtype) for i in range(N)]
|
|
res = torch._foreach_abs(tensors1)
|
|
if (dtype is torch.float16 or dtype is torch.bfloat16) and TEST_WITH_ROCM:
|
|
self.assertEqual(res, expected, atol=1.e-3, rtol=self.dtype_precisions[dtype][0])
|
|
|
|
torch._foreach_abs_(tensors1)
|
|
self.assertEqual(res, tensors1)
|
|
else:
|
|
expected = [torch.abs(tensors1[i]) for i in range(N)]
|
|
self.assertEqual(res, expected)
|
|
|
|
if dtype in [torch.complex64, torch.complex128]:
|
|
with self.assertRaisesRegex(RuntimeError, r"In-place abs is not supported for complex tensors."):
|
|
torch._foreach_abs_(tensors1)
|
|
else:
|
|
torch._foreach_abs_(tensors1)
|
|
self.assertEqual(res, tensors1)
|
|
|
|
#
|
|
# Pointwise ops
|
|
#
|
|
@dtypes(*torch.testing.get_all_dtypes(include_bfloat16=False, include_bool=False, include_complex=False))
|
|
def test_addcmul(self, device, dtype):
|
|
if self.device_type == 'cpu':
|
|
if dtype == torch.half:
|
|
with self.assertRaisesRegex(RuntimeError, r"\"addcmul_cpu_out\" not implemented for \'Half\'"):
|
|
self._test_pointwise_op(device, dtype, torch._foreach_addcmul,
|
|
torch._foreach_addcmul_, torch.addcmul)
|
|
return
|
|
|
|
self._test_pointwise_op(device, dtype, torch._foreach_addcmul, torch._foreach_addcmul_, torch.addcmul)
|
|
|
|
@dtypes(*torch.testing.get_all_dtypes(include_bfloat16=False, include_bool=False, include_complex=False))
|
|
def test_addcdiv(self, device, dtype):
|
|
if dtype in [torch.int8, torch.int16, torch.int32, torch.int64, torch.uint8]:
|
|
with self.assertRaisesRegex(RuntimeError,
|
|
"Integer division with addcdiv is no longer supported, and in a future"):
|
|
self._test_pointwise_op(device, dtype, torch._foreach_addcdiv, torch._foreach_addcdiv_, torch.addcdiv)
|
|
return
|
|
|
|
if self.device_type == 'cpu':
|
|
if dtype == torch.half:
|
|
with self.assertRaisesRegex(RuntimeError, r"\"addcdiv_cpu_out\" not implemented for \'Half\'"):
|
|
self._test_pointwise_op(device, dtype, torch._foreach_addcdiv,
|
|
torch._foreach_addcdiv_, torch.addcdiv)
|
|
return
|
|
self._test_pointwise_op(device, dtype, torch._foreach_addcdiv, torch._foreach_addcdiv_, torch.addcdiv)
|
|
|
|
@dtypes(*torch.testing.get_all_dtypes(include_bfloat16=False, include_bool=False, include_complex=False))
|
|
def test_min_max(self, device, dtype):
|
|
for N in N_values:
|
|
tensors1 = self._get_test_data(device, dtype, N)
|
|
tensors2 = self._get_test_data(device, dtype, N)
|
|
|
|
# Mimics cuda kernel dtype flow. With fp16/bf16 input, runs in fp32 and casts output back to fp16/bf16.
|
|
control_dtype = torch.float32 if (self.device_type == 'cuda' and
|
|
(dtype is torch.float16 or dtype is torch.bfloat16)) else dtype
|
|
|
|
expected_max = [torch.max(tensors1[i].to(dtype=control_dtype),
|
|
tensors2[i].to(dtype=control_dtype)).to(dtype=dtype) for i in range(N)]
|
|
|
|
expected_min = [torch.min(tensors1[i].to(dtype=control_dtype),
|
|
tensors2[i].to(dtype=control_dtype)).to(dtype=dtype) for i in range(N)]
|
|
|
|
res_max = torch._foreach_maximum(tensors1, tensors2)
|
|
self.assertEqual(res_max, expected_max)
|
|
|
|
res_min = torch._foreach_minimum(tensors1, tensors2)
|
|
self.assertEqual(res_min, expected_min)
|
|
|
|
|
|
@dtypes(*(torch.testing.get_all_fp_dtypes(include_half=True, include_bfloat16=False)))
|
|
def test_max_min_float_inf_nan(self, device, dtype):
|
|
a = [
|
|
torch.tensor([float('inf')], device=device, dtype=dtype),
|
|
torch.tensor([-float('inf')], device=device, dtype=dtype),
|
|
torch.tensor([float('nan')], device=device, dtype=dtype),
|
|
torch.tensor([float('nan')], device=device, dtype=dtype)
|
|
]
|
|
|
|
b = [
|
|
torch.tensor([-float('inf')], device=device, dtype=dtype),
|
|
torch.tensor([float('inf')], device=device, dtype=dtype),
|
|
torch.tensor([float('inf')], device=device, dtype=dtype),
|
|
torch.tensor([float('nan')], device=device, dtype=dtype)
|
|
]
|
|
|
|
expected = [torch.max(a1, b1) for a1, b1 in zip(a, b)]
|
|
res = torch._foreach_maximum(a, b)
|
|
self.assertEqual(expected, res)
|
|
|
|
expected = [torch.min(a1, b1) for a1, b1 in zip(a, b)]
|
|
res = torch._foreach_minimum(a, b)
|
|
self.assertEqual(expected, res)
|
|
|
|
@dtypes(*(torch.testing.get_all_fp_dtypes(include_half=True, include_bfloat16=False)))
|
|
def test_max_min_inf_nan(self, device, dtype):
|
|
a = [
|
|
torch.tensor([inf], device=device, dtype=dtype),
|
|
torch.tensor([-inf], device=device, dtype=dtype),
|
|
torch.tensor([nan], device=device, dtype=dtype),
|
|
torch.tensor([nan], device=device, dtype=dtype)
|
|
]
|
|
|
|
b = [
|
|
torch.tensor([-inf], device=device, dtype=dtype),
|
|
torch.tensor([inf], device=device, dtype=dtype),
|
|
torch.tensor([inf], device=device, dtype=dtype),
|
|
torch.tensor([nan], device=device, dtype=dtype)
|
|
]
|
|
|
|
expected_max = [torch.max(a1, b1) for a1, b1 in zip(a, b)]
|
|
res_max = torch._foreach_maximum(a, b)
|
|
self.assertEqual(expected_max, res_max)
|
|
|
|
expected_min = [torch.min(a1, b1) for a1, b1 in zip(a, b)]
|
|
res_min = torch._foreach_minimum(a, b)
|
|
self.assertEqual(expected_min, res_min)
|
|
|
|
#
|
|
# Ops with scalar
|
|
#
|
|
@skipCUDAIfRocm
|
|
@dtypes(*torch.testing.get_all_dtypes())
|
|
def test_int_scalar(self, device, dtype):
|
|
for N in N_values:
|
|
for foreach_bin_op, foreach_bin_op_, torch_bin_op in zip(self.foreach_bin_ops,
|
|
self.foreach_bin_ops_,
|
|
self.torch_bin_ops):
|
|
tensors = self._get_test_data(device, dtype, N)
|
|
scalar = 3
|
|
expected = [torch_bin_op(t, scalar) for t in tensors]
|
|
|
|
res = foreach_bin_op(tensors, scalar)
|
|
|
|
if dtype == torch.bool:
|
|
self.assertEqual(res, expected)
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "can't be cast to the desired output type"):
|
|
foreach_bin_op_(tensors, scalar)
|
|
return
|
|
|
|
|
|
if foreach_bin_op_ == torch._foreach_div_ and dtype in torch.testing.integral_types() and self.device_type == "cpu":
|
|
with self.assertRaisesRegex(RuntimeError,
|
|
"can't be cast to the desired output type"):
|
|
foreach_bin_op_(tensors, scalar)
|
|
return
|
|
|
|
# TODO[type promotion]: Fix once type promotion is enabled.
|
|
if dtype in torch.testing.integral_types() and self.device_type == 'cuda':
|
|
self.assertEqual(res, [e.to(dtype) for e in expected])
|
|
|
|
foreach_bin_op_(tensors, scalar)
|
|
self.assertEqual(tensors, [e.to(dtype) for e in expected])
|
|
else:
|
|
self.assertEqual(res, expected)
|
|
foreach_bin_op_(tensors, scalar)
|
|
self.assertEqual(tensors, expected)
|
|
|
|
# TODO[Fix scalar list]:
|
|
# We need to update codegen to correctly handle function overloads with float[] and int[].
|
|
# As optimizers work with float tensors, the result will always be torch.float32 for now.
|
|
# Current schema is using 'float[]' as scalar list type.
|
|
@skipCUDAIfRocm
|
|
@dtypes(*torch.testing.get_all_dtypes())
|
|
def test_int_scalarlist(self, device, dtype):
|
|
for N in N_values:
|
|
for foreach_bin_op, foreach_bin_op_, torch_bin_op in zip(self.foreach_bin_ops,
|
|
self.foreach_bin_ops_,
|
|
self.torch_bin_ops):
|
|
tensors = self._get_test_data(device, dtype, N)
|
|
scalars = [1 for _ in range(N)]
|
|
expected = [torch_bin_op(t, s) for t, s in zip(tensors, scalars)]
|
|
|
|
# we dont support bool and complex types on CUDA for now
|
|
if (dtype in torch.testing.get_all_complex_dtypes() or dtype == torch.bool) and self.device_type == 'cuda':
|
|
with self.assertRaisesRegex(RuntimeError, "not implemented for"):
|
|
foreach_bin_op_(tensors, scalars)
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "not implemented for"):
|
|
foreach_bin_op(tensors, scalars)
|
|
return
|
|
|
|
res = foreach_bin_op(tensors, scalars)
|
|
|
|
if dtype == torch.bool:
|
|
self.assertEqual(res, [torch_bin_op(t.to(torch.float32), s) for t, s in zip(tensors, scalars)])
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "result type Float can't be cast to the desired output type"):
|
|
foreach_bin_op_(tensors, scalars)
|
|
return
|
|
|
|
if dtype in torch.testing.integral_types():
|
|
if self.device_type == 'cpu':
|
|
self.assertEqual(res, [e.to(torch.float32) for e in expected])
|
|
else:
|
|
# TODO[type promotion]: Fix once type promotion is enabled.
|
|
self.assertEqual(res, [e.to(dtype) for e in expected])
|
|
else:
|
|
self.assertEqual(res, expected)
|
|
|
|
if dtype in torch.testing.integral_types() and self.device_type == 'cpu':
|
|
with self.assertRaisesRegex(RuntimeError, "result type Float can't be cast to the desired output type"):
|
|
foreach_bin_op_(tensors, scalars)
|
|
return
|
|
else:
|
|
foreach_bin_op_(tensors, scalars)
|
|
self.assertEqual(res, tensors)
|
|
|
|
@skipCUDAIfRocm
|
|
@dtypes(*torch.testing.get_all_dtypes())
|
|
def test_float_scalar(self, device, dtype):
|
|
for N in N_values:
|
|
for foreach_bin_op, foreach_bin_op_, torch_bin_op in zip(self.foreach_bin_ops,
|
|
self.foreach_bin_ops_,
|
|
self.torch_bin_ops):
|
|
tensors = self._get_test_data(device, dtype, N)
|
|
scalar = 3.3
|
|
|
|
# Mimics cuda kernel dtype flow. With fp16/bf16 input, runs in fp32 and casts output back to fp16/bf16.
|
|
control_dtype = torch.float32 if (self.device_type == 'cuda' and
|
|
(dtype is torch.float16 or dtype is torch.bfloat16)) else dtype
|
|
expected = [torch_bin_op(t.to(dtype=control_dtype),
|
|
scalar) for t in tensors]
|
|
if (dtype is torch.float16 or dtype is torch.bfloat16):
|
|
expected = [e.to(dtype=dtype) for e in expected]
|
|
|
|
if dtype == torch.bool:
|
|
if foreach_bin_op == torch._foreach_sub:
|
|
with self.assertRaisesRegex(RuntimeError, "Subtraction, the `-` operator, with two bool"):
|
|
foreach_bin_op_(tensors, scalar)
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "Subtraction, the `-` operator, with two bool"):
|
|
foreach_bin_op(tensors, scalar)
|
|
return
|
|
|
|
res = foreach_bin_op(tensors, scalar)
|
|
if (dtype is torch.float16 or dtype is torch.bfloat16) and TEST_WITH_ROCM:
|
|
self.assertEqual(res, expected, atol=1.e-3, rtol=self.dtype_precisions[dtype][0])
|
|
else:
|
|
self.assertEqual(res, expected)
|
|
|
|
if dtype in torch.testing.integral_types():
|
|
with self.assertRaisesRegex(RuntimeError, "result type Float can't be cast to the desired output type"):
|
|
foreach_bin_op_(tensors, scalar)
|
|
return
|
|
|
|
foreach_bin_op_(tensors, scalar)
|
|
if (dtype is torch.float16 or dtype is torch.bfloat16) and TEST_WITH_ROCM:
|
|
self.assertEqual(tensors, expected, atol=1.e-3, rtol=self.dtype_precisions[dtype][0])
|
|
else:
|
|
self.assertEqual(tensors, expected)
|
|
|
|
@skipCUDAIfRocm
|
|
@dtypes(*torch.testing.get_all_dtypes())
|
|
def test_float_scalarlist(self, device, dtype):
|
|
for N in N_values:
|
|
for foreach_bin_op, foreach_bin_op_, torch_bin_op in zip(self.foreach_bin_ops,
|
|
self.foreach_bin_ops_,
|
|
self.torch_bin_ops):
|
|
tensors = self._get_test_data(device, dtype, N)
|
|
scalars = [1.1 for _ in range(N)]
|
|
|
|
# If incoming dtype is float16 or bfloat16, runs in float32 and casts output back to dtype.
|
|
control_dtype = torch.float32 if (self.device_type == 'cuda' and
|
|
(dtype is torch.float16 or dtype is torch.bfloat16)) else dtype
|
|
expected = [torch_bin_op(t.to(dtype=control_dtype),
|
|
s) for t, s in zip(tensors, scalars)]
|
|
if (dtype is torch.float16 or dtype is torch.bfloat16):
|
|
expected = [e.to(dtype=dtype) for e in expected]
|
|
|
|
# we dont support bool and complex types on CUDA for now
|
|
if (dtype in torch.testing.get_all_complex_dtypes() or dtype == torch.bool) and self.device_type == 'cuda':
|
|
with self.assertRaisesRegex(RuntimeError, "not implemented for"):
|
|
foreach_bin_op_(tensors, scalars)
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "not implemented for"):
|
|
foreach_bin_op(tensors, scalars)
|
|
return
|
|
|
|
res = foreach_bin_op(tensors, scalars)
|
|
|
|
if dtype == torch.bool:
|
|
# see TODO[Fix scalar list]
|
|
self.assertEqual(res, [torch_bin_op(t.to(torch.float32), s) for t, s in zip(tensors, scalars)])
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "result type Float can't be cast to the desired output type"):
|
|
foreach_bin_op_(tensors, scalars)
|
|
return
|
|
|
|
if dtype in torch.testing.integral_types() and self.device_type == 'cuda':
|
|
# see TODO[Fix scalar list]
|
|
self.assertEqual(res, [e.to(dtype) for e in expected])
|
|
|
|
foreach_bin_op_(tensors, scalars)
|
|
self.assertEqual(tensors, res)
|
|
return
|
|
else:
|
|
if (dtype is torch.float16 or dtype is torch.bfloat16) and TEST_WITH_ROCM:
|
|
self.assertEqual(res, expected, atol=1.e-3, rtol=self.dtype_precisions[dtype][0])
|
|
else:
|
|
self.assertEqual(res, expected)
|
|
|
|
if dtype in torch.testing.integral_types() and self.device_type == "cpu":
|
|
with self.assertRaisesRegex(RuntimeError, "result type Float can't be cast to the desired output type"):
|
|
foreach_bin_op_(tensors, scalars)
|
|
return
|
|
|
|
foreach_bin_op_(tensors, scalars)
|
|
if (dtype is torch.float16 or dtype is torch.bfloat16) and TEST_WITH_ROCM:
|
|
self.assertEqual(tensors, expected, atol=1.e-3, rtol=self.dtype_precisions[dtype][0])
|
|
else:
|
|
self.assertEqual(tensors, expected)
|
|
|
|
@skipCUDAIfRocm
|
|
@dtypes(*torch.testing.get_all_dtypes())
|
|
def test_complex_scalar(self, device, dtype):
|
|
for N in N_values:
|
|
for foreach_bin_op, foreach_bin_op_, torch_bin_op in zip(self.foreach_bin_ops,
|
|
self.foreach_bin_ops_,
|
|
self.torch_bin_ops):
|
|
tensors = self._get_test_data(device, dtype, N)
|
|
scalar = 3 + 5j
|
|
expected = [torch_bin_op(t, scalar) for t in tensors]
|
|
|
|
if dtype == torch.bool:
|
|
if foreach_bin_op == torch._foreach_sub:
|
|
with self.assertRaisesRegex(RuntimeError, "Subtraction, the `-` operator, with two bool"):
|
|
foreach_bin_op_(tensors, scalar)
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "Subtraction, the `-` operator, with two bool"):
|
|
foreach_bin_op(tensors, scalar)
|
|
return
|
|
|
|
if dtype in torch.testing.get_all_fp_dtypes(include_half=True, include_bfloat16=True) and \
|
|
self.device_type == 'cuda':
|
|
with self.assertRaisesRegex(RuntimeError, "value cannot be converted to type"):
|
|
foreach_bin_op_(tensors, scalar)
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "value cannot be converted to type"):
|
|
foreach_bin_op(tensors, scalar)
|
|
return
|
|
|
|
res = foreach_bin_op(tensors, scalar)
|
|
self.assertEqual(res, expected)
|
|
|
|
if dtype not in [torch.complex64, torch.complex128]:
|
|
with self.assertRaisesRegex(RuntimeError, "can't be cast to the desired output type"):
|
|
foreach_bin_op_(tensors, scalar)
|
|
else:
|
|
foreach_bin_op_(tensors, scalar)
|
|
self.assertEqual(res, tensors)
|
|
|
|
@dtypes(*torch.testing.get_all_dtypes())
|
|
def test_complex_scalarlist(self, device, dtype):
|
|
for N in N_values:
|
|
for foreach_bin_op, foreach_bin_op_, torch_bin_op in zip(self.foreach_bin_ops,
|
|
self.foreach_bin_ops_,
|
|
self.torch_bin_ops):
|
|
tensors = self._get_test_data(device, dtype, N)
|
|
scalars = [3 + 5j for _ in range(N)]
|
|
expected = [torch_bin_op(t, s) for t, s in zip(tensors, scalars)]
|
|
|
|
if dtype == torch.bool:
|
|
if foreach_bin_op == torch._foreach_sub:
|
|
with self.assertRaisesRegex(RuntimeError, "Subtraction, the `-` operator, with two bool"):
|
|
foreach_bin_op_(tensors, scalar)
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "Subtraction, the `-` operator, with two bool"):
|
|
foreach_bin_op(tensors, scalar)
|
|
return
|
|
|
|
with self.assertRaisesRegex(TypeError, "argument 'scalars' must be tuple of floats"):
|
|
res = foreach_bin_op(tensors, scalars)
|
|
|
|
with self.assertRaisesRegex(TypeError, "argument 'scalars' must be tuple of floats"):
|
|
foreach_bin_op_(tensors, scalars)
|
|
|
|
@skipCUDAIfRocm
|
|
@dtypes(*torch.testing.get_all_dtypes())
|
|
def test_bool_scalar(self, device, dtype):
|
|
for N in N_values:
|
|
for foreach_bin_op, foreach_bin_op_, torch_bin_op in zip(self.foreach_bin_ops,
|
|
self.foreach_bin_ops_,
|
|
self.torch_bin_ops):
|
|
tensors = self._get_test_data(device, dtype, N)
|
|
scalar = True
|
|
|
|
if dtype == torch.bool:
|
|
expected = [torch_bin_op(t, scalar) for t in tensors]
|
|
res = foreach_bin_op(tensors, scalar)
|
|
|
|
foreach_bin_op_(tensors, scalar)
|
|
self.assertEqual(tensors, res)
|
|
return
|
|
|
|
if foreach_bin_op == torch._foreach_sub and self.device_type == "cpu":
|
|
with self.assertRaisesRegex(RuntimeError, "Subtraction, the `-` operator"):
|
|
res = foreach_bin_op(tensors, scalar)
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "Subtraction, the `-` operator"):
|
|
foreach_bin_op_(tensors, scalar)
|
|
elif foreach_bin_op == torch._foreach_sub and self.device_type == 'cuda':
|
|
res = foreach_bin_op(tensors, scalar)
|
|
self.assertEqual(res, foreach_bin_op(tensors, 1))
|
|
|
|
foreach_bin_op_(tensors, scalar)
|
|
self.assertEqual(tensors, res)
|
|
else:
|
|
expected = [torch_bin_op(t, scalar) for t in tensors]
|
|
res = foreach_bin_op(tensors, scalar)
|
|
|
|
# TODO[type promotion]: Fix once type promotion is enabled.
|
|
if dtype in torch.testing.integral_types() and self.device_type == 'cuda':
|
|
self.assertEqual(res, [e.to(dtype) for e in expected])
|
|
else:
|
|
self.assertEqual(res, expected)
|
|
|
|
if dtype in torch.testing.integral_types():
|
|
if foreach_bin_op == torch._foreach_div and self.device_type == "cpu":
|
|
with self.assertRaisesRegex(RuntimeError, "result type Float can't be cast to the desired "):
|
|
foreach_bin_op_(tensors, scalar)
|
|
else:
|
|
foreach_bin_op_(tensors, scalar)
|
|
self.assertEqual(tensors, res)
|
|
else:
|
|
foreach_bin_op_(tensors, scalar)
|
|
self.assertEqual(tensors, expected)
|
|
|
|
@skipCUDAIfRocm
|
|
@dtypes(*torch.testing.get_all_dtypes())
|
|
def test_bool_scalarlist(self, device, dtype):
|
|
for N in N_values:
|
|
for foreach_bin_op, foreach_bin_op_, torch_bin_op in zip(self.foreach_bin_ops,
|
|
self.foreach_bin_ops_,
|
|
self.torch_bin_ops):
|
|
tensors = self._get_test_data(device, dtype, N)
|
|
scalars = [True for _ in range(N)]
|
|
|
|
if dtype == torch.bool:
|
|
if self.device_type == 'cuda':
|
|
with self.assertRaisesRegex(RuntimeError, "not implemented for"):
|
|
foreach_bin_op(tensors, scalars)
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "not implemented for"):
|
|
foreach_bin_op_(tensors, scalars)
|
|
return
|
|
else:
|
|
if foreach_bin_op == torch._foreach_sub:
|
|
with self.assertRaisesRegex(RuntimeError, "Subtraction, the `-` operator, with a bool tensor"):
|
|
foreach_bin_op_(tensors, scalars)
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "Subtraction, the `-` operator, with a bool tensor"):
|
|
foreach_bin_op(tensors, scalars)
|
|
else:
|
|
with self.assertRaisesRegex(RuntimeError, "result type Float can't be cast to the desired"):
|
|
foreach_bin_op_(tensors, scalars)
|
|
|
|
res = foreach_bin_op(tensors, scalars)
|
|
for r in res:
|
|
self.assertTrue(r.dtype == torch.float32)
|
|
else:
|
|
# we dont support bool and complex types on CUDA for now
|
|
if (dtype in torch.testing.get_all_complex_dtypes()) and self.device_type == 'cuda':
|
|
with self.assertRaisesRegex(RuntimeError, "not implemented for"):
|
|
foreach_bin_op_(tensors, scalars)
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "not implemented for"):
|
|
foreach_bin_op(tensors, scalars)
|
|
return
|
|
|
|
if foreach_bin_op == torch._foreach_sub:
|
|
if self.device_type == "cpu":
|
|
# see TODO[Fix scalar list]
|
|
res = foreach_bin_op(tensors, scalars)
|
|
if dtype in torch.testing.integral_types():
|
|
self.assertEqual(res, [r.to(torch.float32) for r in [torch_bin_op(t, 1) for t in tensors]])
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "result type Float can't be cast to the "):
|
|
foreach_bin_op_(tensors, scalars)
|
|
else:
|
|
self.assertEqual(res, [torch_bin_op(t, 1) for t in tensors])
|
|
foreach_bin_op_(tensors, scalars)
|
|
self.assertEqual(res, tensors)
|
|
else:
|
|
# see TODO[Fix scalar list]
|
|
res = foreach_bin_op(tensors, scalars)
|
|
if dtype in torch.testing.integral_types():
|
|
self.assertEqual(res, [r.to(dtype) for r in [torch_bin_op(t, 1) for t in tensors]])
|
|
else:
|
|
self.assertEqual(res, [torch_bin_op(t, 1) for t in tensors])
|
|
|
|
foreach_bin_op_(tensors, scalars)
|
|
self.assertEqual(res, tensors)
|
|
else:
|
|
if self.device_type == "cpu":
|
|
expected = [torch_bin_op(t, s) for t, s in zip(tensors, scalars)]
|
|
res = foreach_bin_op(tensors, scalars)
|
|
|
|
# see TODO[Fix scalar list]
|
|
if dtype in torch.testing.integral_types():
|
|
self.assertEqual(res, [e.to(torch.float32) for e in expected])
|
|
else:
|
|
self.assertEqual(res, expected)
|
|
|
|
if dtype in torch.testing.integral_types():
|
|
with self.assertRaisesRegex(RuntimeError, "result type Float can't be cast to the desired "):
|
|
foreach_bin_op_(tensors, scalars)
|
|
else:
|
|
foreach_bin_op_(tensors, scalars)
|
|
self.assertEqual(tensors, expected)
|
|
else:
|
|
expected = [torch_bin_op(t, s) for t, s in zip(tensors, scalars)]
|
|
res = foreach_bin_op(tensors, scalars)
|
|
|
|
if dtype in torch.testing.integral_types():
|
|
self.assertEqual(res, [e.to(dtype) for e in expected])
|
|
else:
|
|
self.assertEqual(res, expected)
|
|
|
|
foreach_bin_op_(tensors, scalars)
|
|
self.assertEqual(res, tensors)
|
|
|
|
@dtypes(*torch.testing.get_all_dtypes())
|
|
def test_add_with_different_size_tensors(self, device, dtype):
|
|
if dtype == torch.bool:
|
|
return
|
|
tensors = [torch.zeros(10 + n, 10 + n, device=device, dtype=dtype) for n in range(10)]
|
|
expected = [torch.ones(10 + n, 10 + n, device=device, dtype=dtype) for n in range(10)]
|
|
|
|
torch._foreach_add_(tensors, 1)
|
|
self.assertEqual(expected, tensors)
|
|
|
|
@dtypes(*torch.testing.get_all_dtypes())
|
|
def test_add_scalar_with_empty_list_and_empty_tensor(self, device, dtype):
|
|
# TODO: enable empty list case
|
|
for tensors in [[torch.randn([0])]]:
|
|
res = torch._foreach_add(tensors, 1)
|
|
self.assertEqual(res, tensors)
|
|
|
|
torch._foreach_add_(tensors, 1)
|
|
self.assertEqual(res, tensors)
|
|
|
|
@dtypes(*torch.testing.get_all_dtypes())
|
|
def test_add_scalar_with_overlapping_tensors(self, device, dtype):
|
|
tensors = [torch.ones(1, 1, device=device, dtype=dtype).expand(2, 1, 3)]
|
|
expected = [torch.tensor([[[2, 2, 2]], [[2, 2, 2]]], dtype=dtype, device=device)]
|
|
|
|
# bool tensor + 1 will result in int64 tensor
|
|
if dtype == torch.bool:
|
|
expected[0] = expected[0].to(torch.int64).add(1)
|
|
|
|
res = torch._foreach_add(tensors, 1)
|
|
self.assertEqual(res, expected)
|
|
|
|
def test_bin_op_scalar_with_different_tensor_dtypes(self, device):
|
|
tensors = [torch.tensor([1.1], dtype=torch.float, device=device),
|
|
torch.tensor([1], dtype=torch.long, device=device)]
|
|
self.assertRaises(RuntimeError, lambda: torch._foreach_add(tensors, 1))
|
|
|
|
#
|
|
# Ops with list
|
|
#
|
|
def test_bin_op_list_error_cases(self, device):
|
|
for bin_op, bin_op_ in zip(self.foreach_bin_ops, self.foreach_bin_ops_):
|
|
tensors1 = []
|
|
tensors2 = []
|
|
|
|
# Empty lists
|
|
with self.assertRaisesRegex(RuntimeError, "There were no tensor arguments to this function"):
|
|
bin_op(tensors1, tensors2)
|
|
with self.assertRaisesRegex(RuntimeError, "There were no tensor arguments to this function"):
|
|
bin_op_(tensors1, tensors2)
|
|
|
|
# One empty list
|
|
tensors1.append(torch.tensor([1], device=device))
|
|
with self.assertRaisesRegex(RuntimeError, "Tensor list must have same number of elements as scalar list."):
|
|
bin_op(tensors1, tensors2)
|
|
with self.assertRaisesRegex(RuntimeError, "Tensor list must have same number of elements as scalar list."):
|
|
bin_op_(tensors1, tensors2)
|
|
|
|
# Lists have different amount of tensors
|
|
tensors2.append(torch.tensor([1], device=device))
|
|
tensors2.append(torch.tensor([1], device=device))
|
|
with self.assertRaisesRegex(RuntimeError, "Tensor lists must have the same number of tensors, got 1 and 2"):
|
|
bin_op(tensors1, tensors2)
|
|
with self.assertRaisesRegex(RuntimeError, "Tensor lists must have the same number of tensors, got 1 and 2"):
|
|
bin_op_(tensors1, tensors2)
|
|
|
|
# Different dtypes
|
|
tensors1 = [torch.zeros(10, 10, device=device, dtype=torch.float) for _ in range(10)]
|
|
tensors2 = [torch.ones(10, 10, device=device, dtype=torch.int) for _ in range(10)]
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "All tensors in the tensor list must have the same dtype."):
|
|
bin_op(tensors1, tensors2)
|
|
with self.assertRaisesRegex(RuntimeError, "All tensors in the tensor list must have the same dtype."):
|
|
bin_op_(tensors1, tensors2)
|
|
|
|
# different devices
|
|
if torch.cuda.is_available() and torch.cuda.device_count() > 1:
|
|
tensor1 = torch.zeros(10, 10, device="cuda:0")
|
|
tensor2 = torch.ones(10, 10, device="cuda:1")
|
|
with self.assertRaisesRegex(RuntimeError, "Expected all tensors to be on the same device"):
|
|
bin_op([tensor1], [tensor2])
|
|
with self.assertRaisesRegex(RuntimeError, "Expected all tensors to be on the same device"):
|
|
bin_op_([tensor1], [tensor2])
|
|
|
|
# Corresponding tensors with different sizes
|
|
tensors1 = [torch.zeros(10, 10, device=device) for _ in range(10)]
|
|
tensors2 = [torch.ones(11, 11, device=device) for _ in range(10)]
|
|
with self.assertRaisesRegex(RuntimeError, "Corresponding tensors in lists must have the same size"):
|
|
bin_op(tensors1, tensors2)
|
|
with self.assertRaisesRegex(RuntimeError, r", got \[10, 10\] and \[11, 11\]"):
|
|
bin_op_(tensors1, tensors2)
|
|
|
|
@dtypes(*torch.testing.get_all_dtypes())
|
|
def test_add_list(self, device, dtype):
|
|
self._test_bin_op_list(device, dtype, torch._foreach_add, torch._foreach_add_, torch.add)
|
|
self._test_bin_op_list_alpha(device, dtype, torch._foreach_add, torch._foreach_add_, torch.add)
|
|
|
|
@dtypes(*torch.testing.get_all_dtypes())
|
|
def test_sub_list(self, device, dtype):
|
|
if dtype == torch.bool:
|
|
with self.assertRaisesRegex(RuntimeError, "Subtraction, the `-` operator, with two bool"):
|
|
self._test_bin_op_list(device, dtype, torch._foreach_sub, torch._foreach_sub_, torch.sub)
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "Subtraction, the `-` operator, with a bool tensor"):
|
|
self._test_bin_op_list_alpha(device, dtype, torch._foreach_sub, torch._foreach_sub_, torch.sub)
|
|
else:
|
|
self._test_bin_op_list(device, dtype, torch._foreach_sub, torch._foreach_sub_, torch.sub)
|
|
self._test_bin_op_list_alpha(device, dtype, torch._foreach_sub, torch._foreach_sub_, torch.sub)
|
|
|
|
@dtypes(*torch.testing.get_all_dtypes())
|
|
def test_mul_list(self, device, dtype):
|
|
self._test_bin_op_list(device, dtype, torch._foreach_mul, torch._foreach_mul_, torch.mul)
|
|
|
|
@dtypes(*torch.testing.get_all_dtypes())
|
|
def test_div_list(self, device, dtype):
|
|
if dtype in torch.testing.integral_types_and(torch.bool):
|
|
if self.device_type == 'cpu':
|
|
with self.assertRaisesRegex(RuntimeError, "result type Float can't be cast to the desired output type"):
|
|
self._test_bin_op_list(device, dtype, torch._foreach_div, torch._foreach_div_, torch.div)
|
|
else:
|
|
self.skipTest("Skipped! See https://github.com/pytorch/pytorch/issues/44489")
|
|
return
|
|
|
|
for N in N_values:
|
|
tensors1 = self._get_test_data(device, dtype, N)
|
|
|
|
if dtype in [torch.bfloat16, torch.bool, torch.float16]:
|
|
tensors2 = [torch.zeros(N, N, device=device, dtype=dtype).add(2) for _ in range(N)]
|
|
else:
|
|
tensors2 = self._get_test_data(device, dtype, N)
|
|
|
|
expected = [torch.div(tensors1[i], tensors2[i]) for i in range(N)]
|
|
res = torch._foreach_div(tensors1, tensors2)
|
|
torch._foreach_div_(tensors1, tensors2)
|
|
self.assertEqual(res, tensors1)
|
|
self.assertEqual(tensors1, res)
|
|
|
|
@dtypes(*torch.testing.get_all_dtypes())
|
|
def test_add_list_different_sizes(self, device, dtype):
|
|
tensors1 = [torch.zeros(10 + n, 10 + n, device=device, dtype=dtype) for n in range(10)]
|
|
tensors2 = [torch.ones(10 + n, 10 + n, device=device, dtype=dtype) for n in range(10)]
|
|
|
|
res = torch._foreach_add(tensors1, tensors2)
|
|
torch._foreach_add_(tensors1, tensors2)
|
|
self.assertEqual(res, tensors1)
|
|
self.assertEqual(res, [torch.ones(10 + n, 10 + n, device=device, dtype=dtype) for n in range(10)])
|
|
|
|
@unittest.skipIf(not torch.cuda.is_available(), "CUDA not found")
|
|
@dtypes(*torch.testing.get_all_dtypes())
|
|
def test_add_list_slow_path(self, device, dtype):
|
|
# different strides
|
|
tensor1 = torch.zeros(10, 10, device=device, dtype=dtype)
|
|
tensor2 = torch.ones(10, 10, device=device, dtype=dtype)
|
|
res = torch._foreach_add([tensor1], [tensor2.t()])
|
|
torch._foreach_add_([tensor1], [tensor2])
|
|
self.assertEqual(res, [tensor1])
|
|
|
|
# non contiguous
|
|
tensor1 = torch.randn(5, 2, 1, 3, device=device)[:, 0]
|
|
tensor2 = torch.randn(5, 2, 1, 3, device=device)[:, 0]
|
|
self.assertFalse(tensor1.is_contiguous())
|
|
self.assertFalse(tensor2.is_contiguous())
|
|
res = torch._foreach_add([tensor1], [tensor2])
|
|
torch._foreach_add_([tensor1], [tensor2])
|
|
self.assertEqual(res, [tensor1])
|
|
|
|
instantiate_device_type_tests(TestForeach, globals())
|
|
|
|
if __name__ == '__main__':
|
|
run_tests()
|