mirror of
https://github.com/zebrajr/pytorch.git
synced 2025-12-07 00:21:07 +01:00
49c41b87a2
Syncing nvfuser devel branch to upstream master. https://github.com/csarofeen/pytorch/ Bug fixes and minor refactor Squashed commits to WAR github API Commits that's actually in this PR from the devel branch: ``` 4c60e7dff22a494632370e5df55c011007340d06 Add examples infrastructure for using nvFuser in a standalone program (#1725) 02a05d98334ffa580d73ccb28fdb8c577ad296fe Fix issue #1751 (#1753) 8a69aa320bd7629e1709fe5ceb7104d2c88ec84c Refactor NvFuser transpose API to match eager mode behavior (#1746) ffdf6b7709048170d768217fcd7083fc8387f932 Remove BroadcastWithoutStride. (#1738) 02bab16035e70734450c02124f5cdaa95cf5749d Fix flipping of a boolean flag (#1745) 465d66890c8242e811224359cbdb1c2915490741 cleanup (#1744) 26d354e68720bc7dd2d3b1338ac01b707a230b6a fixing noncontig broadcast (#1742) 856b6b2f9073662dd98ca22ba6c3540e20eb1cdd Add IterDomainBuilder (#1736) 1fd974f912cd4c1e21cbd16e2abb23598d66a02f fixing warning for gcc7 (#1732) de2740a43a869f8272c2648e091d7b8235097db9 disabling complex in python tests for #1730 (#1733) fbbbe0a2e7c7a63e0e2719b8bfccb759b714221a fixing MSVC build (#1728) b5feee5e2b28be688dbddc766f3c0220389c8175 Fix the fused reduction runtime kernel (#1729) 5247682dff5980bb66edf8d3aac25dea2ef2ced5 Re-entrant GroupedGridReduction (#1727) ``` RUN_TORCHBENCH: nvfuser Pull Request resolved: https://github.com/pytorch/pytorch/pull/79147 Approved by: https://github.com/davidberard98
879 lines
25 KiB
C++
879 lines
25 KiB
C++
#include <torch/csrc/jit/codegen/cuda/fusion.h>
|
|
#include <torch/csrc/jit/codegen/cuda/ir_all_nodes.h>
|
|
#include <torch/csrc/jit/codegen/cuda/type.h>
|
|
|
|
#include <torch/csrc/jit/codegen/cuda/dispatch.h>
|
|
|
|
namespace torch {
|
|
namespace jit {
|
|
namespace fuser {
|
|
namespace cuda {
|
|
|
|
template <typename T>
|
|
T* ptr(T& obj) {
|
|
return &obj;
|
|
}
|
|
|
|
template <typename T>
|
|
T* ptr(T* obj) {
|
|
return obj;
|
|
}
|
|
|
|
/*
|
|
* Generic dispatch for any handler that does not modify the IR directly.
|
|
* For example we may want to walk the graph to construct a topologically sorted
|
|
* set of exprs. This doesn't modify the IR directly. We also use this to print
|
|
* the IR itself.
|
|
* This dispatch is paired with a class that implements the functions:
|
|
* template <typenname node_type>
|
|
* int handler(node_type* node)
|
|
*
|
|
* handler should call:
|
|
* dispatch(this, node_to_dispatch)
|
|
*
|
|
* It could also implement:
|
|
* int handler(Statement* stmt){
|
|
* dispatch(this, stmt);
|
|
* }
|
|
*
|
|
* And therefore dispatch should never call:
|
|
* ptr(mutator)->mutate(this->as<Statement>());
|
|
*/
|
|
|
|
template <typename T>
|
|
void Val::dispatch(T handler, Val* val) {
|
|
switch (*(val->getValType())) {
|
|
case ValType::Scalar:
|
|
switch (*(val->getDataType())) {
|
|
case DataType::Bool:
|
|
ptr(handler)->handle(val->as<Bool>());
|
|
return;
|
|
case DataType::Double:
|
|
ptr(handler)->handle(val->as<Double>());
|
|
return;
|
|
case DataType::Int:
|
|
case DataType::Int32:
|
|
// Dispatch to Int even with Int32 as we don't have Int32 IR
|
|
// node.
|
|
ptr(handler)->handle(val->as<Int>());
|
|
return;
|
|
case DataType::ComplexDouble:
|
|
ptr(handler)->handle(val->as<ComplexDouble>());
|
|
return;
|
|
default:
|
|
break;
|
|
}
|
|
break;
|
|
case ValType::NamedScalar:
|
|
ptr(handler)->handle(val->as<NamedScalar>());
|
|
return;
|
|
|
|
case ValType::IterDomain:
|
|
ptr(handler)->handle(val->as<IterDomain>());
|
|
return;
|
|
case ValType::TensorDomain:
|
|
ptr(handler)->handle(val->as<TensorDomain>());
|
|
return;
|
|
case ValType::TensorView:
|
|
ptr(handler)->handle(val->as<TensorView>());
|
|
return;
|
|
case ValType::Predicate:
|
|
ptr(handler)->handle(val->as<kir::Predicate>());
|
|
return;
|
|
case ValType::TensorIndex:
|
|
ptr(handler)->handle(val->as<kir::TensorIndex>());
|
|
return;
|
|
default:
|
|
break;
|
|
}
|
|
TORCH_INTERNAL_ASSERT(false, "Unknown valtype in dispatch!");
|
|
}
|
|
|
|
template <typename T>
|
|
void Expr::dispatch(T handler, Expr* expr) {
|
|
switch (*(expr->getExprType())) {
|
|
case ExprType::UnaryOp:
|
|
ptr(handler)->handle(expr->as<UnaryOp>());
|
|
return;
|
|
case ExprType::BinaryOp:
|
|
ptr(handler)->handle(expr->as<BinaryOp>());
|
|
return;
|
|
case ExprType::TernaryOp:
|
|
ptr(handler)->handle(expr->as<TernaryOp>());
|
|
return;
|
|
case ExprType::ReductionOp:
|
|
ptr(handler)->handle(expr->as<ReductionOp>());
|
|
return;
|
|
case ExprType::GroupedReductionOp:
|
|
ptr(handler)->handle(expr->as<GroupedReductionOp>());
|
|
return;
|
|
case ExprType::WelfordOp:
|
|
ptr(handler)->handle(expr->as<WelfordOp>());
|
|
return;
|
|
case ExprType::LoadStoreOp:
|
|
ptr(handler)->handle(expr->as<LoadStoreOp>());
|
|
return;
|
|
case ExprType::MmaOp:
|
|
ptr(handler)->handle(expr->as<MmaOp>());
|
|
return;
|
|
case ExprType::BroadcastOp:
|
|
ptr(handler)->handle(expr->as<BroadcastOp>());
|
|
return;
|
|
|
|
case ExprType::Split:
|
|
ptr(handler)->handle(expr->as<Split>());
|
|
return;
|
|
case ExprType::Merge:
|
|
ptr(handler)->handle(expr->as<Merge>());
|
|
return;
|
|
case ExprType::TransposeOp:
|
|
ptr(handler)->handle(expr->as<TransposeOp>());
|
|
return;
|
|
case ExprType::ExpandOp:
|
|
ptr(handler)->handle(expr->as<ExpandOp>());
|
|
return;
|
|
case ExprType::ShiftOp:
|
|
ptr(handler)->handle(expr->as<ShiftOp>());
|
|
return;
|
|
case ExprType::GatherOp:
|
|
ptr(handler)->handle(expr->as<GatherOp>());
|
|
return;
|
|
case ExprType::ViewAsScalar:
|
|
ptr(handler)->handle(expr->as<ViewAsScalar>());
|
|
return;
|
|
case ExprType::ViewOp:
|
|
ptr(handler)->handle(expr->as<ViewOp>());
|
|
return;
|
|
|
|
case ExprType::Allocate:
|
|
ptr(handler)->handle(expr->as<kir::Allocate>());
|
|
return;
|
|
case ExprType::BlockSync:
|
|
ptr(handler)->handle(expr->as<kir::BlockSync>());
|
|
return;
|
|
case ExprType::GridSync:
|
|
ptr(handler)->handle(expr->as<kir::GridSync>());
|
|
return;
|
|
case ExprType::CpAsyncWait:
|
|
ptr(handler)->handle(expr->as<kir::CpAsyncWait>());
|
|
return;
|
|
case ExprType::InitMagicZero:
|
|
ptr(handler)->handle(expr->as<kir::InitMagicZero>());
|
|
return;
|
|
case ExprType::UpdateMagicZero:
|
|
ptr(handler)->handle(expr->as<kir::UpdateMagicZero>());
|
|
return;
|
|
case ExprType::ForLoop:
|
|
ptr(handler)->handle(expr->as<kir::ForLoop>());
|
|
return;
|
|
case ExprType::IfThenElse:
|
|
ptr(handler)->handle(expr->as<kir::IfThenElse>());
|
|
return;
|
|
case ExprType::GridReduction:
|
|
ptr(handler)->handle(expr->as<kir::GridReduction>());
|
|
return;
|
|
case ExprType::GroupedGridReduction:
|
|
ptr(handler)->handle(expr->as<kir::GroupedGridReduction>());
|
|
return;
|
|
case ExprType::GridBroadcast:
|
|
ptr(handler)->handle(expr->as<kir::GridBroadcast>());
|
|
return;
|
|
case ExprType::GridWelford:
|
|
ptr(handler)->handle(expr->as<kir::GridWelford>());
|
|
return;
|
|
case ExprType::AllocateFusedReduction:
|
|
ptr(handler)->handle(expr->as<kir::AllocateFusedReduction>());
|
|
return;
|
|
default:
|
|
TORCH_INTERNAL_ASSERT(false, "Unknown exprtype in dispatch!");
|
|
}
|
|
}
|
|
|
|
template <typename T>
|
|
void Statement::dispatch(T handler, Statement* stmt) {
|
|
if (stmt->isVal()) {
|
|
ptr(handler)->handle(stmt->as<Val>());
|
|
} else if (stmt->isExpr()) {
|
|
ptr(handler)->handle(stmt->as<Expr>());
|
|
} else
|
|
TORCH_INTERNAL_ASSERT(false, "Unknown stmttype in dispatch!");
|
|
}
|
|
|
|
template <typename T>
|
|
void Val::constDispatch(T handler, const Val* val) {
|
|
switch (*(val->getValType())) {
|
|
case ValType::Scalar:
|
|
switch (*(val->getDataType())) {
|
|
case DataType::Bool:
|
|
ptr(handler)->handle(val->as<Bool>());
|
|
return;
|
|
case DataType::Double:
|
|
ptr(handler)->handle(val->as<Double>());
|
|
return;
|
|
case DataType::Int:
|
|
case DataType::Int32:
|
|
// Dispatch to Int even with Int32 as we don't have Int32 IR
|
|
// node.
|
|
ptr(handler)->handle(val->as<Int>());
|
|
return;
|
|
case DataType::ComplexDouble:
|
|
ptr(handler)->handle(val->as<ComplexDouble>());
|
|
return;
|
|
default:
|
|
break;
|
|
}
|
|
break;
|
|
case ValType::NamedScalar:
|
|
ptr(handler)->handle(val->as<NamedScalar>());
|
|
return;
|
|
|
|
case ValType::IterDomain:
|
|
ptr(handler)->handle(val->as<IterDomain>());
|
|
return;
|
|
case ValType::TensorDomain:
|
|
ptr(handler)->handle(val->as<TensorDomain>());
|
|
return;
|
|
case ValType::TensorView:
|
|
ptr(handler)->handle(val->as<TensorView>());
|
|
return;
|
|
case ValType::Predicate:
|
|
ptr(handler)->handle(val->as<kir::Predicate>());
|
|
return;
|
|
case ValType::TensorIndex:
|
|
ptr(handler)->handle(val->as<kir::TensorIndex>());
|
|
return;
|
|
default:
|
|
break;
|
|
}
|
|
TORCH_INTERNAL_ASSERT(false, "Unknown valtype in dispatch!");
|
|
}
|
|
|
|
template <typename T>
|
|
void Expr::constDispatch(T handler, const Expr* expr) {
|
|
switch (*(expr->getExprType())) {
|
|
case ExprType::UnaryOp:
|
|
ptr(handler)->handle(expr->as<UnaryOp>());
|
|
return;
|
|
case ExprType::BinaryOp:
|
|
ptr(handler)->handle(expr->as<BinaryOp>());
|
|
return;
|
|
case ExprType::TernaryOp:
|
|
ptr(handler)->handle(expr->as<TernaryOp>());
|
|
return;
|
|
case ExprType::ReductionOp:
|
|
ptr(handler)->handle(expr->as<ReductionOp>());
|
|
return;
|
|
case ExprType::GroupedReductionOp:
|
|
ptr(handler)->handle(expr->as<GroupedReductionOp>());
|
|
return;
|
|
case ExprType::WelfordOp:
|
|
ptr(handler)->handle(expr->as<WelfordOp>());
|
|
return;
|
|
case ExprType::LoadStoreOp:
|
|
ptr(handler)->handle(expr->as<LoadStoreOp>());
|
|
return;
|
|
case ExprType::MmaOp:
|
|
ptr(handler)->handle(expr->as<MmaOp>());
|
|
return;
|
|
case ExprType::BroadcastOp:
|
|
ptr(handler)->handle(expr->as<BroadcastOp>());
|
|
return;
|
|
|
|
case ExprType::Split:
|
|
ptr(handler)->handle(expr->as<Split>());
|
|
return;
|
|
case ExprType::Merge:
|
|
ptr(handler)->handle(expr->as<Merge>());
|
|
return;
|
|
case ExprType::TransposeOp:
|
|
ptr(handler)->handle(expr->as<TransposeOp>());
|
|
return;
|
|
case ExprType::ExpandOp:
|
|
ptr(handler)->handle(expr->as<ExpandOp>());
|
|
return;
|
|
case ExprType::ShiftOp:
|
|
ptr(handler)->handle(expr->as<ShiftOp>());
|
|
return;
|
|
case ExprType::GatherOp:
|
|
ptr(handler)->handle(expr->as<GatherOp>());
|
|
return;
|
|
case ExprType::ViewAsScalar:
|
|
ptr(handler)->handle(expr->as<ViewAsScalar>());
|
|
return;
|
|
case ExprType::ViewOp:
|
|
ptr(handler)->handle(expr->as<ViewOp>());
|
|
return;
|
|
|
|
case ExprType::Allocate:
|
|
ptr(handler)->handle(expr->as<kir::Allocate>());
|
|
return;
|
|
case ExprType::BlockSync:
|
|
ptr(handler)->handle(expr->as<kir::BlockSync>());
|
|
return;
|
|
case ExprType::GridSync:
|
|
ptr(handler)->handle(expr->as<kir::GridSync>());
|
|
return;
|
|
case ExprType::CpAsyncWait:
|
|
ptr(handler)->handle(expr->as<kir::CpAsyncWait>());
|
|
return;
|
|
case ExprType::InitMagicZero:
|
|
ptr(handler)->handle(expr->as<kir::InitMagicZero>());
|
|
return;
|
|
case ExprType::UpdateMagicZero:
|
|
ptr(handler)->handle(expr->as<kir::UpdateMagicZero>());
|
|
return;
|
|
case ExprType::ForLoop:
|
|
ptr(handler)->handle(expr->as<kir::ForLoop>());
|
|
return;
|
|
case ExprType::IfThenElse:
|
|
ptr(handler)->handle(expr->as<kir::IfThenElse>());
|
|
return;
|
|
case ExprType::GridReduction:
|
|
ptr(handler)->handle(expr->as<kir::GridReduction>());
|
|
return;
|
|
case ExprType::GroupedGridReduction:
|
|
ptr(handler)->handle(expr->as<kir::GroupedGridReduction>());
|
|
return;
|
|
case ExprType::GridBroadcast:
|
|
ptr(handler)->handle(expr->as<kir::GridBroadcast>());
|
|
return;
|
|
case ExprType::GridWelford:
|
|
ptr(handler)->handle(expr->as<kir::GridWelford>());
|
|
return;
|
|
case ExprType::AllocateFusedReduction:
|
|
ptr(handler)->handle(expr->as<kir::AllocateFusedReduction>());
|
|
return;
|
|
default:
|
|
TORCH_INTERNAL_ASSERT(false, "Unknown exprtype in dispatch!");
|
|
}
|
|
}
|
|
|
|
template <typename T>
|
|
void Statement::constDispatch(T handler, const Statement* stmt) {
|
|
if (stmt->isVal()) {
|
|
ptr(handler)->handle(stmt->as<Val>());
|
|
} else if (stmt->isExpr()) {
|
|
ptr(handler)->handle(stmt->as<Expr>());
|
|
} else
|
|
TORCH_INTERNAL_ASSERT(false, "Unknown stmttype in dispatch!");
|
|
}
|
|
|
|
/*
|
|
* Generic mutatorDispatch for any handler that modifies the IR. This could be
|
|
* a transformation on loop structures, or parallelizing a loop. This
|
|
* mutatorDispatch is paired with a class that implements the functions
|
|
* template <typenname node_type> Statement* mutate(node_type* node) mutate
|
|
* should call (statement* node_to_dispatch)->mutatorDispatch() It could also
|
|
* implement Statement* mutate(Statement* stmt){ stmt->mutatorDispatch(this);
|
|
* }
|
|
* And therefore dispatch should never call:
|
|
* ptr(mutator)->mutate(this->as<Statement>());
|
|
*/
|
|
template <typename T>
|
|
void Val::mutatorDispatch(T mutator, Val* val) {
|
|
switch (*(val->getValType())) {
|
|
case ValType::Scalar:
|
|
switch (*(val->getDataType())) {
|
|
case DataType::Bool:
|
|
ptr(mutator)->mutate(val->as<Bool>());
|
|
return;
|
|
case DataType::Double:
|
|
ptr(mutator)->mutate(val->as<Double>());
|
|
return;
|
|
case DataType::Int:
|
|
ptr(mutator)->mutate(val->as<Int>());
|
|
return;
|
|
case DataType::ComplexDouble:
|
|
ptr(mutator)->mutate(val->as<ComplexDouble>());
|
|
return;
|
|
default:
|
|
break;
|
|
}
|
|
break;
|
|
case ValType::NamedScalar:
|
|
ptr(mutator)->mutate(val->as<NamedScalar>());
|
|
return;
|
|
|
|
case ValType::IterDomain:
|
|
ptr(mutator)->mutate(val->as<IterDomain>());
|
|
return;
|
|
case ValType::TensorDomain:
|
|
ptr(mutator)->mutate(val->as<TensorDomain>());
|
|
return;
|
|
case ValType::TensorView:
|
|
ptr(mutator)->mutate(val->as<TensorView>());
|
|
return;
|
|
case ValType::Predicate:
|
|
ptr(mutator)->mutate(val->as<kir::Predicate>());
|
|
return;
|
|
case ValType::TensorIndex:
|
|
ptr(mutator)->mutate(val->as<kir::TensorIndex>());
|
|
return;
|
|
default:
|
|
break;
|
|
}
|
|
TORCH_INTERNAL_ASSERT(false, "Unknown valtype in dispatch!");
|
|
}
|
|
|
|
template <typename T>
|
|
void Expr::mutatorDispatch(T mutator, Expr* expr) {
|
|
switch (*(expr->getExprType())) {
|
|
case ExprType::UnaryOp:
|
|
ptr(mutator)->mutate(expr->as<UnaryOp>());
|
|
return;
|
|
case ExprType::BinaryOp:
|
|
ptr(mutator)->mutate(expr->as<BinaryOp>());
|
|
return;
|
|
case ExprType::TernaryOp:
|
|
ptr(mutator)->mutate(expr->as<TernaryOp>());
|
|
return;
|
|
case ExprType::ReductionOp:
|
|
ptr(mutator)->mutate(expr->as<ReductionOp>());
|
|
return;
|
|
case ExprType::GroupedReductionOp:
|
|
ptr(mutator)->mutate(expr->as<GroupedReductionOp>());
|
|
return;
|
|
case ExprType::WelfordOp:
|
|
ptr(mutator)->mutate(expr->as<WelfordOp>());
|
|
return;
|
|
case ExprType::LoadStoreOp:
|
|
ptr(mutator)->mutate(expr->as<LoadStoreOp>());
|
|
return;
|
|
case ExprType::MmaOp:
|
|
ptr(mutator)->mutate(expr->as<MmaOp>());
|
|
return;
|
|
case ExprType::BroadcastOp:
|
|
ptr(mutator)->mutate(expr->as<BroadcastOp>());
|
|
return;
|
|
|
|
case ExprType::Split:
|
|
ptr(mutator)->mutate(expr->as<Split>());
|
|
return;
|
|
case ExprType::Merge:
|
|
ptr(mutator)->mutate(expr->as<Merge>());
|
|
return;
|
|
case ExprType::TransposeOp:
|
|
ptr(mutator)->mutate(expr->as<TransposeOp>());
|
|
return;
|
|
case ExprType::ExpandOp:
|
|
ptr(mutator)->mutate(expr->as<ExpandOp>());
|
|
return;
|
|
case ExprType::ShiftOp:
|
|
ptr(mutator)->mutate(expr->as<ShiftOp>());
|
|
return;
|
|
case ExprType::GatherOp:
|
|
ptr(mutator)->mutate(expr->as<GatherOp>());
|
|
return;
|
|
case ExprType::ViewAsScalar:
|
|
ptr(mutator)->mutate(expr->as<ViewAsScalar>());
|
|
return;
|
|
case ExprType::ViewOp:
|
|
ptr(mutator)->mutate(expr->as<ViewOp>());
|
|
return;
|
|
|
|
case ExprType::Allocate:
|
|
ptr(mutator)->mutate(expr->as<kir::Allocate>());
|
|
return;
|
|
case ExprType::BlockSync:
|
|
ptr(mutator)->mutate(expr->as<kir::BlockSync>());
|
|
return;
|
|
case ExprType::GridSync:
|
|
ptr(mutator)->mutate(expr->as<kir::GridSync>());
|
|
return;
|
|
case ExprType::CpAsyncWait:
|
|
ptr(mutator)->mutate(expr->as<kir::CpAsyncWait>());
|
|
return;
|
|
case ExprType::InitMagicZero:
|
|
ptr(mutator)->mutate(expr->as<kir::InitMagicZero>());
|
|
return;
|
|
case ExprType::UpdateMagicZero:
|
|
ptr(mutator)->mutate(expr->as<kir::UpdateMagicZero>());
|
|
return;
|
|
case ExprType::ForLoop:
|
|
ptr(mutator)->mutate(expr->as<kir::ForLoop>());
|
|
return;
|
|
case ExprType::IfThenElse:
|
|
ptr(mutator)->mutate(expr->as<kir::IfThenElse>());
|
|
return;
|
|
case ExprType::GridReduction:
|
|
ptr(mutator)->mutate(expr->as<kir::GridReduction>());
|
|
return;
|
|
case ExprType::GroupedGridReduction:
|
|
ptr(mutator)->mutate(expr->as<kir::GroupedGridReduction>());
|
|
return;
|
|
case ExprType::GridBroadcast:
|
|
ptr(mutator)->mutate(expr->as<kir::GridBroadcast>());
|
|
return;
|
|
case ExprType::GridWelford:
|
|
ptr(mutator)->mutate(expr->as<kir::GridWelford>());
|
|
return;
|
|
case ExprType::AllocateFusedReduction:
|
|
ptr(mutator)->mutate(expr->as<kir::AllocateFusedReduction>());
|
|
return;
|
|
default:
|
|
TORCH_INTERNAL_ASSERT(false, "Unknown exprtype in dispatch!");
|
|
}
|
|
}
|
|
|
|
template <typename T>
|
|
void Statement::mutatorDispatch(T mutator, Statement* stmt) {
|
|
if (stmt->isVal()) {
|
|
ptr(mutator)->mutate(stmt->as<Val>());
|
|
return;
|
|
}
|
|
if (stmt->isExpr()) {
|
|
ptr(mutator)->mutate(stmt->as<Expr>());
|
|
return;
|
|
}
|
|
TORCH_INTERNAL_ASSERT(false, "Unknown stmttype in dispatch!");
|
|
}
|
|
|
|
/*
|
|
* Handler template instantiations. These should only have to be done on base
|
|
* classes. Actual visitors/mutators should inhereit from these classes and call
|
|
* ->dispatch(this) to avoid needing an explicit instantiation.
|
|
*/
|
|
template void Statement::dispatch(OptOutDispatch&, Statement*);
|
|
template void Statement::dispatch(OptOutDispatch*, Statement*);
|
|
template void Val::dispatch(OptOutDispatch&, Val*);
|
|
template void Val::dispatch(OptOutDispatch*, Val*);
|
|
template void Expr::dispatch(OptOutDispatch&, Expr*);
|
|
template void Expr::dispatch(OptOutDispatch*, Expr*);
|
|
|
|
template void Statement::dispatch(OptInDispatch, Statement*);
|
|
template void Statement::dispatch(OptInDispatch*, Statement*);
|
|
template void Val::dispatch(OptInDispatch, Val*);
|
|
template void Val::dispatch(OptInDispatch*, Val*);
|
|
template void Expr::dispatch(OptInDispatch, Expr*);
|
|
template void Expr::dispatch(OptInDispatch*, Expr*);
|
|
|
|
template void Statement::constDispatch(OptOutConstDispatch&, const Statement*);
|
|
template void Statement::constDispatch(OptOutConstDispatch*, const Statement*);
|
|
template void Val::constDispatch(OptOutConstDispatch&, const Val*);
|
|
template void Val::constDispatch(OptOutConstDispatch*, const Val*);
|
|
template void Expr::constDispatch(OptOutConstDispatch&, const Expr*);
|
|
template void Expr::constDispatch(OptOutConstDispatch*, const Expr*);
|
|
|
|
template void Statement::constDispatch(OptInConstDispatch&, const Statement*);
|
|
template void Statement::constDispatch(OptInConstDispatch*, const Statement*);
|
|
template void Val::constDispatch(OptInConstDispatch&, const Val*);
|
|
template void Val::constDispatch(OptInConstDispatch*, const Val*);
|
|
template void Expr::constDispatch(OptInConstDispatch&, const Expr*);
|
|
template void Expr::constDispatch(OptInConstDispatch*, const Expr*);
|
|
|
|
template void Statement::mutatorDispatch(OptOutMutator&, Statement*);
|
|
template void Statement::mutatorDispatch(OptOutMutator*, Statement*);
|
|
template void Val::mutatorDispatch(OptOutMutator&, Val*);
|
|
template void Val::mutatorDispatch(OptOutMutator*, Val*);
|
|
template void Expr::mutatorDispatch(OptOutMutator&, Expr*);
|
|
template void Expr::mutatorDispatch(OptOutMutator*, Expr*);
|
|
|
|
void OptOutDispatch::handle(Statement* s) {
|
|
Statement::dispatch(this, s);
|
|
}
|
|
|
|
void OptOutDispatch::handle(Expr* e) {
|
|
Expr::dispatch(this, e);
|
|
}
|
|
|
|
void OptOutDispatch::handle(Val* v) {
|
|
Val::dispatch(this, v);
|
|
}
|
|
|
|
void OptOutConstDispatch::handle(const Statement* s) {
|
|
Statement::constDispatch(this, s);
|
|
}
|
|
|
|
void OptOutConstDispatch::handle(const Expr* e) {
|
|
Expr::constDispatch(this, e);
|
|
}
|
|
|
|
void OptOutConstDispatch::handle(const Val* v) {
|
|
Val::constDispatch(this, v);
|
|
}
|
|
|
|
void OptInConstDispatch::unhandled(const Statement* stmt) {
|
|
if (stmt->isExpr()) {
|
|
TORCH_INTERNAL_ASSERT(
|
|
false, "Handle not overriden for ", stmt->getExprType().value(), ".");
|
|
} else if (stmt->isVal()) {
|
|
TORCH_INTERNAL_ASSERT(
|
|
false, "Handle not overriden for ", stmt->getValType().value(), ".");
|
|
} else {
|
|
TORCH_INTERNAL_ASSERT(false, "Unrecognized statement type.");
|
|
}
|
|
}
|
|
|
|
void OptInDispatch::unhandled(Statement* stmt) {
|
|
if (stmt->isExpr()) {
|
|
TORCH_INTERNAL_ASSERT(
|
|
false, "Handle not overriden for ", stmt->getExprType().value(), ".");
|
|
} else if (stmt->isVal()) {
|
|
TORCH_INTERNAL_ASSERT(
|
|
false, "Handle not overriden for ", stmt->getValType().value(), ".");
|
|
} else {
|
|
TORCH_INTERNAL_ASSERT(false, "Unrecognized statement type.");
|
|
}
|
|
}
|
|
|
|
// Vals
|
|
void OptOutConstDispatch::handle(const Bool* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const Double* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const Int* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const ComplexDouble* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const NamedScalar* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const IterDomain* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const TensorDomain* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const TensorView* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
|
|
void OptOutConstDispatch::handle(const kir::Predicate* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const kir::TensorIndex* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
|
|
// Exprs
|
|
void OptOutConstDispatch::handle(const UnaryOp* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const BinaryOp* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const TernaryOp* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const ReductionOp* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const GroupedReductionOp* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const WelfordOp* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const LoadStoreOp* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const MmaOp* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const BroadcastOp* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
|
|
void OptOutConstDispatch::handle(const Split* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const Merge* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const TransposeOp* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const ExpandOp* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const ShiftOp* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const GatherOp* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const ViewAsScalar* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const ViewOp* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
|
|
void OptOutConstDispatch::handle(const kir::Allocate* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const kir::BlockSync* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const kir::GridSync* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const kir::CpAsyncWait* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const kir::InitMagicZero* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const kir::UpdateMagicZero* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const kir::ForLoop* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const kir::IfThenElse* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const kir::GridReduction* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const kir::GroupedGridReduction* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const kir::GridBroadcast* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const kir::GridWelford* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutConstDispatch::handle(const kir::AllocateFusedReduction* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
|
|
void OptOutDispatch::unhandled(Statement*) {}
|
|
|
|
// Vals
|
|
void OptOutDispatch::handle(Bool* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(Double* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(Int* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(ComplexDouble* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(NamedScalar* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(IterDomain* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(TensorDomain* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(TensorView* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
|
|
void OptOutDispatch::handle(kir::Predicate* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(kir::TensorIndex* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
|
|
// Exprs
|
|
void OptOutDispatch::handle(UnaryOp* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(BinaryOp* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(TernaryOp* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(ReductionOp* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(GroupedReductionOp* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(WelfordOp* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(LoadStoreOp* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(MmaOp* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(BroadcastOp* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
|
|
void OptOutDispatch::handle(Split* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(Merge* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(TransposeOp* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(ExpandOp* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(ShiftOp* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(GatherOp* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(ViewAsScalar* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(ViewOp* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
|
|
void OptOutDispatch::handle(kir::Allocate* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(kir::BlockSync* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(kir::GridSync* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(kir::CpAsyncWait* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(kir::InitMagicZero* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(kir::UpdateMagicZero* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(kir::ForLoop* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(kir::IfThenElse* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(kir::GridReduction* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(kir::GroupedGridReduction* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(kir::GridBroadcast* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(kir::GridWelford* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
void OptOutDispatch::handle(kir::AllocateFusedReduction* stmt) {
|
|
unhandled(stmt);
|
|
}
|
|
|
|
} // namespace cuda
|
|
} // namespace fuser
|
|
} // namespace jit
|
|
} // namespace torch
|