Merge triton-mlir branch - Complete rewrite of the backend from scratch (#1004)
This PR merges the `triton-mlir` branch, in which we have been quietly rewriting the Triton backend from scratch to increase maintainability, stability and ultimately performance. Changes to the runtime are minimal, and this new version aims to remain backward-compatible with the previous commit. The legacy backend is now officially deprecated, but can still be accessed via the `legacy-backend` tag. Co-authored-by: Keren Zhou <kerenzhou@openai.com> Co-authored-by: Yan Chunwei <yanchunwei@outlook.com> Co-authored-by: goostavz <109190422+goostavz@users.noreply.github.com> Co-authored-by: Shintaro Iwasaki <siwasaki@fb.com> Co-authored-by: Yan Da <dyanab@connect.ust.hk> Co-authored-by: Jun Yang <yangjunpro@gmail.com> Co-authored-by: Ian Bearman <ianb@microsoft.com> Co-authored-by: Jason Ansel <jansel@jansel.net> Co-authored-by: Qingyi Liu <qingyil@nvidia.com> Co-authored-by: ben-zhang-609 <110140741+ben-zhang-609@users.noreply.github.com> Co-authored-by: Chenggang Zhao <lyricz@yeah.net> Co-authored-by: ben-zhang-609 <benzh609@gmail.com> Co-authored-by: dongdongl <dongdongl@nvidia.com>
This commit is contained in:
Philippe Tillet
GitHub
11
lib/Dialect/Triton/Transforms/CMakeLists.txt
Normal file
11
lib/Dialect/Triton/Transforms/CMakeLists.txt
Normal file
@@ -0,0 +1,11 @@
|
||||
set(LLVM_TARGET_DEFINITIONS Combine.td)
|
||||
mlir_tablegen(TritonCombine.inc -gen-rewriters)
|
||||
add_public_tablegen_target(TritonCombineIncGen)
|
||||
|
||||
add_mlir_dialect_library(TritonTransforms
|
||||
Combine.cpp
|
||||
|
||||
DEPENDS
|
||||
TritonTransformsIncGen
|
||||
TritonCombineIncGen
|
||||
)
|
||||
209
lib/Dialect/Triton/Transforms/Combine.cpp
Normal file
209
lib/Dialect/Triton/Transforms/Combine.cpp
Normal file
@@ -0,0 +1,209 @@
|
||||
#include "mlir/IR/BuiltinAttributes.h"
|
||||
#include "mlir/IR/Matchers.h"
|
||||
#include "mlir/IR/PatternMatch.h"
|
||||
#include "mlir/Pass/Pass.h"
|
||||
#include "mlir/Support/LogicalResult.h"
|
||||
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
|
||||
|
||||
#include "triton/Dialect/Triton/IR/Dialect.h"
|
||||
#include "triton/Dialect/Triton/Transforms/Passes.h"
|
||||
|
||||
#include <memory>
|
||||
|
||||
using namespace mlir;
|
||||
|
||||
namespace {
|
||||
|
||||
bool isZero(mlir::Value val) {
|
||||
if (mlir::matchPattern(val, mlir::m_Zero()) ||
|
||||
mlir::matchPattern(val, mlir::m_AnyZeroFloat()))
|
||||
return true;
|
||||
// broadcast(constant_0)
|
||||
if (auto bc = val.getDefiningOp<mlir::triton::BroadcastOp>()) {
|
||||
if (mlir::matchPattern(bc.src(), mlir::m_Zero()) ||
|
||||
mlir::matchPattern(bc.src(), mlir::m_AnyZeroFloat()))
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool isBroadcastConstantCombinable(Attribute value) {
|
||||
if (auto denseValue = value.dyn_cast<DenseElementsAttr>()) {
|
||||
return denseValue.isSplat();
|
||||
}
|
||||
return value.isa<FloatAttr, IntegerAttr>();
|
||||
}
|
||||
|
||||
DenseElementsAttr getConstantValue(Builder &builder, Attribute value,
|
||||
Value bcast_res) {
|
||||
|
||||
Type resType = bcast_res.getType();
|
||||
DenseElementsAttr res;
|
||||
if (auto denseValue = value.dyn_cast<DenseElementsAttr>()) {
|
||||
res =
|
||||
DenseElementsAttr::get(resType, denseValue.getSplatValue<Attribute>());
|
||||
} else {
|
||||
res = DenseElementsAttr::get(resType, value);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
#include "TritonCombine.inc"
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
// select(cond, load(ptrs, broadcast(cond), ???), other)
|
||||
// => load(ptrs, broadcast(cond), other)
|
||||
class CombineSelectMaskedLoadPattern : public mlir::RewritePattern {
|
||||
public:
|
||||
CombineSelectMaskedLoadPattern(mlir::MLIRContext *context)
|
||||
: mlir::RewritePattern(mlir::SelectOp::getOperationName(), 3, context,
|
||||
{triton::LoadOp::getOperationName()}) {}
|
||||
|
||||
mlir::LogicalResult
|
||||
matchAndRewrite(mlir::Operation *op,
|
||||
mlir::PatternRewriter &rewriter) const override {
|
||||
auto selectOp = llvm::dyn_cast<mlir::SelectOp>(op);
|
||||
if (!selectOp)
|
||||
return mlir::failure();
|
||||
|
||||
mlir::Value trueValue = selectOp.getTrueValue();
|
||||
mlir::Value falseValue = selectOp.getFalseValue();
|
||||
|
||||
auto *loadOpCandidate = trueValue.getDefiningOp();
|
||||
auto loadOp = llvm::dyn_cast_or_null<triton::LoadOp>(loadOpCandidate);
|
||||
if (!loadOp)
|
||||
return mlir::failure();
|
||||
|
||||
mlir::Value mask = loadOp.mask();
|
||||
if (!mask)
|
||||
return mlir::failure();
|
||||
|
||||
auto *broadcastOpCandidate = mask.getDefiningOp();
|
||||
auto broadcastOp =
|
||||
llvm::dyn_cast_or_null<triton::BroadcastOp>(broadcastOpCandidate);
|
||||
if (!broadcastOp)
|
||||
return mlir::failure();
|
||||
|
||||
rewriter.replaceOpWithNewOp<triton::LoadOp>(
|
||||
op, loadOp.ptr(), loadOp.mask(), falseValue, loadOp.cache(),
|
||||
loadOp.evict(), loadOp.isVolatile());
|
||||
return mlir::success();
|
||||
}
|
||||
};
|
||||
|
||||
// load(ptr, splat(1), ...) -> load(ptr, ...)
|
||||
// load(ptr, splat(0), other, ...) -> other
|
||||
struct CanonicalizeMaskedLoadPattern
|
||||
: public mlir::OpRewritePattern<triton::LoadOp> {
|
||||
CanonicalizeMaskedLoadPattern(mlir::MLIRContext *context)
|
||||
: OpRewritePattern<triton::LoadOp>(context, 1) {}
|
||||
|
||||
mlir::LogicalResult
|
||||
matchAndRewrite(triton::LoadOp loadOp,
|
||||
mlir::PatternRewriter &rewriter) const override {
|
||||
auto mask = loadOp.mask();
|
||||
if (!mask)
|
||||
return mlir::failure();
|
||||
|
||||
auto constantMask =
|
||||
llvm::dyn_cast_or_null<arith::ConstantOp>(mask.getDefiningOp());
|
||||
if (!constantMask)
|
||||
return mlir::failure();
|
||||
|
||||
auto splatMask = constantMask.getValue().dyn_cast<SplatElementsAttr>();
|
||||
if (!splatMask)
|
||||
return mlir::failure();
|
||||
|
||||
if (splatMask.getSplatValue<IntegerAttr>().getValue() == true) {
|
||||
// mask = splat(1)
|
||||
rewriter.replaceOpWithNewOp<triton::LoadOp>(
|
||||
loadOp, loadOp.getType(), loadOp.ptr(), Value(), Value(),
|
||||
loadOp.cache(), loadOp.evict(), loadOp.isVolatile());
|
||||
} else {
|
||||
// mask = splat(0)
|
||||
|
||||
// If there's no "other", the value is "undef". Perhaps we want to
|
||||
// optimize it in the future.x
|
||||
auto otherVal = loadOp.other();
|
||||
if (!otherVal)
|
||||
return mlir::failure();
|
||||
rewriter.replaceOp(loadOp, otherVal);
|
||||
}
|
||||
return mlir::success();
|
||||
}
|
||||
};
|
||||
|
||||
void triton::LoadOp::getCanonicalizationPatterns(RewritePatternSet &results,
|
||||
MLIRContext *context) {
|
||||
results.add<CanonicalizeMaskedLoadPattern>(context);
|
||||
}
|
||||
|
||||
// store(ptr, value, splat(1), ...) -> store(ptr, value, ...)
|
||||
// store(ptr, value, splat(0), ...) -> [none]
|
||||
struct CanonicalizeMaskedStorePattern
|
||||
: public mlir::OpRewritePattern<triton::StoreOp> {
|
||||
CanonicalizeMaskedStorePattern(mlir::MLIRContext *context)
|
||||
: OpRewritePattern<triton::StoreOp>(context, 1) {}
|
||||
|
||||
mlir::LogicalResult
|
||||
matchAndRewrite(triton::StoreOp storeOp,
|
||||
mlir::PatternRewriter &rewriter) const override {
|
||||
auto mask = storeOp.mask();
|
||||
if (!mask)
|
||||
return mlir::failure();
|
||||
|
||||
auto constantMask =
|
||||
llvm::dyn_cast_or_null<arith::ConstantOp>(mask.getDefiningOp());
|
||||
if (!constantMask)
|
||||
return mlir::failure();
|
||||
|
||||
auto splatMask = constantMask.getValue().dyn_cast<SplatElementsAttr>();
|
||||
if (!splatMask)
|
||||
return mlir::failure();
|
||||
|
||||
if (splatMask.getSplatValue<IntegerAttr>().getValue() == true) {
|
||||
// mask = splat(1)
|
||||
rewriter.replaceOpWithNewOp<triton::StoreOp>(storeOp, storeOp.ptr(),
|
||||
storeOp.value());
|
||||
} else {
|
||||
// mask = splat(0)
|
||||
rewriter.eraseOp(storeOp);
|
||||
}
|
||||
return mlir::success();
|
||||
}
|
||||
};
|
||||
|
||||
void triton::StoreOp::getCanonicalizationPatterns(RewritePatternSet &results,
|
||||
MLIRContext *context) {
|
||||
results.add<CanonicalizeMaskedStorePattern>(context);
|
||||
}
|
||||
|
||||
#define GEN_PASS_CLASSES
|
||||
#include "triton/Dialect/Triton/Transforms/Passes.h.inc"
|
||||
|
||||
class CombineOpsPass : public TritonCombineOpsBase<CombineOpsPass> {
|
||||
public:
|
||||
void runOnOperation() override {
|
||||
mlir::MLIRContext *context = &getContext();
|
||||
mlir::RewritePatternSet patterns(context);
|
||||
mlir::ModuleOp m = getOperation();
|
||||
|
||||
// Dot Add %{
|
||||
patterns.add<CombineDotAddIPattern>(context);
|
||||
patterns.add<CombineDotAddFPattern>(context);
|
||||
patterns.add<CombineDotAddIRevPattern>(context);
|
||||
patterns.add<CombineDotAddFRevPattern>(context);
|
||||
// %}
|
||||
patterns.add<CombineSelectMaskedLoadPattern>(context);
|
||||
// patterns.add<CombineAddPtrPattern>(context);
|
||||
patterns.add<CombineBroadcastConstantPattern>(context);
|
||||
|
||||
if (applyPatternsAndFoldGreedily(m, std::move(patterns)).failed())
|
||||
signalPassFailure();
|
||||
}
|
||||
};
|
||||
|
||||
std::unique_ptr<mlir::Pass> mlir::triton::createCombineOpsPass() {
|
||||
return std::make_unique<CombineOpsPass>();
|
||||
}
|
||||
48
lib/Dialect/Triton/Transforms/Combine.td
Normal file
48
lib/Dialect/Triton/Transforms/Combine.td
Normal file
@@ -0,0 +1,48 @@
|
||||
#ifndef TRITON_PATTERNS
|
||||
#define TRITON_PATTERNS
|
||||
|
||||
include "mlir/Dialect/StandardOps/IR/Ops.td"
|
||||
include "mlir/Dialect/Arithmetic/IR/ArithmeticOps.td"
|
||||
include "triton/Dialect/Triton/IR/TritonOps.td"
|
||||
|
||||
|
||||
// AddIOp(DotOp(a, b, c), d) and c==0 => DotOp(a, b, d)
|
||||
// AddFOp(DotOp(a, b, c), d) and c==0 => DotOp(a, b, d)
|
||||
|
||||
// AddIOp(d, DotOp(a, b, c)) and c==0 => DotOp(a, b, d)
|
||||
// AddFOp(d, DotOp(a, b, c)) and c==0 => DotOp(a, b, d)
|
||||
def CombineDotAddIPattern : Pat<
|
||||
(Arith_AddIOp $d, (TT_DotOp:$res $a, $b, $c, $allowTF32)),
|
||||
(TT_DotOp $a, $b, $d, $allowTF32),
|
||||
[(Constraint<CPred<"isZero($0)">> $c)]>;
|
||||
def CombineDotAddFPattern : Pat<
|
||||
(Arith_AddFOp $d, (TT_DotOp:$res $a, $b, $c, $allowTF32)),
|
||||
(TT_DotOp $a, $b, $d, $allowTF32),
|
||||
[(Constraint<CPred<"isZero($0)">> $c)]>;
|
||||
|
||||
def CombineDotAddIRevPattern : Pat<
|
||||
(Arith_AddIOp (TT_DotOp:$res $a, $b, $c, $allowTF32), $d),
|
||||
(TT_DotOp $a, $b, $d, $allowTF32),
|
||||
[(Constraint<CPred<"isZero($0)">> $c)]>;
|
||||
def CombineDotAddFRevPattern : Pat<
|
||||
(Arith_AddFOp (TT_DotOp:$res $a, $b, $c, $allowTF32), $d),
|
||||
(TT_DotOp $a, $b, $d, $allowTF32),
|
||||
[(Constraint<CPred<"isZero($0)">> $c)]>;
|
||||
|
||||
// TODO: this fails for addptr(addptr(ptr, i32), i64)
|
||||
// Commented out until fixed
|
||||
// addptr(addptr(%ptr, %idx0), %idx1) => addptr(%ptr, AddI(%idx0, %idx1))
|
||||
// Note: leave (sub %c0, %c0) canceling to ArithmeticDialect
|
||||
// (ref: ArithmeticCanonicalization.td)
|
||||
// def CombineAddPtrPattern : Pat<
|
||||
// (TT_AddPtrOp (TT_AddPtrOp $ptr, $idx0), $idx1),
|
||||
// (TT_AddPtrOp $ptr, (Arith_AddIOp $idx0, $idx1))>;
|
||||
|
||||
// broadcast(cst) => cst
|
||||
def getConstantValue : NativeCodeCall<"getConstantValue($_builder, $0, $1)">;
|
||||
def CombineBroadcastConstantPattern : Pat<
|
||||
(TT_BroadcastOp:$bcast_res (Arith_ConstantOp $value)),
|
||||
(Arith_ConstantOp (getConstantValue $value, $bcast_res)),
|
||||
[(Constraint<CPred<"isBroadcastConstantCombinable($0)">> $value)]>;
|
||||
|
||||
#endif
|
||||
Reference in New Issue
Block a user