Loading...
Searching...
No Matches
IRMatch.h
Go to the documentation of this file.
#ifndef HALIDE_IR_MATCH_H
#define HALIDE_IR_MATCH_H
/** \file
* Defines a method to match a fragment of IR against a pattern containing wildcards
*/
#include <map>
#include <random>
#include <set>
#include <vector>
#include "IR.h"
#include "IREquality.h"
#include "IROperator.h"
namespace Halide {
namespace Internal {
/** Does the first expression have the same structure as the second?
* Variables in the first expression with the name * are interpreted
* as wildcards, and their matching equivalent in the second
* expression is placed in the vector give as the third argument.
* Wildcards require the types to match. For the type bits and width,
* a 0 indicates "match anything". So an Int(8, 0) will match 8-bit
* integer vectors of any width (including scalars), and a UInt(0, 0)
* will match any unsigned integer type.
*
* For example:
\code
Expr x = Variable::make(Int(32), "*");
match(x + x, 3 + (2*k), result)
\endcode
* should return true, and set result[0] to 3 and
* result[1] to 2*k.
*/
bool expr_match(const Expr &pattern, const Expr &expr, std::vector<Expr> &result);
/** Does the first expression have the same structure as the second?
* Variables are matched consistently. The first time a variable is
* matched, it assumes the value of the matching part of the second
* expression. Subsequent matches must be equal to the first match.
*
* For example:
\code
Var x("x"), y("y");
match(x*(x + y), a*(a + b), result)
\endcode
* should return true, and set result["x"] = a, and result["y"] = b.
*/
bool expr_match(const Expr &pattern, const Expr &expr, std::map<std::string, Expr> &result);
/** Rewrite the expression x to have `lanes` lanes. This is useful
* for substituting the results of expr_match into a pattern expression. */
Expr with_lanes(const Expr &x, int lanes);
void expr_match_test();
/** An alternative template-metaprogramming approach to expression
* matching. Potentially more efficient. We lift the expression
* pattern into a type, and then use force-inlined functions to
* generate efficient matching and reconstruction code for any
* pattern. Pattern elements are either one of the classes in the
* namespace IRMatcher, or are non-null Exprs (represented as
* BaseExprNode &).
*
* Pattern elements that are fully specified by their pattern can be
* built into an expression using the make method. Some patterns,
* such as a broadcast that matches any number of lanes, don't have
* enough information to recreate an Expr.
*/
namespace IRMatcher {
constexpr int max_wild = 6;
static const halide_type_t i64_type = {halide_type_int, 64, 1};
/** To save stack space, the matcher objects are largely stateless and
* immutable. This state object is built up during matching and then
* consumed when constructing a replacement Expr.
*/
struct MatcherState {
const BaseExprNode *bindings[max_wild];
halide_scalar_value_t bound_const[max_wild];
// values of the lanes field with special meaning.
static constexpr uint16_t signed_integer_overflow = 0x8000;
static constexpr uint16_t special_values_mask = 0x8000; // currently only one
halide_type_t bound_const_type[max_wild];
HALIDE_ALWAYS_INLINE
void set_binding(int i, const BaseExprNode &n) noexcept {
bindings[i] = &n;
}
HALIDE_ALWAYS_INLINE
const BaseExprNode *get_binding(int i) const noexcept {
return bindings[i];
}
HALIDE_ALWAYS_INLINE
void set_bound_const(int i, int64_t s, halide_type_t t) noexcept {
bound_const[i].u.i64 = s;
bound_const_type[i] = t;
}
HALIDE_ALWAYS_INLINE
void set_bound_const(int i, uint64_t u, halide_type_t t) noexcept {
bound_const[i].u.u64 = u;
bound_const_type[i] = t;
}
HALIDE_ALWAYS_INLINE
void set_bound_const(int i, double f, halide_type_t t) noexcept {
bound_const[i].u.f64 = f;
bound_const_type[i] = t;
}
HALIDE_ALWAYS_INLINE
void set_bound_const(int i, halide_scalar_value_t val, halide_type_t t) noexcept {
bound_const[i] = val;
bound_const_type[i] = t;
}
HALIDE_ALWAYS_INLINE
void get_bound_const(int i, halide_scalar_value_t &val, halide_type_t &type) const noexcept {
val = bound_const[i];
type = bound_const_type[i];
}
HALIDE_ALWAYS_INLINE
// NOLINTNEXTLINE(modernize-use-equals-default): Can't use `= default`; clang-tidy complains about noexcept mismatch
MatcherState() noexcept {
}
};
template<typename T,
typename = typename std::remove_reference<T>::type::pattern_tag>
struct enable_if_pattern {
struct type {};
};
template<typename T>
struct bindings {
constexpr static uint32_t mask = std::remove_reference<T>::type::binds;
};
inline HALIDE_NEVER_INLINE Expr make_const_special_expr(halide_type_t ty) {
const uint16_t flags = ty.lanes & MatcherState::special_values_mask;
ty.lanes &= ~MatcherState::special_values_mask;
if (flags & MatcherState::signed_integer_overflow) {
return make_signed_integer_overflow(ty);
}
// unreachable
return Expr();
}
HALIDE_ALWAYS_INLINE
Expr make_const_expr(halide_scalar_value_t val, halide_type_t ty) {
halide_type_t scalar_type = ty;
if (scalar_type.lanes & MatcherState::special_values_mask) {
return make_const_special_expr(scalar_type);
}
const int lanes = scalar_type.lanes;
scalar_type.lanes = 1;
Expr e;
switch (scalar_type.code) {
case halide_type_int:
e = IntImm::make(scalar_type, val.u.i64);
break;
case halide_type_uint:
e = UIntImm::make(scalar_type, val.u.u64);
break;
case halide_type_float:
case halide_type_bfloat:
e = FloatImm::make(scalar_type, val.u.f64);
break;
default:
// Unreachable
return Expr();
}
if (lanes > 1) {
e = Broadcast::make(std::move(e), lanes);
}
return e;
}
// A pattern that matches a specific expression
struct SpecificExpr {
struct pattern_tag {};
constexpr static uint32_t binds = 0;
// What is the weakest and strongest IR node this could possibly be
constexpr static IRNodeType min_node_type = IRNodeType::IntImm;
constexpr static IRNodeType max_node_type = StrongestExprNodeType;
constexpr static bool canonical = true;
const BaseExprNode &expr;
template<uint32_t bound>
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
return equal(expr, e);
}
HALIDE_ALWAYS_INLINE
Expr make(MatcherState &state, halide_type_t type_hint) const {
return Expr(&expr);
}
constexpr static bool foldable = false;
};
inline std::ostream &operator<<(std::ostream &s, const SpecificExpr &e) {
s << Expr(&e.expr);
return s;
}
template<int i>
struct WildConstInt {
struct pattern_tag {};
constexpr static uint32_t binds = 1 << i;
constexpr static IRNodeType min_node_type = IRNodeType::IntImm;
constexpr static IRNodeType max_node_type = IRNodeType::IntImm;
constexpr static bool canonical = true;
template<uint32_t bound>
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
static_assert(i >= 0 && i < max_wild, "Wild with out-of-range index");
const BaseExprNode *op = &e;
if (op->node_type == IRNodeType::Broadcast) {
op = ((const Broadcast *)op)->value.get();
}
if (op->node_type != IRNodeType::IntImm) {
return false;
}
int64_t value = ((const IntImm *)op)->value;
if (bound & binds) {
halide_scalar_value_t val;
halide_type_t type;
state.get_bound_const(i, val, type);
return (halide_type_t)e.type == type && value == val.u.i64;
}
state.set_bound_const(i, value, e.type);
return true;
}
template<uint32_t bound>
HALIDE_ALWAYS_INLINE bool match(int64_t value, MatcherState &state) const noexcept {
static_assert(i >= 0 && i < max_wild, "Wild with out-of-range index");
if (bound & binds) {
halide_scalar_value_t val;
halide_type_t type;
state.get_bound_const(i, val, type);
return type == i64_type && value == val.u.i64;
}
state.set_bound_const(i, value, i64_type);
return true;
}
HALIDE_ALWAYS_INLINE
Expr make(MatcherState &state, halide_type_t type_hint) const {
halide_scalar_value_t val;
halide_type_t type;
state.get_bound_const(i, val, type);
return make_const_expr(val, type);
}
constexpr static bool foldable = true;
HALIDE_ALWAYS_INLINE
void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const {
state.get_bound_const(i, val, ty);
}
};
template<int i>
std::ostream &operator<<(std::ostream &s, const WildConstInt<i> &c) {
s << "ci" << i;
return s;
}
template<int i>
struct WildConstUInt {
struct pattern_tag {};
constexpr static uint32_t binds = 1 << i;
constexpr static IRNodeType min_node_type = IRNodeType::UIntImm;
constexpr static IRNodeType max_node_type = IRNodeType::UIntImm;
constexpr static bool canonical = true;
template<uint32_t bound>
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
static_assert(i >= 0 && i < max_wild, "Wild with out-of-range index");
const BaseExprNode *op = &e;
if (op->node_type == IRNodeType::Broadcast) {
op = ((const Broadcast *)op)->value.get();
}
if (op->node_type != IRNodeType::UIntImm) {
return false;
}
uint64_t value = ((const UIntImm *)op)->value;
if (bound & binds) {
halide_scalar_value_t val;
halide_type_t type;
state.get_bound_const(i, val, type);
return (halide_type_t)e.type == type && value == val.u.u64;
}
state.set_bound_const(i, value, e.type);
return true;
}
HALIDE_ALWAYS_INLINE
Expr make(MatcherState &state, halide_type_t type_hint) const {
halide_scalar_value_t val;
halide_type_t type;
state.get_bound_const(i, val, type);
return make_const_expr(val, type);
}
constexpr static bool foldable = true;
HALIDE_ALWAYS_INLINE
void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept {
state.get_bound_const(i, val, ty);
}
};
template<int i>
std::ostream &operator<<(std::ostream &s, const WildConstUInt<i> &c) {
s << "cu" << i;
return s;
}
template<int i>
struct WildConstFloat {
struct pattern_tag {};
constexpr static uint32_t binds = 1 << i;
constexpr static IRNodeType min_node_type = IRNodeType::FloatImm;
constexpr static IRNodeType max_node_type = IRNodeType::FloatImm;
constexpr static bool canonical = true;
template<uint32_t bound>
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
static_assert(i >= 0 && i < max_wild, "Wild with out-of-range index");
const BaseExprNode *op = &e;
if (op->node_type == IRNodeType::Broadcast) {
op = ((const Broadcast *)op)->value.get();
}
if (op->node_type != IRNodeType::FloatImm) {
return false;
}
double value = ((const FloatImm *)op)->value;
if (bound & binds) {
halide_scalar_value_t val;
halide_type_t type;
state.get_bound_const(i, val, type);
return (halide_type_t)e.type == type && value == val.u.f64;
}
state.set_bound_const(i, value, e.type);
return true;
}
HALIDE_ALWAYS_INLINE
Expr make(MatcherState &state, halide_type_t type_hint) const {
halide_scalar_value_t val;
halide_type_t type;
state.get_bound_const(i, val, type);
return make_const_expr(val, type);
}
constexpr static bool foldable = true;
HALIDE_ALWAYS_INLINE
void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept {
state.get_bound_const(i, val, ty);
}
};
template<int i>
std::ostream &operator<<(std::ostream &s, const WildConstFloat<i> &c) {
s << "cf" << i;
return s;
}
// Matches and binds to any constant Expr. Does not support constant-folding.
template<int i>
struct WildConst {
struct pattern_tag {};
constexpr static uint32_t binds = 1 << i;
constexpr static IRNodeType min_node_type = IRNodeType::IntImm;
constexpr static IRNodeType max_node_type = IRNodeType::FloatImm;
constexpr static bool canonical = true;
template<uint32_t bound>
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
static_assert(i >= 0 && i < max_wild, "Wild with out-of-range index");
const BaseExprNode *op = &e;
if (op->node_type == IRNodeType::Broadcast) {
op = ((const Broadcast *)op)->value.get();
}
switch (op->node_type) {
case IRNodeType::IntImm:
return WildConstInt<i>().template match<bound>(e, state);
case IRNodeType::UIntImm:
return WildConstUInt<i>().template match<bound>(e, state);
case IRNodeType::FloatImm:
return WildConstFloat<i>().template match<bound>(e, state);
default:
return false;
}
}
template<uint32_t bound>
HALIDE_ALWAYS_INLINE bool match(int64_t e, MatcherState &state) const noexcept {
static_assert(i >= 0 && i < max_wild, "Wild with out-of-range index");
return WildConstInt<i>().template match<bound>(e, state);
}
HALIDE_ALWAYS_INLINE
Expr make(MatcherState &state, halide_type_t type_hint) const {
halide_scalar_value_t val;
halide_type_t type;
state.get_bound_const(i, val, type);
return make_const_expr(val, type);
}
constexpr static bool foldable = true;
HALIDE_ALWAYS_INLINE
void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept {
state.get_bound_const(i, val, ty);
}
};
template<int i>
std::ostream &operator<<(std::ostream &s, const WildConst<i> &c) {
s << "c" << i;
return s;
}
// Matches and binds to any Expr
template<int i>
struct Wild {
struct pattern_tag {};
constexpr static uint32_t binds = 1 << (i + 16);
constexpr static IRNodeType min_node_type = IRNodeType::IntImm;
constexpr static IRNodeType max_node_type = StrongestExprNodeType;
constexpr static bool canonical = true;
template<uint32_t bound>
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
if (bound & binds) {
return equal(*state.get_binding(i), e);
}
state.set_binding(i, e);
return true;
}
HALIDE_ALWAYS_INLINE
Expr make(MatcherState &state, halide_type_t type_hint) const {
return state.get_binding(i);
}
constexpr static bool foldable = false;
};
template<int i>
std::ostream &operator<<(std::ostream &s, const Wild<i> &op) {
s << "_" << i;
return s;
}
// Matches a specific constant or broadcast of that constant. The
// constant must be representable as an int64_t.
struct IntLiteral {
struct pattern_tag {};
int64_t v;
constexpr static uint32_t binds = 0;
constexpr static IRNodeType min_node_type = IRNodeType::IntImm;
constexpr static IRNodeType max_node_type = IRNodeType::FloatImm;
constexpr static bool canonical = true;
HALIDE_ALWAYS_INLINE
explicit IntLiteral(int64_t v)
: v(v) {
}
template<uint32_t bound>
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
const BaseExprNode *op = &e;
if (e.node_type == IRNodeType::Broadcast) {
op = ((const Broadcast *)op)->value.get();
}
switch (op->node_type) {
case IRNodeType::IntImm:
return ((const IntImm *)op)->value == (int64_t)v;
case IRNodeType::UIntImm:
return ((const UIntImm *)op)->value == (uint64_t)v;
case IRNodeType::FloatImm:
return ((const FloatImm *)op)->value == (double)v;
default:
return false;
}
}
template<uint32_t bound>
HALIDE_ALWAYS_INLINE bool match(int64_t val, MatcherState &state) const noexcept {
return v == val;
}
template<uint32_t bound>
HALIDE_ALWAYS_INLINE bool match(const IntLiteral &b, MatcherState &state) const noexcept {
return v == b.v;
}
HALIDE_ALWAYS_INLINE
Expr make(MatcherState &state, halide_type_t type_hint) const {
return make_const(type_hint, v);
}
constexpr static bool foldable = true;
HALIDE_ALWAYS_INLINE
void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept {
// Assume type is already correct
switch (ty.code) {
case halide_type_int:
val.u.i64 = v;
break;
case halide_type_uint:
val.u.u64 = (uint64_t)v;
break;
case halide_type_float:
case halide_type_bfloat:
val.u.f64 = (double)v;
break;
default:
// Unreachable
;
}
}
};
HALIDE_ALWAYS_INLINE int64_t unwrap(IntLiteral t) {
return t.v;
}
// Convert a provided pattern, expr, or constant int into the internal
// representation we use in the matcher trees.
template<typename T,
typename = typename std::decay<T>::type::pattern_tag>
HALIDE_ALWAYS_INLINE T pattern_arg(T t) {
return t;
}
HALIDE_ALWAYS_INLINE
IntLiteral pattern_arg(int64_t x) {
return IntLiteral{x};
}
template<typename T>
HALIDE_ALWAYS_INLINE void assert_is_lvalue_if_expr() {
static_assert(!std::is_same<typename std::decay<T>::type, Expr>::value || std::is_lvalue_reference<T>::value,
"Exprs are captured by reference by IRMatcher objects and so must be lvalues");
}
HALIDE_ALWAYS_INLINE SpecificExpr pattern_arg(const Expr &e) {
return {*e.get()};
}
// Helpers to deref SpecificExprs to const BaseExprNode & rather than
// passing them by value anywhere (incurring lots of refcounting)
template<typename T,
// T must be a pattern node
typename = typename std::decay<T>::type::pattern_tag,
// But T may not be SpecificExpr
typename = typename std::enable_if<!std::is_same<typename std::decay<T>::type, SpecificExpr>::value>::type>
HALIDE_ALWAYS_INLINE T unwrap(T t) {
return t;
}
HALIDE_ALWAYS_INLINE
const BaseExprNode &unwrap(const SpecificExpr &e) {
return e.expr;
}
inline std::ostream &operator<<(std::ostream &s, const IntLiteral &op) {
s << op.v;
return s;
}
template<typename Op>
int64_t constant_fold_bin_op(halide_type_t &, int64_t, int64_t) noexcept;
template<typename Op>
uint64_t constant_fold_bin_op(halide_type_t &, uint64_t, uint64_t) noexcept;
template<typename Op>
double constant_fold_bin_op(halide_type_t &, double, double) noexcept;
constexpr bool commutative(IRNodeType t) {
return (t == IRNodeType::Add ||
t == IRNodeType::Mul ||
t == IRNodeType::And ||
t == IRNodeType::Or ||
t == IRNodeType::Min ||
t == IRNodeType::Max ||
t == IRNodeType::EQ ||
t == IRNodeType::NE);
}
// Matches one of the binary operators
template<typename Op, typename A, typename B>
struct BinOp {
struct pattern_tag {};
A a;
B b;
constexpr static uint32_t binds = bindings<A>::mask | bindings<B>::mask;
constexpr static IRNodeType min_node_type = Op::_node_type;
constexpr static IRNodeType max_node_type = Op::_node_type;
// For commutative bin ops, we expect the weaker IR node type on
// the right. That is, for the rule to be canonical it must be
// possible that A is at least as strong as B.
constexpr static bool canonical =
A::canonical && B::canonical && (!commutative(Op::_node_type) || (A::max_node_type >= B::min_node_type));
template<uint32_t bound>
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
if (e.node_type != Op::_node_type) {
return false;
}
const Op &op = (const Op &)e;
return (a.template match<bound>(*op.a.get(), state) &&
b.template match<(bound | bindings<A>::mask)>(*op.b.get(), state));
}
template<uint32_t bound, typename Op2, typename A2, typename B2>
HALIDE_ALWAYS_INLINE bool match(const BinOp<Op2, A2, B2> &op, MatcherState &state) const noexcept {
return (std::is_same<Op, Op2>::value &&
a.template match<bound>(unwrap(op.a), state) &&
b.template match<(bound | bindings<A>::mask)>(unwrap(op.b), state));
}
constexpr static bool foldable = A::foldable && B::foldable;
HALIDE_ALWAYS_INLINE
void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept {
halide_scalar_value_t val_a, val_b;
if (std::is_same<A, IntLiteral>::value) {
b.make_folded_const(val_b, ty, state);
if ((std::is_same<Op, And>::value && val_b.u.u64 == 0) ||
(std::is_same<Op, Or>::value && val_b.u.u64 == 1)) {
// Short circuit
val = val_b;
return;
}
const uint16_t l = ty.lanes;
a.make_folded_const(val_a, ty, state);
ty.lanes |= l; // Make sure the overflow bits are sticky
} else {
a.make_folded_const(val_a, ty, state);
if ((std::is_same<Op, And>::value && val_a.u.u64 == 0) ||
(std::is_same<Op, Or>::value && val_a.u.u64 == 1)) {
// Short circuit
val = val_a;
return;
}
const uint16_t l = ty.lanes;
b.make_folded_const(val_b, ty, state);
ty.lanes |= l;
}
switch (ty.code) {
case halide_type_int:
val.u.i64 = constant_fold_bin_op<Op>(ty, val_a.u.i64, val_b.u.i64);
break;
case halide_type_uint:
val.u.u64 = constant_fold_bin_op<Op>(ty, val_a.u.u64, val_b.u.u64);
break;
case halide_type_float:
case halide_type_bfloat:
val.u.f64 = constant_fold_bin_op<Op>(ty, val_a.u.f64, val_b.u.f64);
break;
default:
// unreachable
;
}
}
HALIDE_ALWAYS_INLINE
Expr make(MatcherState &state, halide_type_t type_hint) const noexcept {
Expr ea, eb;
if (std::is_same<A, IntLiteral>::value) {
eb = b.make(state, type_hint);
ea = a.make(state, eb.type());
} else {
ea = a.make(state, type_hint);
eb = b.make(state, ea.type());
}
return Op::make(std::move(ea), std::move(eb));
}
};
template<typename Op>
uint64_t constant_fold_cmp_op(int64_t, int64_t) noexcept;
template<typename Op>
uint64_t constant_fold_cmp_op(uint64_t, uint64_t) noexcept;
template<typename Op>
uint64_t constant_fold_cmp_op(double, double) noexcept;
// Matches one of the comparison operators
template<typename Op, typename A, typename B>
struct CmpOp {
struct pattern_tag {};
A a;
B b;
constexpr static uint32_t binds = bindings<A>::mask | bindings<B>::mask;
constexpr static IRNodeType min_node_type = Op::_node_type;
constexpr static IRNodeType max_node_type = Op::_node_type;
constexpr static bool canonical = (A::canonical &&
B::canonical &&
(!commutative(Op::_node_type) || A::max_node_type >= B::min_node_type) &&
(Op::_node_type != IRNodeType::GE) &&
(Op::_node_type != IRNodeType::GT));
template<uint32_t bound>
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
if (e.node_type != Op::_node_type) {
return false;
}
const Op &op = (const Op &)e;
return (a.template match<bound>(*op.a.get(), state) &&
b.template match<(bound | bindings<A>::mask)>(*op.b.get(), state));
}
template<uint32_t bound, typename Op2, typename A2, typename B2>
HALIDE_ALWAYS_INLINE bool match(const CmpOp<Op2, A2, B2> &op, MatcherState &state) const noexcept {
return (std::is_same<Op, Op2>::value &&
a.template match<bound>(unwrap(op.a), state) &&
b.template match<(bound | bindings<A>::mask)>(unwrap(op.b), state));
}
constexpr static bool foldable = A::foldable && B::foldable;
HALIDE_ALWAYS_INLINE
void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept {
halide_scalar_value_t val_a, val_b;
// If one side is an untyped const, evaluate the other side first to get a type hint.
if (std::is_same<A, IntLiteral>::value) {
b.make_folded_const(val_b, ty, state);
const uint16_t l = ty.lanes;
a.make_folded_const(val_a, ty, state);
ty.lanes |= l;
} else {
a.make_folded_const(val_a, ty, state);
const uint16_t l = ty.lanes;
b.make_folded_const(val_b, ty, state);
ty.lanes |= l;
}
switch (ty.code) {
case halide_type_int:
val.u.u64 = constant_fold_cmp_op<Op>(val_a.u.i64, val_b.u.i64);
break;
case halide_type_uint:
val.u.u64 = constant_fold_cmp_op<Op>(val_a.u.u64, val_b.u.u64);
break;
case halide_type_float:
case halide_type_bfloat:
val.u.u64 = constant_fold_cmp_op<Op>(val_a.u.f64, val_b.u.f64);
break;
default:
// unreachable
;
}
ty.code = halide_type_uint;
ty.bits = 1;
}
HALIDE_ALWAYS_INLINE
Expr make(MatcherState &state, halide_type_t type_hint) const {
// If one side is an untyped const, evaluate the other side first to get a type hint.
Expr ea, eb;
if (std::is_same<A, IntLiteral>::value) {
eb = b.make(state, {});
ea = a.make(state, eb.type());
} else {
ea = a.make(state, {});
eb = b.make(state, ea.type());
}
return Op::make(std::move(ea), std::move(eb));
}
};
template<typename A, typename B>
std::ostream &operator<<(std::ostream &s, const BinOp<Add, A, B> &op) {
s << "(" << op.a << " + " << op.b << ")";
return s;
}
template<typename A, typename B>
std::ostream &operator<<(std::ostream &s, const BinOp<Sub, A, B> &op) {
s << "(" << op.a << " - " << op.b << ")";
return s;
}
template<typename A, typename B>
std::ostream &operator<<(std::ostream &s, const BinOp<Mul, A, B> &op) {
s << "(" << op.a << " * " << op.b << ")";
return s;
}
template<typename A, typename B>
std::ostream &operator<<(std::ostream &s, const BinOp<Div, A, B> &op) {
s << "(" << op.a << " / " << op.b << ")";
return s;
}
template<typename A, typename B>
std::ostream &operator<<(std::ostream &s, const BinOp<And, A, B> &op) {
s << "(" << op.a << " && " << op.b << ")";
return s;
}
template<typename A, typename B>
std::ostream &operator<<(std::ostream &s, const BinOp<Or, A, B> &op) {
s << "(" << op.a << " || " << op.b << ")";
return s;
}
template<typename A, typename B>
std::ostream &operator<<(std::ostream &s, const BinOp<Min, A, B> &op) {
s << "min(" << op.a << ", " << op.b << ")";
return s;
}
template<typename A, typename B>
std::ostream &operator<<(std::ostream &s, const BinOp<Max, A, B> &op) {
s << "max(" << op.a << ", " << op.b << ")";
return s;
}
template<typename A, typename B>
std::ostream &operator<<(std::ostream &s, const CmpOp<LE, A, B> &op) {
s << "(" << op.a << " <= " << op.b << ")";
return s;
}
template<typename A, typename B>
std::ostream &operator<<(std::ostream &s, const CmpOp<LT, A, B> &op) {
s << "(" << op.a << " < " << op.b << ")";
return s;
}
template<typename A, typename B>
std::ostream &operator<<(std::ostream &s, const CmpOp<GE, A, B> &op) {
s << "(" << op.a << " >= " << op.b << ")";
return s;
}
template<typename A, typename B>
std::ostream &operator<<(std::ostream &s, const CmpOp<GT, A, B> &op) {
s << "(" << op.a << " > " << op.b << ")";
return s;
}
template<typename A, typename B>
std::ostream &operator<<(std::ostream &s, const CmpOp<EQ, A, B> &op) {
s << "(" << op.a << " == " << op.b << ")";
return s;
}
template<typename A, typename B>
std::ostream &operator<<(std::ostream &s, const CmpOp<NE, A, B> &op) {
s << "(" << op.a << " != " << op.b << ")";
return s;
}
template<typename A, typename B>
std::ostream &operator<<(std::ostream &s, const BinOp<Mod, A, B> &op) {
s << "(" << op.a << " % " << op.b << ")";
return s;
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto operator+(A &&a, B &&b) noexcept -> BinOp<Add, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
assert_is_lvalue_if_expr<A>();
assert_is_lvalue_if_expr<B>();
return {pattern_arg(a), pattern_arg(b)};
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto add(A &&a, B &&b) -> decltype(IRMatcher::operator+(a, b)) {
assert_is_lvalue_if_expr<A>();
assert_is_lvalue_if_expr<B>();
return IRMatcher::operator+(a, b);
}
template<>
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op<Add>(halide_type_t &t, int64_t a, int64_t b) noexcept {
t.lanes |= ((t.bits >= 32) && add_would_overflow(t.bits, a, b)) ? MatcherState::signed_integer_overflow : 0;
int dead_bits = 64 - t.bits;
// Drop the high bits then sign-extend them back
return int64_t((uint64_t(a) + uint64_t(b)) << dead_bits) >> dead_bits;
}
template<>
HALIDE_ALWAYS_INLINE uint64_t constant_fold_bin_op<Add>(halide_type_t &t, uint64_t a, uint64_t b) noexcept {
uint64_t ones = (uint64_t)(-1);
return (a + b) & (ones >> (64 - t.bits));
}
template<>
HALIDE_ALWAYS_INLINE double constant_fold_bin_op<Add>(halide_type_t &t, double a, double b) noexcept {
return a + b;
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto operator-(A &&a, B &&b) noexcept -> BinOp<Sub, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
assert_is_lvalue_if_expr<A>();
assert_is_lvalue_if_expr<B>();
return {pattern_arg(a), pattern_arg(b)};
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto sub(A &&a, B &&b) -> decltype(IRMatcher::operator-(a, b)) {
assert_is_lvalue_if_expr<A>();
assert_is_lvalue_if_expr<B>();
return IRMatcher::operator-(a, b);
}
template<>
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op<Sub>(halide_type_t &t, int64_t a, int64_t b) noexcept {
t.lanes |= ((t.bits >= 32) && sub_would_overflow(t.bits, a, b)) ? MatcherState::signed_integer_overflow : 0;
// Drop the high bits then sign-extend them back
int dead_bits = 64 - t.bits;
return int64_t((uint64_t(a) - uint64_t(b)) << dead_bits) >> dead_bits;
}
template<>
HALIDE_ALWAYS_INLINE uint64_t constant_fold_bin_op<Sub>(halide_type_t &t, uint64_t a, uint64_t b) noexcept {
uint64_t ones = (uint64_t)(-1);
return (a - b) & (ones >> (64 - t.bits));
}
template<>
HALIDE_ALWAYS_INLINE double constant_fold_bin_op<Sub>(halide_type_t &t, double a, double b) noexcept {
return a - b;
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto operator*(A &&a, B &&b) noexcept -> BinOp<Mul, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
assert_is_lvalue_if_expr<A>();
assert_is_lvalue_if_expr<B>();
return {pattern_arg(a), pattern_arg(b)};
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto mul(A &&a, B &&b) -> decltype(IRMatcher::operator*(a, b)) {
assert_is_lvalue_if_expr<A>();
assert_is_lvalue_if_expr<B>();
return IRMatcher::operator*(a, b);
}
template<>
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op<Mul>(halide_type_t &t, int64_t a, int64_t b) noexcept {
t.lanes |= ((t.bits >= 32) && mul_would_overflow(t.bits, a, b)) ? MatcherState::signed_integer_overflow : 0;
int dead_bits = 64 - t.bits;
// Drop the high bits then sign-extend them back
return int64_t((uint64_t(a) * uint64_t(b)) << dead_bits) >> dead_bits;
}
template<>
HALIDE_ALWAYS_INLINE uint64_t constant_fold_bin_op<Mul>(halide_type_t &t, uint64_t a, uint64_t b) noexcept {
uint64_t ones = (uint64_t)(-1);
return (a * b) & (ones >> (64 - t.bits));
}
template<>
HALIDE_ALWAYS_INLINE double constant_fold_bin_op<Mul>(halide_type_t &t, double a, double b) noexcept {
return a * b;
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto operator/(A &&a, B &&b) noexcept -> BinOp<Div, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
assert_is_lvalue_if_expr<A>();
assert_is_lvalue_if_expr<B>();
return {pattern_arg(a), pattern_arg(b)};
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto div(A &&a, B &&b) -> decltype(IRMatcher::operator/(a, b)) {
return IRMatcher::operator/(a, b);
}
template<>
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op<Div>(halide_type_t &t, int64_t a, int64_t b) noexcept {
return div_imp(a, b);
}
template<>
HALIDE_ALWAYS_INLINE uint64_t constant_fold_bin_op<Div>(halide_type_t &t, uint64_t a, uint64_t b) noexcept {
return div_imp(a, b);
}
template<>
HALIDE_ALWAYS_INLINE double constant_fold_bin_op<Div>(halide_type_t &t, double a, double b) noexcept {
return div_imp(a, b);
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto operator%(A &&a, B &&b) noexcept -> BinOp<Mod, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
assert_is_lvalue_if_expr<A>();
assert_is_lvalue_if_expr<B>();
return {pattern_arg(a), pattern_arg(b)};
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto mod(A &&a, B &&b) -> decltype(IRMatcher::operator%(a, b)) {
assert_is_lvalue_if_expr<A>();
assert_is_lvalue_if_expr<B>();
return IRMatcher::operator%(a, b);
}
template<>
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op<Mod>(halide_type_t &t, int64_t a, int64_t b) noexcept {
return mod_imp(a, b);
}
template<>
HALIDE_ALWAYS_INLINE uint64_t constant_fold_bin_op<Mod>(halide_type_t &t, uint64_t a, uint64_t b) noexcept {
return mod_imp(a, b);
}
template<>
HALIDE_ALWAYS_INLINE double constant_fold_bin_op<Mod>(halide_type_t &t, double a, double b) noexcept {
return mod_imp(a, b);
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto min(A &&a, B &&b) noexcept -> BinOp<Min, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
assert_is_lvalue_if_expr<A>();
assert_is_lvalue_if_expr<B>();
return {pattern_arg(a), pattern_arg(b)};
}
template<>
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op<Min>(halide_type_t &t, int64_t a, int64_t b) noexcept {
return std::min(a, b);
}
template<>
HALIDE_ALWAYS_INLINE uint64_t constant_fold_bin_op<Min>(halide_type_t &t, uint64_t a, uint64_t b) noexcept {
return std::min(a, b);
}
template<>
HALIDE_ALWAYS_INLINE double constant_fold_bin_op<Min>(halide_type_t &t, double a, double b) noexcept {
return std::min(a, b);
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto max(A &&a, B &&b) noexcept -> BinOp<Max, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
assert_is_lvalue_if_expr<A>();
assert_is_lvalue_if_expr<B>();
return {pattern_arg(std::forward<A>(a)), pattern_arg(std::forward<B>(b))};
}
template<>
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op<Max>(halide_type_t &t, int64_t a, int64_t b) noexcept {
return std::max(a, b);
}
template<>
HALIDE_ALWAYS_INLINE uint64_t constant_fold_bin_op<Max>(halide_type_t &t, uint64_t a, uint64_t b) noexcept {
return std::max(a, b);
}
template<>
HALIDE_ALWAYS_INLINE double constant_fold_bin_op<Max>(halide_type_t &t, double a, double b) noexcept {
return std::max(a, b);
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto operator<(A &&a, B &&b) noexcept -> CmpOp<LT, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
return {pattern_arg(a), pattern_arg(b)};
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto lt(A &&a, B &&b) -> decltype(IRMatcher::operator<(a, b)) {
return IRMatcher::operator<(a, b);
}
template<>
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op<LT>(int64_t a, int64_t b) noexcept {
return a < b;
}
template<>
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op<LT>(uint64_t a, uint64_t b) noexcept {
return a < b;
}
template<>
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op<LT>(double a, double b) noexcept {
return a < b;
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto operator>(A &&a, B &&b) noexcept -> CmpOp<GT, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
return {pattern_arg(a), pattern_arg(b)};
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto gt(A &&a, B &&b) -> decltype(IRMatcher::operator>(a, b)) {
return IRMatcher::operator>(a, b);
}
template<>
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op<GT>(int64_t a, int64_t b) noexcept {
return a > b;
}
template<>
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op<GT>(uint64_t a, uint64_t b) noexcept {
return a > b;
}
template<>
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op<GT>(double a, double b) noexcept {
return a > b;
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto operator<=(A &&a, B &&b) noexcept -> CmpOp<LE, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
return {pattern_arg(a), pattern_arg(b)};
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto le(A &&a, B &&b) -> decltype(IRMatcher::operator<=(a, b)) {
return IRMatcher::operator<=(a, b);
}
template<>
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op<LE>(int64_t a, int64_t b) noexcept {
return a <= b;
}
template<>
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op<LE>(uint64_t a, uint64_t b) noexcept {
return a <= b;
}
template<>
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op<LE>(double a, double b) noexcept {
return a <= b;
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto operator>=(A &&a, B &&b) noexcept -> CmpOp<GE, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
return {pattern_arg(a), pattern_arg(b)};
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto ge(A &&a, B &&b) -> decltype(IRMatcher::operator>=(a, b)) {
return IRMatcher::operator>=(a, b);
}
template<>
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op<GE>(int64_t a, int64_t b) noexcept {
return a >= b;
}
template<>
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op<GE>(uint64_t a, uint64_t b) noexcept {
return a >= b;
}
template<>
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op<GE>(double a, double b) noexcept {
return a >= b;
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto operator==(A &&a, B &&b) noexcept -> CmpOp<EQ, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
return {pattern_arg(a), pattern_arg(b)};
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto eq(A &&a, B &&b) -> decltype(IRMatcher::operator==(a, b)) {
return IRMatcher::operator==(a, b);
}
template<>
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op<EQ>(int64_t a, int64_t b) noexcept {
return a == b;
}
template<>
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op<EQ>(uint64_t a, uint64_t b) noexcept {
return a == b;
}
template<>
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op<EQ>(double a, double b) noexcept {
return a == b;
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto operator!=(A &&a, B &&b) noexcept -> CmpOp<NE, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
return {pattern_arg(a), pattern_arg(b)};
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto ne(A &&a, B &&b) -> decltype(IRMatcher::operator!=(a, b)) {
return IRMatcher::operator!=(a, b);
}
template<>
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op<NE>(int64_t a, int64_t b) noexcept {
return a != b;
}
template<>
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op<NE>(uint64_t a, uint64_t b) noexcept {
return a != b;
}
template<>
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op<NE>(double a, double b) noexcept {
return a != b;
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto operator||(A &&a, B &&b) noexcept -> BinOp<Or, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
return {pattern_arg(a), pattern_arg(b)};
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto or_op(A &&a, B &&b) -> decltype(IRMatcher::operator||(a, b)) {
return IRMatcher::operator||(a, b);
}
template<>
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op<Or>(halide_type_t &t, int64_t a, int64_t b) noexcept {
return (a | b) & 1;
}
template<>
HALIDE_ALWAYS_INLINE uint64_t constant_fold_bin_op<Or>(halide_type_t &t, uint64_t a, uint64_t b) noexcept {
return (a | b) & 1;
}
template<>
HALIDE_ALWAYS_INLINE double constant_fold_bin_op<Or>(halide_type_t &t, double a, double b) noexcept {
// Unreachable, as it would be a type mismatch.
return 0;
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto operator&&(A &&a, B &&b) noexcept -> BinOp<And, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
return {pattern_arg(a), pattern_arg(b)};
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto and_op(A &&a, B &&b) -> decltype(IRMatcher::operator&&(a, b)) {
return IRMatcher::operator&&(a, b);
}
template<>
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op<And>(halide_type_t &t, int64_t a, int64_t b) noexcept {
return a & b & 1;
}
template<>
HALIDE_ALWAYS_INLINE uint64_t constant_fold_bin_op<And>(halide_type_t &t, uint64_t a, uint64_t b) noexcept {
return a & b & 1;
}
template<>
HALIDE_ALWAYS_INLINE double constant_fold_bin_op<And>(halide_type_t &t, double a, double b) noexcept {
// Unreachable
return 0;
}
constexpr uint32_t bitwise_or_reduce() {
return 0;
}
template<typename... Args>
constexpr uint32_t bitwise_or_reduce(uint32_t first, Args... rest) {
return first | bitwise_or_reduce(rest...);
}
constexpr bool and_reduce() {
return true;
}
template<typename... Args>
constexpr bool and_reduce(bool first, Args... rest) {
return first && and_reduce(rest...);
}
template<Call::IntrinsicOp intrin>
struct OptionalIntrinType {
bool check(const Type &) const {
return true;
}
};
template<>
struct OptionalIntrinType<Call::saturating_cast> {
halide_type_t type;
bool check(const Type &t) const {
return t == Type(type);
}
};
template<Call::IntrinsicOp intrin, typename... Args>
struct Intrin {
struct pattern_tag {};
std::tuple<Args...> args;
// The type of the output of the intrinsic node.
// Only necessary in cases where it can't be inferred
// from the input types (e.g. saturating_cast).
OptionalIntrinType<intrin> optional_type_hint;
static constexpr uint32_t binds = bitwise_or_reduce((bindings<Args>::mask)...);
constexpr static IRNodeType min_node_type = IRNodeType::Call;
constexpr static IRNodeType max_node_type = IRNodeType::Call;
constexpr static bool canonical = and_reduce((Args::canonical)...);
template<int i,
uint32_t bound,
typename = typename std::enable_if<(i < sizeof...(Args))>::type>
HALIDE_ALWAYS_INLINE bool match_args(int, const Call &c, MatcherState &state) const noexcept {
using T = decltype(std::get<i>(args));
return (std::get<i>(args).template match<bound>(*c.args[i].get(), state) &&
match_args<i + 1, (bound | bindings<T>::mask)>(0, c, state));
}
template<int i, uint32_t binds>
HALIDE_ALWAYS_INLINE bool match_args(double, const Call &c, MatcherState &state) const noexcept {
return true;
}
template<uint32_t bound>
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
if (e.node_type != IRNodeType::Call) {
return false;
}
const Call &c = (const Call &)e;
return (c.is_intrinsic(intrin) &&
optional_type_hint.check(e.type) &&
match_args<0, bound>(0, c, state));
}
template<int i,
typename = typename std::enable_if<(i < sizeof...(Args))>::type>
HALIDE_ALWAYS_INLINE void print_args(int, std::ostream &s) const {
s << std::get<i>(args);
if (i + 1 < sizeof...(Args)) {
s << ", ";
}
print_args<i + 1>(0, s);
}
template<int i>
HALIDE_ALWAYS_INLINE void print_args(double, std::ostream &s) const {
}
HALIDE_ALWAYS_INLINE
void print_args(std::ostream &s) const {
print_args<0>(0, s);
}
HALIDE_ALWAYS_INLINE
Expr make(MatcherState &state, halide_type_t type_hint) const {
Expr arg0 = std::get<0>(args).make(state, type_hint);
if (intrin == Call::likely) {
return likely(std::move(arg0));
} else if (intrin == Call::likely_if_innermost) {
return likely_if_innermost(std::move(arg0));
} else if (intrin == Call::abs) {
return abs(std::move(arg0));
} else if constexpr (intrin == Call::saturating_cast) {
return saturating_cast(optional_type_hint.type, std::move(arg0));
}
Expr arg1 = std::get<std::min<size_t>(1, sizeof...(Args) - 1)>(args).make(state, type_hint);
if (intrin == Call::absd) {
return absd(std::move(arg0), std::move(arg1));
} else if (intrin == Call::widen_right_add) {
return widen_right_add(std::move(arg0), std::move(arg1));
} else if (intrin == Call::widen_right_mul) {
return widen_right_mul(std::move(arg0), std::move(arg1));
} else if (intrin == Call::widen_right_sub) {
return widen_right_sub(std::move(arg0), std::move(arg1));
} else if (intrin == Call::widening_add) {
return widening_add(std::move(arg0), std::move(arg1));
} else if (intrin == Call::widening_sub) {
return widening_sub(std::move(arg0), std::move(arg1));
} else if (intrin == Call::widening_mul) {
return widening_mul(std::move(arg0), std::move(arg1));
} else if (intrin == Call::saturating_add) {
return saturating_add(std::move(arg0), std::move(arg1));
} else if (intrin == Call::saturating_sub) {
return saturating_sub(std::move(arg0), std::move(arg1));
} else if (intrin == Call::halving_add) {
return halving_add(std::move(arg0), std::move(arg1));
} else if (intrin == Call::halving_sub) {
return halving_sub(std::move(arg0), std::move(arg1));
} else if (intrin == Call::rounding_halving_add) {
return rounding_halving_add(std::move(arg0), std::move(arg1));
} else if (intrin == Call::shift_left) {
return std::move(arg0) << std::move(arg1);
} else if (intrin == Call::shift_right) {
return std::move(arg0) >> std::move(arg1);
} else if (intrin == Call::rounding_shift_left) {
return rounding_shift_left(std::move(arg0), std::move(arg1));
} else if (intrin == Call::rounding_shift_right) {
return rounding_shift_right(std::move(arg0), std::move(arg1));
}
Expr arg2 = std::get<std::min<size_t>(2, sizeof...(Args) - 1)>(args).make(state, type_hint);
if (intrin == Call::mul_shift_right) {
return mul_shift_right(std::move(arg0), std::move(arg1), std::move(arg2));
} else if (intrin == Call::rounding_mul_shift_right) {
return rounding_mul_shift_right(std::move(arg0), std::move(arg1), std::move(arg2));
}
internal_error << "Unhandled intrinsic in IRMatcher: " << intrin;
return Expr();
}
constexpr static bool foldable = true;
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept {
halide_scalar_value_t arg1;
// Assuming the args have the same type as the intrinsic is incorrect in
// general. But for the intrinsics we can fold (just shifts), the LHS
// has the same type as the intrinsic, and we can always treat the RHS
// as a signed int, because we're using 64 bits for it.
std::get<0>(args).make_folded_const(val, ty, state);
halide_type_t signed_ty = ty;
signed_ty.code = halide_type_int;
// We can just directly get the second arg here, because we only want to
// instantiate this method for shifts, which have two args.
std::get<1>(args).make_folded_const(arg1, signed_ty, state);
if (intrin == Call::shift_left) {
if (arg1.u.i64 < 0) {
if (ty.code == halide_type_int) {
// Arithmetic shift
val.u.i64 >>= -arg1.u.i64;
} else {
// Logical shift
val.u.u64 >>= -arg1.u.i64;
}
} else {
val.u.u64 <<= arg1.u.i64;
}
} else if (intrin == Call::shift_right) {
if (arg1.u.i64 > 0) {
if (ty.code == halide_type_int) {
// Arithmetic shift
val.u.i64 >>= arg1.u.i64;
} else {
// Logical shift
val.u.u64 >>= arg1.u.i64;
}
} else {
val.u.u64 <<= -arg1.u.i64;
}
} else {
internal_error << "Folding not implemented for intrinsic: " << intrin;
}
}
HALIDE_ALWAYS_INLINE
Intrin(Args... args) noexcept
: args(args...) {
}
};
template<Call::IntrinsicOp intrin, typename... Args>
std::ostream &operator<<(std::ostream &s, const Intrin<intrin, Args...> &op) {
s << intrin << "(";
op.print_args(s);
s << ")";
return s;
}
template<typename A, typename B>
auto widen_right_add(A &&a, B &&b) noexcept -> Intrin<Call::widen_right_add, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
return {pattern_arg(a), pattern_arg(b)};
}
template<typename A, typename B>
auto widen_right_mul(A &&a, B &&b) noexcept -> Intrin<Call::widen_right_mul, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
return {pattern_arg(a), pattern_arg(b)};
}
template<typename A, typename B>
auto widen_right_sub(A &&a, B &&b) noexcept -> Intrin<Call::widen_right_sub, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
return {pattern_arg(a), pattern_arg(b)};
}
template<typename A, typename B>
auto widening_add(A &&a, B &&b) noexcept -> Intrin<Call::widening_add, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
return {pattern_arg(a), pattern_arg(b)};
}
template<typename A, typename B>
auto widening_sub(A &&a, B &&b) noexcept -> Intrin<Call::widening_sub, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
return {pattern_arg(a), pattern_arg(b)};
}
template<typename A, typename B>
auto widening_mul(A &&a, B &&b) noexcept -> Intrin<Call::widening_mul, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
return {pattern_arg(a), pattern_arg(b)};
}
template<typename A, typename B>
auto saturating_add(A &&a, B &&b) noexcept -> Intrin<Call::saturating_add, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
return {pattern_arg(a), pattern_arg(b)};
}
template<typename A, typename B>
auto saturating_sub(A &&a, B &&b) noexcept -> Intrin<Call::saturating_sub, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
return {pattern_arg(a), pattern_arg(b)};
}
template<typename A>
auto saturating_cast(const Type &t, A &&a) noexcept -> Intrin<Call::saturating_cast, decltype(pattern_arg(a))> {
Intrin<Call::saturating_cast, decltype(pattern_arg(a))> p = {pattern_arg(a)};
p.optional_type_hint.type = t;
return p;
}
template<typename A, typename B>
auto halving_add(A &&a, B &&b) noexcept -> Intrin<Call::halving_add, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
return {pattern_arg(a), pattern_arg(b)};
}
template<typename A, typename B>
auto halving_sub(A &&a, B &&b) noexcept -> Intrin<Call::halving_sub, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
return {pattern_arg(a), pattern_arg(b)};
}
template<typename A, typename B>
auto rounding_halving_add(A &&a, B &&b) noexcept -> Intrin<Call::rounding_halving_add, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
return {pattern_arg(a), pattern_arg(b)};
}
template<typename A, typename B>
auto shift_left(A &&a, B &&b) noexcept -> Intrin<Call::shift_left, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
return {pattern_arg(a), pattern_arg(b)};
}
template<typename A, typename B>
auto shift_right(A &&a, B &&b) noexcept -> Intrin<Call::shift_right, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
return {pattern_arg(a), pattern_arg(b)};
}
template<typename A, typename B>
auto rounding_shift_left(A &&a, B &&b) noexcept -> Intrin<Call::rounding_shift_left, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
return {pattern_arg(a), pattern_arg(b)};
}
template<typename A, typename B>
auto rounding_shift_right(A &&a, B &&b) noexcept -> Intrin<Call::rounding_shift_right, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
return {pattern_arg(a), pattern_arg(b)};
}
template<typename A, typename B, typename C>
auto mul_shift_right(A &&a, B &&b, C &&c) noexcept -> Intrin<Call::mul_shift_right, decltype(pattern_arg(a)), decltype(pattern_arg(b)), decltype(pattern_arg(c))> {
return {pattern_arg(a), pattern_arg(b), pattern_arg(c)};
}
template<typename A, typename B, typename C>
auto rounding_mul_shift_right(A &&a, B &&b, C &&c) noexcept -> Intrin<Call::rounding_mul_shift_right, decltype(pattern_arg(a)), decltype(pattern_arg(b)), decltype(pattern_arg(c))> {
return {pattern_arg(a), pattern_arg(b), pattern_arg(c)};
}
template<typename A>
auto abs(A &&a) noexcept -> Intrin<Call::abs, decltype(pattern_arg(a))> {
return {pattern_arg(a)};
}
template<typename A, typename B>
auto absd(A &&a, B &&b) noexcept -> Intrin<Call::absd, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
return {pattern_arg(a), pattern_arg(b)};
}
template<typename A>
auto likely(A &&a) noexcept -> Intrin<Call::likely, decltype(pattern_arg(a))> {
return {pattern_arg(a)};
}
template<typename A>
auto likely_if_innermost(A &&a) noexcept -> Intrin<Call::likely_if_innermost, decltype(pattern_arg(a))> {
return {pattern_arg(a)};
}
template<typename A>
struct NotOp {
struct pattern_tag {};
A a;
constexpr static uint32_t binds = bindings<A>::mask;
constexpr static IRNodeType min_node_type = IRNodeType::Not;
constexpr static IRNodeType max_node_type = IRNodeType::Not;
constexpr static bool canonical = A::canonical;
template<uint32_t bound>
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
if (e.node_type != IRNodeType::Not) {
return false;
}
const Not &op = (const Not &)e;
return (a.template match<bound>(*op.a.get(), state));
}
template<uint32_t bound, typename A2>
HALIDE_ALWAYS_INLINE bool match(const NotOp<A2> &op, MatcherState &state) const noexcept {
return a.template match<bound>(unwrap(op.a), state);
}
HALIDE_ALWAYS_INLINE
Expr make(MatcherState &state, halide_type_t type_hint) const {
return Not::make(a.make(state, type_hint));
}
constexpr static bool foldable = A::foldable;
template<typename A1 = A>
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept {
a.make_folded_const(val, ty, state);
val.u.u64 = ~val.u.u64;
val.u.u64 &= 1;
}
};
template<typename A>
HALIDE_ALWAYS_INLINE auto operator!(A &&a) noexcept -> NotOp<decltype(pattern_arg(a))> {
assert_is_lvalue_if_expr<A>();
return {pattern_arg(a)};
}
template<typename A>
HALIDE_ALWAYS_INLINE auto not_op(A &&a) -> decltype(IRMatcher::operator!(a)) {
assert_is_lvalue_if_expr<A>();
return IRMatcher::operator!(a);
}
template<typename A>
inline std::ostream &operator<<(std::ostream &s, const NotOp<A> &op) {
s << "!(" << op.a << ")";
return s;
}
template<typename C, typename T, typename F>
struct SelectOp {
struct pattern_tag {};
C c;
T t;
F f;
constexpr static uint32_t binds = bindings<C>::mask | bindings<T>::mask | bindings<F>::mask;
constexpr static IRNodeType min_node_type = IRNodeType::Select;
constexpr static IRNodeType max_node_type = IRNodeType::Select;
constexpr static bool canonical = C::canonical && T::canonical && F::canonical;
template<uint32_t bound>
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
if (e.node_type != Select::_node_type) {
return false;
}
const Select &op = (const Select &)e;
return (c.template match<bound>(*op.condition.get(), state) &&
t.template match<(bound | bindings<C>::mask)>(*op.true_value.get(), state) &&
f.template match<(bound | bindings<C>::mask | bindings<T>::mask)>(*op.false_value.get(), state));
}
template<uint32_t bound, typename C2, typename T2, typename F2>
HALIDE_ALWAYS_INLINE bool match(const SelectOp<C2, T2, F2> &instance, MatcherState &state) const noexcept {
return (c.template match<bound>(unwrap(instance.c), state) &&
t.template match<(bound | bindings<C>::mask)>(unwrap(instance.t), state) &&
f.template match<(bound | bindings<C>::mask | bindings<T>::mask)>(unwrap(instance.f), state));
}
HALIDE_ALWAYS_INLINE
Expr make(MatcherState &state, halide_type_t type_hint) const {
return Select::make(c.make(state, {}), t.make(state, type_hint), f.make(state, type_hint));
}
constexpr static bool foldable = C::foldable && T::foldable && F::foldable;
template<typename C1 = C>
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept {
halide_scalar_value_t c_val, t_val, f_val;
halide_type_t c_ty;
c.make_folded_const(c_val, c_ty, state);
if ((c_val.u.u64 & 1) == 1) {
t.make_folded_const(val, ty, state);
} else {
f.make_folded_const(val, ty, state);
}
ty.lanes |= c_ty.lanes & MatcherState::special_values_mask;
}
};
template<typename C, typename T, typename F>
std::ostream &operator<<(std::ostream &s, const SelectOp<C, T, F> &op) {
s << "select(" << op.c << ", " << op.t << ", " << op.f << ")";
return s;
}
template<typename C, typename T, typename F>
HALIDE_ALWAYS_INLINE auto select(C &&c, T &&t, F &&f) noexcept -> SelectOp<decltype(pattern_arg(c)), decltype(pattern_arg(t)), decltype(pattern_arg(f))> {
assert_is_lvalue_if_expr<C>();
assert_is_lvalue_if_expr<T>();
assert_is_lvalue_if_expr<F>();
return {pattern_arg(c), pattern_arg(t), pattern_arg(f)};
}
template<typename A, typename B>
struct BroadcastOp {
struct pattern_tag {};
A a;
B lanes;
constexpr static uint32_t binds = bindings<A>::mask | bindings<B>::mask;
constexpr static IRNodeType min_node_type = IRNodeType::Broadcast;
constexpr static IRNodeType max_node_type = IRNodeType::Broadcast;
constexpr static bool canonical = A::canonical && B::canonical;
template<uint32_t bound>
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
if (e.node_type == Broadcast::_node_type) {
const Broadcast &op = (const Broadcast &)e;
if (a.template match<bound>(*op.value.get(), state) &&
lanes.template match<bound>(op.lanes, state)) {
return true;
}
}
return false;
}
template<uint32_t bound, typename A2, typename B2>
HALIDE_ALWAYS_INLINE bool match(const BroadcastOp<A2, B2> &op, MatcherState &state) const noexcept {
return (a.template match<bound>(unwrap(op.a), state) &&
lanes.template match<(bound | bindings<A>::mask)>(unwrap(op.lanes), state));
}
HALIDE_ALWAYS_INLINE
Expr make(MatcherState &state, halide_type_t type_hint) const {
halide_scalar_value_t lanes_val;
halide_type_t ty;
lanes.make_folded_const(lanes_val, ty, state);
int32_t l = (int32_t)lanes_val.u.i64;
type_hint.lanes /= l;
Expr val = a.make(state, type_hint);
if (l == 1) {
return val;
} else {
return Broadcast::make(std::move(val), l);
}
}
constexpr static bool foldable = false;
template<typename A1 = A>
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept {
halide_scalar_value_t lanes_val;
halide_type_t lanes_ty;
lanes.make_folded_const(lanes_val, lanes_ty, state);
uint16_t l = (uint16_t)lanes_val.u.i64;
a.make_folded_const(val, ty, state);
ty.lanes = l | (ty.lanes & MatcherState::special_values_mask);
}
};
template<typename A, typename B>
inline std::ostream &operator<<(std::ostream &s, const BroadcastOp<A, B> &op) {
s << "broadcast(" << op.a << ", " << op.lanes << ")";
return s;
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto broadcast(A &&a, B lanes) noexcept -> BroadcastOp<decltype(pattern_arg(a)), decltype(pattern_arg(lanes))> {
assert_is_lvalue_if_expr<A>();
return {pattern_arg(a), pattern_arg(lanes)};
}
template<typename A, typename B, typename C>
struct RampOp {
struct pattern_tag {};
A a;
B b;
C lanes;
constexpr static uint32_t binds = bindings<A>::mask | bindings<B>::mask | bindings<C>::mask;
constexpr static IRNodeType min_node_type = IRNodeType::Ramp;
constexpr static IRNodeType max_node_type = IRNodeType::Ramp;
constexpr static bool canonical = A::canonical && B::canonical && C::canonical;
template<uint32_t bound>
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
if (e.node_type != Ramp::_node_type) {
return false;
}
const Ramp &op = (const Ramp &)e;
if (a.template match<bound>(*op.base.get(), state) &&
b.template match<(bound | bindings<A>::mask)>(*op.stride.get(), state) &&
lanes.template match<(bound | bindings<A>::mask | bindings<B>::mask)>(op.lanes, state)) {
return true;
} else {
return false;
}
}
template<uint32_t bound, typename A2, typename B2, typename C2>
HALIDE_ALWAYS_INLINE bool match(const RampOp<A2, B2, C2> &op, MatcherState &state) const noexcept {
return (a.template match<bound>(unwrap(op.a), state) &&
b.template match<(bound | bindings<A>::mask)>(unwrap(op.b), state) &&
lanes.template match<(bound | bindings<A>::mask | bindings<B>::mask)>(unwrap(op.lanes), state));
}
HALIDE_ALWAYS_INLINE
Expr make(MatcherState &state, halide_type_t type_hint) const {
halide_scalar_value_t lanes_val;
halide_type_t ty;
lanes.make_folded_const(lanes_val, ty, state);
int32_t l = (int32_t)lanes_val.u.i64;
type_hint.lanes /= l;
Expr ea, eb;
eb = b.make(state, type_hint);
ea = a.make(state, eb.type());
return Ramp::make(std::move(ea), std::move(eb), l);
}
constexpr static bool foldable = false;
};
template<typename A, typename B, typename C>
std::ostream &operator<<(std::ostream &s, const RampOp<A, B, C> &op) {
s << "ramp(" << op.a << ", " << op.b << ", " << op.lanes << ")";
return s;
}
template<typename A, typename B, typename C>
HALIDE_ALWAYS_INLINE auto ramp(A &&a, B &&b, C &&c) noexcept -> RampOp<decltype(pattern_arg(a)), decltype(pattern_arg(b)), decltype(pattern_arg(c))> {
assert_is_lvalue_if_expr<A>();
assert_is_lvalue_if_expr<B>();
assert_is_lvalue_if_expr<C>();
return {pattern_arg(a), pattern_arg(b), pattern_arg(c)};
}
template<typename A, typename B, VectorReduce::Operator reduce_op>
struct VectorReduceOp {
struct pattern_tag {};
A a;
B lanes;
constexpr static uint32_t binds = bindings<A>::mask | bindings<B>::mask;
constexpr static IRNodeType min_node_type = IRNodeType::VectorReduce;
constexpr static IRNodeType max_node_type = IRNodeType::VectorReduce;
constexpr static bool canonical = A::canonical;
template<uint32_t bound>
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
if (e.node_type == VectorReduce::_node_type) {
const VectorReduce &op = (const VectorReduce &)e;
if (op.op == reduce_op &&
a.template match<bound>(*op.value.get(), state) &&
lanes.template match<(bound | bindings<A>::mask)>(op.type.lanes(), state)) {
return true;
}
}
return false;
}
template<uint32_t bound, typename A2, typename B2, VectorReduce::Operator reduce_op_2>
HALIDE_ALWAYS_INLINE bool match(const VectorReduceOp<A2, B2, reduce_op_2> &op, MatcherState &state) const noexcept {
return (reduce_op == reduce_op_2 &&
a.template match<bound>(unwrap(op.a), state) &&
lanes.template match<(bound | bindings<A>::mask)>(unwrap(op.lanes), state));
}
HALIDE_ALWAYS_INLINE
Expr make(MatcherState &state, halide_type_t type_hint) const {
halide_scalar_value_t lanes_val;
halide_type_t ty;
lanes.make_folded_const(lanes_val, ty, state);
int l = (int)lanes_val.u.i64;
return VectorReduce::make(reduce_op, a.make(state, type_hint), l);
}
constexpr static bool foldable = false;
};
template<typename A, typename B, VectorReduce::Operator reduce_op>
inline std::ostream &operator<<(std::ostream &s, const VectorReduceOp<A, B, reduce_op> &op) {
s << "vector_reduce(" << reduce_op << ", " << op.a << ", " << op.lanes << ")";
return s;
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto h_add(A &&a, B lanes) noexcept -> VectorReduceOp<decltype(pattern_arg(a)), decltype(pattern_arg(lanes)), VectorReduce::Add> {
assert_is_lvalue_if_expr<A>();
return {pattern_arg(a), pattern_arg(lanes)};
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto h_min(A &&a, B lanes) noexcept -> VectorReduceOp<decltype(pattern_arg(a)), decltype(pattern_arg(lanes)), VectorReduce::Min> {
assert_is_lvalue_if_expr<A>();
return {pattern_arg(a), pattern_arg(lanes)};
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto h_max(A &&a, B lanes) noexcept -> VectorReduceOp<decltype(pattern_arg(a)), decltype(pattern_arg(lanes)), VectorReduce::Max> {
assert_is_lvalue_if_expr<A>();
return {pattern_arg(a), pattern_arg(lanes)};
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto h_and(A &&a, B lanes) noexcept -> VectorReduceOp<decltype(pattern_arg(a)), decltype(pattern_arg(lanes)), VectorReduce::And> {
assert_is_lvalue_if_expr<A>();
return {pattern_arg(a), pattern_arg(lanes)};
}
template<typename A, typename B>
HALIDE_ALWAYS_INLINE auto h_or(A &&a, B lanes) noexcept -> VectorReduceOp<decltype(pattern_arg(a)), decltype(pattern_arg(lanes)), VectorReduce::Or> {
assert_is_lvalue_if_expr<A>();
return {pattern_arg(a), pattern_arg(lanes)};
}
template<typename A>
struct NegateOp {
struct pattern_tag {};
A a;
constexpr static uint32_t binds = bindings<A>::mask;
constexpr static IRNodeType min_node_type = IRNodeType::Sub;
constexpr static IRNodeType max_node_type = IRNodeType::Sub;
constexpr static bool canonical = A::canonical;
template<uint32_t bound>
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
if (e.node_type != Sub::_node_type) {
return false;
}
const Sub &op = (const Sub &)e;
return (a.template match<bound>(*op.b.get(), state) &&
is_const_zero(op.a));
}
template<uint32_t bound, typename A2>
HALIDE_ALWAYS_INLINE bool match(NegateOp<A2> &&p, MatcherState &state) const noexcept {
return a.template match<bound>(unwrap(p.a), state);
}
HALIDE_ALWAYS_INLINE
Expr make(MatcherState &state, halide_type_t type_hint) const {
Expr ea = a.make(state, type_hint);
Expr z = make_zero(ea.type());
return Sub::make(std::move(z), std::move(ea));
}
constexpr static bool foldable = A::foldable;
template<typename A1 = A>
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept {
a.make_folded_const(val, ty, state);
int dead_bits = 64 - ty.bits;
switch (ty.code) {
case halide_type_int:
if (ty.bits >= 32 && val.u.u64 && (val.u.u64 << (65 - ty.bits)) == 0) {
// Trying to negate the most negative signed int for a no-overflow type.
ty.lanes |= MatcherState::signed_integer_overflow;
} else {
// Negate, drop the high bits, and then sign-extend them back
val.u.i64 = int64_t(uint64_t(-val.u.i64) << dead_bits) >> dead_bits;
}
break;
case halide_type_uint:
val.u.u64 = ((-val.u.u64) << dead_bits) >> dead_bits;
break;
case halide_type_float:
case halide_type_bfloat:
val.u.f64 = -val.u.f64;
break;
default:
// unreachable
;
}
}
};
template<typename A>
std::ostream &operator<<(std::ostream &s, const NegateOp<A> &op) {
s << "-" << op.a;
return s;
}
template<typename A>
HALIDE_ALWAYS_INLINE auto operator-(A &&a) noexcept -> NegateOp<decltype(pattern_arg(a))> {
assert_is_lvalue_if_expr<A>();
return {pattern_arg(a)};
}
template<typename A>
HALIDE_ALWAYS_INLINE auto negate(A &&a) -> decltype(IRMatcher::operator-(a)) {
assert_is_lvalue_if_expr<A>();
return IRMatcher::operator-(a);
}
template<typename A>
struct CastOp {
struct pattern_tag {};
Type t;
A a;
constexpr static uint32_t binds = bindings<A>::mask;
constexpr static IRNodeType min_node_type = IRNodeType::Cast;
constexpr static IRNodeType max_node_type = IRNodeType::Cast;
constexpr static bool canonical = A::canonical;
template<uint32_t bound>
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
if (e.node_type != Cast::_node_type) {
return false;
}
const Cast &op = (const Cast &)e;
return (e.type == t &&
a.template match<bound>(*op.value.get(), state));
}
template<uint32_t bound, typename A2>
HALIDE_ALWAYS_INLINE bool match(const CastOp<A2> &op, MatcherState &state) const noexcept {
return t == op.t && a.template match<bound>(unwrap(op.a), state);
}
HALIDE_ALWAYS_INLINE
Expr make(MatcherState &state, halide_type_t type_hint) const {
return cast(t, a.make(state, {}));
}
constexpr static bool foldable = false;
};
template<typename A>
std::ostream &operator<<(std::ostream &s, const CastOp<A> &op) {
s << "cast(" << op.t << ", " << op.a << ")";
return s;
}
template<typename A>
HALIDE_ALWAYS_INLINE auto cast(halide_type_t t, A &&a) noexcept -> CastOp<decltype(pattern_arg(a))> {
assert_is_lvalue_if_expr<A>();
return {t, pattern_arg(a)};
}
template<typename A>
struct WidenOp {
struct pattern_tag {};
A a;
constexpr static uint32_t binds = bindings<A>::mask;
constexpr static IRNodeType min_node_type = IRNodeType::Cast;
constexpr static IRNodeType max_node_type = IRNodeType::Cast;
constexpr static bool canonical = A::canonical;
template<uint32_t bound>
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
if (e.node_type != Cast::_node_type) {
return false;
}
const Cast &op = (const Cast &)e;
return (e.type == op.value.type().widen() &&
a.template match<bound>(*op.value.get(), state));
}
template<uint32_t bound, typename A2>
HALIDE_ALWAYS_INLINE bool match(const WidenOp<A2> &op, MatcherState &state) const noexcept {
return a.template match<bound>(unwrap(op.a), state);
}
HALIDE_ALWAYS_INLINE
Expr make(MatcherState &state, halide_type_t type_hint) const {
Expr e = a.make(state, {});
Type w = e.type().widen();
return cast(w, std::move(e));
}
constexpr static bool foldable = false;
};
template<typename A>
std::ostream &operator<<(std::ostream &s, const WidenOp<A> &op) {
s << "widen(" << op.a << ")";
return s;
}
template<typename A>
HALIDE_ALWAYS_INLINE auto widen(A &&a) noexcept -> WidenOp<decltype(pattern_arg(a))> {
assert_is_lvalue_if_expr<A>();
return {pattern_arg(a)};
}
template<typename Vec, typename Base, typename Stride, typename Lanes>
struct SliceOp {
struct pattern_tag {};
Vec vec;
Base base;
Stride stride;
Lanes lanes;
static constexpr uint32_t binds = Vec::binds | Base::binds | Stride::binds | Lanes::binds;
constexpr static IRNodeType min_node_type = IRNodeType::Shuffle;
constexpr static IRNodeType max_node_type = IRNodeType::Shuffle;
constexpr static bool canonical = Vec::canonical && Base::canonical && Stride::canonical && Lanes::canonical;
template<uint32_t bound>
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
if (e.node_type != IRNodeType::Shuffle) {
return false;
}
const Shuffle &v = (const Shuffle &)e;
return v.vectors.size() == 1 &&
v.is_slice() &&
vec.template match<bound>(*v.vectors[0].get(), state) &&
base.template match<(bound | bindings<Vec>::mask)>(v.slice_begin(), state) &&
stride.template match<(bound | bindings<Vec>::mask | bindings<Base>::mask)>(v.slice_stride(), state) &&
lanes.template match<(bound | bindings<Vec>::mask | bindings<Base>::mask | bindings<Stride>::mask)>(v.type.lanes(), state);
}
HALIDE_ALWAYS_INLINE
Expr make(MatcherState &state, halide_type_t type_hint) const {
halide_scalar_value_t base_val, stride_val, lanes_val;
halide_type_t ty;
base.make_folded_const(base_val, ty, state);
int b = (int)base_val.u.i64;
stride.make_folded_const(stride_val, ty, state);
int s = (int)stride_val.u.i64;
lanes.make_folded_const(lanes_val, ty, state);
int l = (int)lanes_val.u.i64;
return Shuffle::make_slice(vec.make(state, type_hint), b, s, l);
}
constexpr static bool foldable = false;
HALIDE_ALWAYS_INLINE
SliceOp(Vec v, Base b, Stride s, Lanes l)
: vec(v), base(b), stride(s), lanes(l) {
static_assert(Base::foldable, "Base of slice should consist only of operations that constant-fold");
static_assert(Stride::foldable, "Stride of slice should consist only of operations that constant-fold");
static_assert(Lanes::foldable, "Lanes of slice should consist only of operations that constant-fold");
}
};
template<typename Vec, typename Base, typename Stride, typename Lanes>
std::ostream &operator<<(std::ostream &s, const SliceOp<Vec, Base, Stride, Lanes> &op) {
s << "slice(" << op.vec << ", " << op.base << ", " << op.stride << ", " << op.lanes << ")";
return s;
}
template<typename Vec, typename Base, typename Stride, typename Lanes>
HALIDE_ALWAYS_INLINE auto slice(Vec vec, Base base, Stride stride, Lanes lanes) noexcept
-> SliceOp<decltype(pattern_arg(vec)), decltype(pattern_arg(base)), decltype(pattern_arg(stride)), decltype(pattern_arg(lanes))> {
return {pattern_arg(vec), pattern_arg(base), pattern_arg(stride), pattern_arg(lanes)};
}
template<typename A>
struct Fold {
struct pattern_tag {};
A a;
constexpr static uint32_t binds = bindings<A>::mask;
constexpr static IRNodeType min_node_type = IRNodeType::IntImm;
constexpr static IRNodeType max_node_type = IRNodeType::FloatImm;
constexpr static bool canonical = true;
HALIDE_ALWAYS_INLINE
Expr make(MatcherState &state, halide_type_t type_hint) const noexcept {
halide_scalar_value_t c;
halide_type_t ty = type_hint;
a.make_folded_const(c, ty, state);
// The result of the fold may have an underspecified type
// (e.g. because it's from an int literal). Make the type code
// and bits match the required type, if there is one (we can
// tell from the bits field).
if (type_hint.bits) {
if (((int)ty.code == (int)halide_type_int) &&
((int)type_hint.code == (int)halide_type_float)) {
int64_t x = c.u.i64;
c.u.f64 = (double)x;
}
ty.code = type_hint.code;
ty.bits = type_hint.bits;
}
return make_const_expr(c, ty);
}
constexpr static bool foldable = A::foldable;
template<typename A1 = A>
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept {
a.make_folded_const(val, ty, state);
}
};
template<typename A>
HALIDE_ALWAYS_INLINE auto fold(A &&a) noexcept -> Fold<decltype(pattern_arg(a))> {
assert_is_lvalue_if_expr<A>();
return {pattern_arg(a)};
}
template<typename A>
std::ostream &operator<<(std::ostream &s, const Fold<A> &op) {
s << "fold(" << op.a << ")";
return s;
}
template<typename A>
struct Overflows {
struct pattern_tag {};
A a;
constexpr static uint32_t binds = bindings<A>::mask;
// This rule is a predicate, so it always evaluates to a boolean,
// which has IRNodeType UIntImm
constexpr static IRNodeType min_node_type = IRNodeType::UIntImm;
constexpr static IRNodeType max_node_type = IRNodeType::UIntImm;
constexpr static bool canonical = true;
constexpr static bool foldable = A::foldable;
template<typename A1 = A>
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept {
a.make_folded_const(val, ty, state);
ty.code = halide_type_uint;
ty.bits = 64;
val.u.u64 = (ty.lanes & MatcherState::special_values_mask) != 0;
ty.lanes = 1;
}
};
template<typename A>
HALIDE_ALWAYS_INLINE auto overflows(A &&a) noexcept -> Overflows<decltype(pattern_arg(a))> {
assert_is_lvalue_if_expr<A>();
return {pattern_arg(a)};
}
template<typename A>
std::ostream &operator<<(std::ostream &s, const Overflows<A> &op) {
s << "overflows(" << op.a << ")";
return s;
}
struct Overflow {
struct pattern_tag {};
constexpr static uint32_t binds = 0;
// Overflow is an intrinsic, represented as a Call node
constexpr static IRNodeType min_node_type = IRNodeType::Call;
constexpr static IRNodeType max_node_type = IRNodeType::Call;
constexpr static bool canonical = true;
template<uint32_t bound>
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
if (e.node_type != Call::_node_type) {
return false;
}
const Call &op = (const Call &)e;
return (op.is_intrinsic(Call::signed_integer_overflow));
}
HALIDE_ALWAYS_INLINE
Expr make(MatcherState &state, halide_type_t type_hint) const {
type_hint.lanes |= MatcherState::signed_integer_overflow;
return make_const_special_expr(type_hint);
}
constexpr static bool foldable = true;
HALIDE_ALWAYS_INLINE
void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept {
val.u.u64 = 0;
ty.lanes |= MatcherState::signed_integer_overflow;
}
};
inline std::ostream &operator<<(std::ostream &s, const Overflow &op) {
s << "overflow()";
return s;
}
template<typename A>
struct IsConst {
struct pattern_tag {};
constexpr static uint32_t binds = bindings<A>::mask;
// This rule is a boolean-valued predicate. Bools have type UIntImm.
constexpr static IRNodeType min_node_type = IRNodeType::UIntImm;
constexpr static IRNodeType max_node_type = IRNodeType::UIntImm;
constexpr static bool canonical = true;
A a;
bool check_v;
int64_t v;
constexpr static bool foldable = true;
template<typename A1 = A>
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept {
Expr e = a.make(state, {});
ty.code = halide_type_uint;
ty.bits = 64;
ty.lanes = 1;
if (check_v) {
val.u.u64 = ::Halide::Internal::is_const(e, v) ? 1 : 0;
} else {
val.u.u64 = ::Halide::Internal::is_const(e) ? 1 : 0;
}
}
};
template<typename A>
HALIDE_ALWAYS_INLINE auto is_const(A &&a) noexcept -> IsConst<decltype(pattern_arg(a))> {
assert_is_lvalue_if_expr<A>();
return {pattern_arg(a), false, 0};
}
template<typename A>
HALIDE_ALWAYS_INLINE auto is_const(A &&a, int64_t value) noexcept -> IsConst<decltype(pattern_arg(a))> {
assert_is_lvalue_if_expr<A>();
return {pattern_arg(a), true, value};
}
template<typename A>
std::ostream &operator<<(std::ostream &s, const IsConst<A> &op) {
if (op.check_v) {
s << "is_const(" << op.a << ")";
} else {
s << "is_const(" << op.a << ", " << op.v << ")";
}
return s;
}
template<typename A, typename Prover>
struct CanProve {
struct pattern_tag {};
A a;
Prover *prover; // An existing simplifying mutator
constexpr static uint32_t binds = bindings<A>::mask;
// This rule is a boolean-valued predicate. Bools have type UIntImm.
constexpr static IRNodeType min_node_type = IRNodeType::UIntImm;
constexpr static IRNodeType max_node_type = IRNodeType::UIntImm;
constexpr static bool canonical = true;
constexpr static bool foldable = true;
// Includes a raw call to an inlined make method, so don't inline.
HALIDE_NEVER_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const {
Expr condition = a.make(state, {});
condition = prover->mutate(condition, nullptr);
val.u.u64 = is_const_one(condition);
ty.code = halide_type_uint;
ty.bits = 1;
ty.lanes = condition.type().lanes();
}
};
template<typename A, typename Prover>
HALIDE_ALWAYS_INLINE auto can_prove(A &&a, Prover *p) noexcept -> CanProve<decltype(pattern_arg(a)), Prover> {
assert_is_lvalue_if_expr<A>();
return {pattern_arg(a), p};
}
template<typename A, typename Prover>
std::ostream &operator<<(std::ostream &s, const CanProve<A, Prover> &op) {
s << "can_prove(" << op.a << ")";
return s;
}
template<typename A>
struct IsFloat {
struct pattern_tag {};
A a;
constexpr static uint32_t binds = bindings<A>::mask;
// This rule is a boolean-valued predicate. Bools have type UIntImm.
constexpr static IRNodeType min_node_type = IRNodeType::UIntImm;
constexpr static IRNodeType max_node_type = IRNodeType::UIntImm;
constexpr static bool canonical = true;
constexpr static bool foldable = true;
HALIDE_ALWAYS_INLINE
void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const {
// a is almost certainly a very simple pattern (e.g. a wild), so just inline the make method.
Type t = a.make(state, {}).type();
val.u.u64 = t.is_float();
ty.code = halide_type_uint;
ty.bits = 1;
ty.lanes = t.lanes();
}
};
template<typename A>
HALIDE_ALWAYS_INLINE auto is_float(A &&a) noexcept -> IsFloat<decltype(pattern_arg(a))> {
assert_is_lvalue_if_expr<A>();
return {pattern_arg(a)};
}
template<typename A>
std::ostream &operator<<(std::ostream &s, const IsFloat<A> &op) {
s << "is_float(" << op.a << ")";
return s;
}
template<typename A>
struct IsInt {
struct pattern_tag {};
A a;
uint8_t bits;
uint16_t lanes;
constexpr static uint32_t binds = bindings<A>::mask;
// This rule is a boolean-valued predicate. Bools have type UIntImm.
constexpr static IRNodeType min_node_type = IRNodeType::UIntImm;
constexpr static IRNodeType max_node_type = IRNodeType::UIntImm;
constexpr static bool canonical = true;
constexpr static bool foldable = true;
HALIDE_ALWAYS_INLINE
void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const {
// a is almost certainly a very simple pattern (e.g. a wild), so just inline the make method.
Type t = a.make(state, {}).type();
val.u.u64 = t.is_int() && (bits == 0 || t.bits() == bits) && (lanes == 0 || t.lanes() == lanes);
ty.code = halide_type_uint;
ty.bits = 1;
ty.lanes = t.lanes();
}
};
template<typename A>
HALIDE_ALWAYS_INLINE auto is_int(A &&a, uint8_t bits = 0, uint16_t lanes = 0) noexcept -> IsInt<decltype(pattern_arg(a))> {
assert_is_lvalue_if_expr<A>();
return {pattern_arg(a), bits, lanes};
}
template<typename A>
std::ostream &operator<<(std::ostream &s, const IsInt<A> &op) {
s << "is_int(" << op.a;
if (op.bits > 0) {
s << ", " << op.bits;
}
if (op.lanes > 0) {
s << ", " << op.lanes;
}
s << ")";
return s;
}
template<typename A>
struct IsUInt {
struct pattern_tag {};
A a;
uint8_t bits;
uint16_t lanes;
constexpr static uint32_t binds = bindings<A>::mask;
// This rule is a boolean-valued predicate. Bools have type UIntImm.
constexpr static IRNodeType min_node_type = IRNodeType::UIntImm;
constexpr static IRNodeType max_node_type = IRNodeType::UIntImm;
constexpr static bool canonical = true;
constexpr static bool foldable = true;
HALIDE_ALWAYS_INLINE
void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const {
// a is almost certainly a very simple pattern (e.g. a wild), so just inline the make method.
Type t = a.make(state, {}).type();
val.u.u64 = t.is_uint() && (bits == 0 || t.bits() == bits) && (lanes == 0 || t.lanes() == lanes);
ty.code = halide_type_uint;
ty.bits = 1;
ty.lanes = t.lanes();
}
};
template<typename A>
HALIDE_ALWAYS_INLINE auto is_uint(A &&a, uint8_t bits = 0, uint16_t lanes = 0) noexcept -> IsUInt<decltype(pattern_arg(a))> {
assert_is_lvalue_if_expr<A>();
return {pattern_arg(a), bits, lanes};
}
template<typename A>
std::ostream &operator<<(std::ostream &s, const IsUInt<A> &op) {
s << "is_uint(" << op.a;
if (op.bits > 0) {
s << ", " << op.bits;
}
if (op.lanes > 0) {
s << ", " << op.lanes;
}
s << ")";
return s;
}
template<typename A>
struct IsScalar {
struct pattern_tag {};
A a;
constexpr static uint32_t binds = bindings<A>::mask;
// This rule is a boolean-valued predicate. Bools have type UIntImm.
constexpr static IRNodeType min_node_type = IRNodeType::UIntImm;
constexpr static IRNodeType max_node_type = IRNodeType::UIntImm;
constexpr static bool canonical = true;
constexpr static bool foldable = true;
HALIDE_ALWAYS_INLINE
void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const {
// a is almost certainly a very simple pattern (e.g. a wild), so just inline the make method.
Type t = a.make(state, {}).type();
val.u.u64 = t.is_scalar();
ty.code = halide_type_uint;
ty.bits = 1;
ty.lanes = t.lanes();
}
};
template<typename A>
HALIDE_ALWAYS_INLINE auto is_scalar(A &&a) noexcept -> IsScalar<decltype(pattern_arg(a))> {
assert_is_lvalue_if_expr<A>();
return {pattern_arg(a)};
}
template<typename A>
std::ostream &operator<<(std::ostream &s, const IsScalar<A> &op) {
s << "is_scalar(" << op.a << ")";
return s;
}
template<typename A>
struct IsMaxValue {
struct pattern_tag {};
A a;
constexpr static uint32_t binds = bindings<A>::mask;
// This rule is a boolean-valued predicate. Bools have type UIntImm.
constexpr static IRNodeType min_node_type = IRNodeType::UIntImm;
constexpr static IRNodeType max_node_type = IRNodeType::UIntImm;
constexpr static bool canonical = true;
constexpr static bool foldable = true;
HALIDE_ALWAYS_INLINE
void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const {
// a is almost certainly a very simple pattern (e.g. a wild), so just inline the make method.
a.make_folded_const(val, ty, state);
const uint64_t max_bits = (uint64_t)(-1) >> (64 - ty.bits + (ty.code == halide_type_int));
if (ty.code == halide_type_uint || ty.code == halide_type_int) {
val.u.u64 = (val.u.u64 == max_bits);
} else {
val.u.u64 = 0;
}
ty.code = halide_type_uint;
ty.bits = 1;
}
};
template<typename A>
HALIDE_ALWAYS_INLINE auto is_max_value(A &&a) noexcept -> IsMaxValue<decltype(pattern_arg(a))> {
assert_is_lvalue_if_expr<A>();
return {pattern_arg(a)};
}
template<typename A>
std::ostream &operator<<(std::ostream &s, const IsMaxValue<A> &op) {
s << "is_max_value(" << op.a << ")";
return s;
}
template<typename A>
struct IsMinValue {
struct pattern_tag {};
A a;
constexpr static uint32_t binds = bindings<A>::mask;
// This rule is a boolean-valued predicate. Bools have type UIntImm.
constexpr static IRNodeType min_node_type = IRNodeType::UIntImm;
constexpr static IRNodeType max_node_type = IRNodeType::UIntImm;
constexpr static bool canonical = true;
constexpr static bool foldable = true;
HALIDE_ALWAYS_INLINE
void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const {
// a is almost certainly a very simple pattern (e.g. a wild), so just inline the make method.
a.make_folded_const(val, ty, state);
if (ty.code == halide_type_int) {
const uint64_t min_bits = (uint64_t)(-1) << (ty.bits - 1);
val.u.u64 = (val.u.u64 == min_bits);
} else if (ty.code == halide_type_uint) {
val.u.u64 = (val.u.u64 == 0);
} else {
val.u.u64 = 0;
}
ty.code = halide_type_uint;
ty.bits = 1;
}
};
template<typename A>
HALIDE_ALWAYS_INLINE auto is_min_value(A &&a) noexcept -> IsMinValue<decltype(pattern_arg(a))> {
assert_is_lvalue_if_expr<A>();
return {pattern_arg(a)};
}
template<typename A>
std::ostream &operator<<(std::ostream &s, const IsMinValue<A> &op) {
s << "is_min_value(" << op.a << ")";
return s;
}
template<typename A>
struct LanesOf {
struct pattern_tag {};
A a;
constexpr static uint32_t binds = bindings<A>::mask;
// This rule is a boolean-valued predicate. Bools have type UIntImm.
constexpr static IRNodeType min_node_type = IRNodeType::UIntImm;
constexpr static IRNodeType max_node_type = IRNodeType::UIntImm;
constexpr static bool canonical = true;
constexpr static bool foldable = true;
HALIDE_ALWAYS_INLINE
void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const {
// a is almost certainly a very simple pattern (e.g. a wild), so just inline the make method.
Type t = a.make(state, {}).type();
val.u.u64 = t.lanes();
ty.code = halide_type_uint;
ty.bits = 32;
ty.lanes = 1;
}
};
template<typename A>
HALIDE_ALWAYS_INLINE auto lanes_of(A &&a) noexcept -> LanesOf<decltype(pattern_arg(a))> {
assert_is_lvalue_if_expr<A>();
return {pattern_arg(a)};
}
template<typename A>
std::ostream &operator<<(std::ostream &s, const LanesOf<A> &op) {
s << "lanes_of(" << op.a << ")";
return s;
}
// Verify properties of each rewrite rule. Currently just fuzz tests them.
template<typename Before,
typename After,
typename Predicate,
typename = typename std::enable_if<std::decay<Before>::type::foldable &&
std::decay<After>::type::foldable>::type>
HALIDE_NEVER_INLINE void fuzz_test_rule(Before &&before, After &&after, Predicate &&pred,
halide_type_t wildcard_type, halide_type_t output_type) noexcept {
// We only validate the rules in the scalar case
wildcard_type.lanes = output_type.lanes = 1;
// Track which types this rule has been tested for before
static std::set<uint32_t> tested;
if (!tested.insert(reinterpret_bits<uint32_t>(wildcard_type)).second) {
return;
}
// Print it in a form where it can be piped into a python/z3 validator
debug(0) << "validate('" << before << "', '" << after << "', '" << pred << "', " << Type(wildcard_type) << ", " << Type(output_type) << ")\n";
// Substitute some random constants into the before and after
// expressions and see if the rule holds true. This should catch
// silly errors, but not necessarily corner cases.
static std::mt19937_64 rng(0);
MatcherState state;
Expr exprs[max_wild];
for (int trials = 0; trials < 100; trials++) {
// We want to test small constants more frequently than
// large ones, otherwise we'll just get coverage of
// overflow rules.
int shift = (int)(rng() & (wildcard_type.bits - 1));
for (int i = 0; i < max_wild; i++) {
// Bind all the exprs and constants
switch (wildcard_type.code) {
case halide_type_uint: {
// Normalize to the type's range by adding zero
uint64_t val = constant_fold_bin_op<Add>(wildcard_type, (uint64_t)rng() >> shift, 0);
state.set_bound_const(i, val, wildcard_type);
val = constant_fold_bin_op<Add>(wildcard_type, (uint64_t)rng() >> shift, 0);
exprs[i] = make_const(wildcard_type, val);
state.set_binding(i, *exprs[i].get());
} break;
case halide_type_int: {
int64_t val = constant_fold_bin_op<Add>(wildcard_type, (int64_t)rng() >> shift, 0);
state.set_bound_const(i, val, wildcard_type);
val = constant_fold_bin_op<Add>(wildcard_type, (int64_t)rng() >> shift, 0);
exprs[i] = make_const(wildcard_type, val);
} break;
case halide_type_float:
case halide_type_bfloat: {
// Use a very narrow range of precise floats, so
// that none of the rules a human is likely to
// write have instabilities.
double val = ((int64_t)(rng() & 15) - 8) / 2.0;
state.set_bound_const(i, val, wildcard_type);
val = ((int64_t)(rng() & 15) - 8) / 2.0;
exprs[i] = make_const(wildcard_type, val);
} break;
default:
return; // Don't care about handles
}
state.set_binding(i, *exprs[i].get());
}
halide_scalar_value_t val_pred, val_before, val_after;
halide_type_t type = output_type;
if (!evaluate_predicate(pred, state)) {
continue;
}
before.make_folded_const(val_before, type, state);
uint16_t lanes = type.lanes;
after.make_folded_const(val_after, type, state);
lanes |= type.lanes;
if (lanes & MatcherState::special_values_mask) {
continue;
}
bool ok = true;
switch (output_type.code) {
case halide_type_uint:
// Compare normalized representations
ok &= (constant_fold_bin_op<Add>(output_type, val_before.u.u64, 0) ==
constant_fold_bin_op<Add>(output_type, val_after.u.u64, 0));
break;
case halide_type_int:
ok &= (constant_fold_bin_op<Add>(output_type, val_before.u.i64, 0) ==
constant_fold_bin_op<Add>(output_type, val_after.u.i64, 0));
break;
case halide_type_float:
case halide_type_bfloat: {
double error = std::abs(val_before.u.f64 - val_after.u.f64);
// We accept an equal bit pattern (e.g. inf vs inf),
// a small floating point difference, or turning a nan into not-a-nan.
ok &= (error < 0.01 ||
val_before.u.u64 == val_after.u.u64 ||
std::isnan(val_before.u.f64));
break;
}
default:
return;
}
if (!ok) {
debug(0) << "Fails with values:\n";
for (int i = 0; i < max_wild; i++) {
halide_scalar_value_t val;
state.get_bound_const(i, val, wildcard_type);
debug(0) << " c" << i << ": " << make_const_expr(val, wildcard_type) << "\n";
}
for (int i = 0; i < max_wild; i++) {
debug(0) << " _" << i << ": " << Expr(state.get_binding(i)) << "\n";
}
debug(0) << " Before: " << make_const_expr(val_before, output_type) << "\n";
debug(0) << " After: " << make_const_expr(val_after, output_type) << "\n";
debug(0) << val_before.u.u64 << " " << val_after.u.u64 << "\n";
internal_error;
}
}
}
template<typename Before,
typename After,
typename Predicate,
typename = typename std::enable_if<!(std::decay<Before>::type::foldable &&
std::decay<After>::type::foldable)>::type>
HALIDE_ALWAYS_INLINE void fuzz_test_rule(Before &&before, After &&after, Predicate &&pred,
halide_type_t, halide_type_t, int dummy = 0) noexcept {
// We can't verify rewrite rules that can't be constant-folded.
}
HALIDE_ALWAYS_INLINE
bool evaluate_predicate(bool x, MatcherState &) noexcept {
return x;
}
template<typename Pattern,
typename = typename enable_if_pattern<Pattern>::type>
HALIDE_ALWAYS_INLINE bool evaluate_predicate(Pattern p, MatcherState &state) {
halide_scalar_value_t c;
halide_type_t ty = halide_type_of<bool>();
p.make_folded_const(c, ty, state);
// Overflow counts as a failed predicate
return (c.u.u64 != 0) && ((ty.lanes & MatcherState::special_values_mask) == 0);
}
// #defines for testing
// Print all successful or failed matches
#define HALIDE_DEBUG_MATCHED_RULES 0
#define HALIDE_DEBUG_UNMATCHED_RULES 0
// Set to true if you want to fuzz test every rewrite passed to
// operator() to ensure the input and the output have the same value
// for lots of random values of the wildcards. Run
// correctness_simplify with this on.
#define HALIDE_FUZZ_TEST_RULES 0
template<typename Instance>
struct Rewriter {
Instance instance;
Expr result;
MatcherState state;
halide_type_t output_type, wildcard_type;
bool validate;
HALIDE_ALWAYS_INLINE
Rewriter(Instance instance, halide_type_t ot, halide_type_t wt)
: instance(std::move(instance)), output_type(ot), wildcard_type(wt) {
}
template<typename After>
HALIDE_NEVER_INLINE void build_replacement(After after) {
#if HALIDE_DEBUG_MATCHED_RULES
debug(0) << instance << " -> " << after << "\n";
#endif
result = after.make(state, output_type);
}
template<typename Before,
typename After,
typename = typename enable_if_pattern<Before>::type,
typename = typename enable_if_pattern<After>::type>
HALIDE_ALWAYS_INLINE bool operator()(Before before, After after) {
static_assert((Before::binds & After::binds) == After::binds, "Rule result uses unbound values");
static_assert(Before::canonical, "LHS of rewrite rule should be in canonical form");
static_assert(After::canonical, "RHS of rewrite rule should be in canonical form");
#if HALIDE_FUZZ_TEST_RULES
fuzz_test_rule(before, after, true, wildcard_type, output_type);
#endif
if (before.template match<0>(unwrap(instance), state)) {
build_replacement(after);
#if HALIDE_DEBUG_MATCHED_RULES
debug(0) << instance << " -> " << result << " via " << before << " -> " << after << "\n";
#endif
return true;
} else {
#if HALIDE_DEBUG_UNMATCHED_RULES
debug(0) << instance << " does not match " << before << "\n";
#endif
return false;
}
}
template<typename Before,
typename = typename enable_if_pattern<Before>::type>
HALIDE_ALWAYS_INLINE bool operator()(Before before, const Expr &after) noexcept {
static_assert(Before::canonical, "LHS of rewrite rule should be in canonical form");
if (before.template match<0>(unwrap(instance), state)) {
result = after;
#if HALIDE_DEBUG_MATCHED_RULES
debug(0) << instance << " -> " << result << " via " << before << " -> " << after << "\n";
#endif
return true;
} else {
#if HALIDE_DEBUG_UNMATCHED_RULES
debug(0) << instance << " does not match " << before << "\n";
#endif
return false;
}
}
template<typename Before,
typename = typename enable_if_pattern<Before>::type>
HALIDE_ALWAYS_INLINE bool operator()(Before before, int64_t after) noexcept {
static_assert(Before::canonical, "LHS of rewrite rule should be in canonical form");
#if HALIDE_FUZZ_TEST_RULES
fuzz_test_rule(before, IntLiteral(after), true, wildcard_type, output_type);
#endif
if (before.template match<0>(unwrap(instance), state)) {
result = make_const(output_type, after);
#if HALIDE_DEBUG_MATCHED_RULES
debug(0) << instance << " -> " << result << " via " << before << " -> " << after << "\n";
#endif
return true;
} else {
#if HALIDE_DEBUG_UNMATCHED_RULES
debug(0) << instance << " does not match " << before << "\n";
#endif
return false;
}
}
template<typename Before,
typename After,
typename Predicate,
typename = typename enable_if_pattern<Before>::type,
typename = typename enable_if_pattern<After>::type,
typename = typename enable_if_pattern<Predicate>::type>
HALIDE_ALWAYS_INLINE bool operator()(Before before, After after, Predicate pred) {
static_assert(Predicate::foldable, "Predicates must consist only of operations that can constant-fold");
static_assert((Before::binds & After::binds) == After::binds, "Rule result uses unbound values");
static_assert((Before::binds & Predicate::binds) == Predicate::binds, "Rule predicate uses unbound values");
static_assert(Before::canonical, "LHS of rewrite rule should be in canonical form");
static_assert(After::canonical, "RHS of rewrite rule should be in canonical form");
#if HALIDE_FUZZ_TEST_RULES
fuzz_test_rule(before, after, pred, wildcard_type, output_type);
#endif
if (before.template match<0>(unwrap(instance), state) &&
evaluate_predicate(pred, state)) {
build_replacement(after);
#if HALIDE_DEBUG_MATCHED_RULES
debug(0) << instance << " -> " << result << " via " << before << " -> " << after << " when " << pred << "\n";
#endif
return true;
} else {
#if HALIDE_DEBUG_UNMATCHED_RULES
debug(0) << instance << " does not match " << before << "\n";
#endif
return false;
}
}
template<typename Before,
typename Predicate,
typename = typename enable_if_pattern<Before>::type,
typename = typename enable_if_pattern<Predicate>::type>
HALIDE_ALWAYS_INLINE bool operator()(Before before, const Expr &after, Predicate pred) {
static_assert(Predicate::foldable, "Predicates must consist only of operations that can constant-fold");
static_assert(Before::canonical, "LHS of rewrite rule should be in canonical form");
if (before.template match<0>(unwrap(instance), state) &&
evaluate_predicate(pred, state)) {
result = after;
#if HALIDE_DEBUG_MATCHED_RULES
debug(0) << instance << " -> " << result << " via " << before << " -> " << after << " when " << pred << "\n";
#endif
return true;
} else {
#if HALIDE_DEBUG_UNMATCHED_RULES
debug(0) << instance << " does not match " << before << "\n";
#endif
return false;
}
}
template<typename Before,
typename Predicate,
typename = typename enable_if_pattern<Before>::type,
typename = typename enable_if_pattern<Predicate>::type>
HALIDE_ALWAYS_INLINE bool operator()(Before before, int64_t after, Predicate pred) {
static_assert(Predicate::foldable, "Predicates must consist only of operations that can constant-fold");
static_assert(Before::canonical, "LHS of rewrite rule should be in canonical form");
#if HALIDE_FUZZ_TEST_RULES
fuzz_test_rule(before, IntLiteral(after), pred, wildcard_type, output_type);
#endif
if (before.template match<0>(unwrap(instance), state) &&
evaluate_predicate(pred, state)) {
result = make_const(output_type, after);
#if HALIDE_DEBUG_MATCHED_RULES
debug(0) << instance << " -> " << result << " via " << before << " -> " << after << " when " << pred << "\n";
#endif
return true;
} else {
#if HALIDE_DEBUG_UNMATCHED_RULES
debug(0) << instance << " does not match " << before << "\n";
#endif
return false;
}
}
};
/** Construct a rewriter for the given instance, which may be a pattern
* with concrete expressions as leaves, or just an expression. The
* second optional argument (wildcard_type) is a hint as to what the
* type of the wildcards is likely to be. If omitted it uses the same
* type as the expression itself. They are not required to be this
* type, but the rule will only be tested for wildcards of that type
* when testing is enabled.
*
* The rewriter can be used to check to see if the instance is one of
* some number of patterns and if so rewrite it into another form,
* using its operator() method. See Simplify.cpp for a bunch of
* example usage.
*
* Important: Any Exprs in patterns are captured by reference, not by
* value, so ensure they outlive the rewriter.
*/
// @{
template<typename Instance,
typename = typename enable_if_pattern<Instance>::type>
HALIDE_ALWAYS_INLINE auto rewriter(Instance instance, halide_type_t output_type, halide_type_t wildcard_type) noexcept -> Rewriter<decltype(pattern_arg(instance))> {
return {pattern_arg(instance), output_type, wildcard_type};
}
template<typename Instance,
typename = typename enable_if_pattern<Instance>::type>
HALIDE_ALWAYS_INLINE auto rewriter(Instance instance, halide_type_t output_type) noexcept -> Rewriter<decltype(pattern_arg(instance))> {
return {pattern_arg(instance), output_type, output_type};
}
HALIDE_ALWAYS_INLINE
auto rewriter(const Expr &e, halide_type_t wildcard_type) noexcept -> Rewriter<decltype(pattern_arg(e))> {
return {pattern_arg(e), e.type(), wildcard_type};
}
HALIDE_ALWAYS_INLINE
auto rewriter(const Expr &e) noexcept -> Rewriter<decltype(pattern_arg(e))> {
return {pattern_arg(e), e.type(), e.type()};
}
// @}
} // namespace IRMatcher
} // namespace Internal
} // namespace Halide
#endif
debug
#define debug(n)
For optional debugging during codegen, use the debug macro as follows:
Definition Debug.h:52
internal_error
#define internal_error
Definition Error.h:215
halide_type_float
@ halide_type_float
IEEE floating point numbers.
Definition HalideRuntime.h:483
halide_type_bfloat
@ halide_type_bfloat
floating point numbers in the bfloat format
Definition HalideRuntime.h:485
halide_type_int
@ halide_type_int
signed integers
Definition HalideRuntime.h:481
halide_type_uint
@ halide_type_uint
unsigned integers
Definition HalideRuntime.h:482
HALIDE_NEVER_INLINE
#define HALIDE_NEVER_INLINE
Definition HalideRuntime.h:50
HALIDE_ALWAYS_INLINE
#define HALIDE_ALWAYS_INLINE
Definition HalideRuntime.h:49
IR.h
Subtypes for Halide expressions (Halide::Expr) and statements (Halide::Internal::Stmt).
IREquality.h
Methods to test Exprs and Stmts for equality of value.
IROperator.h
Defines various operator overloads and utility functions that make it more pleasant to work with Hali...
Halide::Internal::IRMatcher
An alternative template-metaprogramming approach to expression matching.
Definition IRMatch.h:72
Halide::Internal::IRMatcher::rewriter
HALIDE_ALWAYS_INLINE auto rewriter(Instance instance, halide_type_t output_type, halide_type_t wildcard_type) noexcept -> Rewriter< decltype(pattern_arg(instance))>
Construct a rewriter for the given instance, which may be a pattern with concrete expressions as leav...
Definition IRMatch.h:3046
Halide::Internal::IRMatcher::pattern_arg
HALIDE_ALWAYS_INLINE T pattern_arg(T t)
Definition IRMatch.h:567
Halide::Internal::IRMatcher::rounding_halving_add
auto rounding_halving_add(A &&a, B &&b) noexcept -> Intrin< Call::rounding_halving_add, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:1563
Halide::Internal::IRMatcher::or_op
HALIDE_ALWAYS_INLINE auto or_op(A &&a, B &&b) -> decltype(IRMatcher::operator||(a, b))
Definition IRMatch.h:1259
Halide::Internal::IRMatcher::operator!
HALIDE_ALWAYS_INLINE auto operator!(A &&a) noexcept -> NotOp< decltype(pattern_arg(a))>
Definition IRMatch.h:1652
Halide::Internal::IRMatcher::shift_right
auto shift_right(A &&a, B &&b) noexcept -> Intrin< Call::shift_right, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:1571
Halide::Internal::IRMatcher::min
HALIDE_ALWAYS_INLINE auto min(A &&a, B &&b) noexcept -> BinOp< Min, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:1060
Halide::Internal::IRMatcher::widening_add
auto widening_add(A &&a, B &&b) noexcept -> Intrin< Call::widening_add, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:1529
Halide::Internal::IRMatcher::is_int
HALIDE_ALWAYS_INLINE auto is_int(A &&a, uint8_t bits=0, uint16_t lanes=0) noexcept -> IsInt< decltype(pattern_arg(a))>
Definition IRMatch.h:2477
Halide::Internal::IRMatcher::evaluate_predicate
HALIDE_ALWAYS_INLINE bool evaluate_predicate(bool x, MatcherState &) noexcept
Definition IRMatch.h:2836
Halide::Internal::IRMatcher::constant_fold_bin_op< Div >
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Div >(halide_type_t &t, int64_t a, int64_t b) noexcept
Definition IRMatch.h:1016
Halide::Internal::IRMatcher::abs
auto abs(A &&a) noexcept -> Intrin< Call::abs, decltype(pattern_arg(a))>
Definition IRMatch.h:1592
Halide::Internal::IRMatcher::ne
HALIDE_ALWAYS_INLINE auto ne(A &&a, B &&b) -> decltype(IRMatcher::operator!=(a, b))
Definition IRMatch.h:1234
Halide::Internal::IRMatcher::is_uint
HALIDE_ALWAYS_INLINE auto is_uint(A &&a, uint8_t bits=0, uint16_t lanes=0) noexcept -> IsUInt< decltype(pattern_arg(a))>
Definition IRMatch.h:2523
Halide::Internal::IRMatcher::negate
HALIDE_ALWAYS_INLINE auto negate(A &&a) -> decltype(IRMatcher::operator-(a))
Definition IRMatch.h:2029
Halide::Internal::IRMatcher::constant_fold_cmp_op
uint64_t constant_fold_cmp_op(int64_t, int64_t) noexcept
Halide::Internal::IRMatcher::operator<=
HALIDE_ALWAYS_INLINE auto operator<=(A &&a, B &&b) noexcept -> CmpOp< LE, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:1154
Halide::Internal::IRMatcher::operator+
HALIDE_ALWAYS_INLINE auto operator+(A &&a, B &&b) noexcept -> BinOp< Add, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:905
Halide::Internal::IRMatcher::is_max_value
HALIDE_ALWAYS_INLINE auto is_max_value(A &&a) noexcept -> IsMaxValue< decltype(pattern_arg(a))>
Definition IRMatch.h:2608
Halide::Internal::IRMatcher::operator<<
std::ostream & operator<<(std::ostream &s, const SpecificExpr &e)
Definition IRMatch.h:217
Halide::Internal::IRMatcher::and_op
HALIDE_ALWAYS_INLINE auto and_op(A &&a, B &&b) -> decltype(IRMatcher::operator&&(a, b))
Definition IRMatch.h:1285
Halide::Internal::IRMatcher::h_and
HALIDE_ALWAYS_INLINE auto h_and(A &&a, B lanes) noexcept -> VectorReduceOp< decltype(pattern_arg(a)), decltype(pattern_arg(lanes)), VectorReduce::And >
Definition IRMatch.h:1941
Halide::Internal::IRMatcher::gt
HALIDE_ALWAYS_INLINE auto gt(A &&a, B &&b) -> decltype(IRMatcher::operator>(a, b))
Definition IRMatch.h:1134
Halide::Internal::IRMatcher::is_const
HALIDE_ALWAYS_INLINE auto is_const(A &&a) noexcept -> IsConst< decltype(pattern_arg(a))>
Definition IRMatch.h:2353
Halide::Internal::IRMatcher::constant_fold_cmp_op< LE >
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op< LE >(int64_t a, int64_t b) noexcept
Definition IRMatch.h:1164
Halide::Internal::IRMatcher::operator*
HALIDE_ALWAYS_INLINE auto operator*(A &&a, B &&b) noexcept -> BinOp< Mul, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:971
Halide::Internal::IRMatcher::add
HALIDE_ALWAYS_INLINE auto add(A &&a, B &&b) -> decltype(IRMatcher::operator+(a, b))
Definition IRMatch.h:912
Halide::Internal::IRMatcher::widen_right_add
auto widen_right_add(A &&a, B &&b) noexcept -> Intrin< Call::widen_right_add, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:1516
Halide::Internal::IRMatcher::div
HALIDE_ALWAYS_INLINE auto div(A &&a, B &&b) -> decltype(IRMatcher::operator/(a, b))
Definition IRMatch.h:1011
Halide::Internal::IRMatcher::widen_right_mul
auto widen_right_mul(A &&a, B &&b) noexcept -> Intrin< Call::widen_right_mul, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:1520
Halide::Internal::IRMatcher::mul
HALIDE_ALWAYS_INLINE auto mul(A &&a, B &&b) -> decltype(IRMatcher::operator*(a, b))
Definition IRMatch.h:978
Halide::Internal::IRMatcher::max
HALIDE_ALWAYS_INLINE auto max(A &&a, B &&b) noexcept -> BinOp< Max, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:1082
Halide::Internal::IRMatcher::absd
auto absd(A &&a, B &&b) noexcept -> Intrin< Call::absd, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:1597
Halide::Internal::IRMatcher::slice
HALIDE_ALWAYS_INLINE auto slice(Vec vec, Base base, Stride stride, Lanes lanes) noexcept -> SliceOp< decltype(pattern_arg(vec)), decltype(pattern_arg(base)), decltype(pattern_arg(stride)), decltype(pattern_arg(lanes))>
Definition IRMatch.h:2186
Halide::Internal::IRMatcher::ramp
HALIDE_ALWAYS_INLINE auto ramp(A &&a, B &&b, C &&c) noexcept -> RampOp< decltype(pattern_arg(a)), decltype(pattern_arg(b)), decltype(pattern_arg(c))>
Definition IRMatch.h:1865
Halide::Internal::IRMatcher::operator/
HALIDE_ALWAYS_INLINE auto operator/(A &&a, B &&b) noexcept -> BinOp< Div, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:1004
Halide::Internal::IRMatcher::widen
HALIDE_ALWAYS_INLINE auto widen(A &&a) noexcept -> WidenOp< decltype(pattern_arg(a))>
Definition IRMatch.h:2122
Halide::Internal::IRMatcher::constant_fold_bin_op< Mod >
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Mod >(halide_type_t &t, int64_t a, int64_t b) noexcept
Definition IRMatch.h:1045
Halide::Internal::IRMatcher::constant_fold_bin_op< And >
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< And >(halide_type_t &t, int64_t a, int64_t b) noexcept
Definition IRMatch.h:1290
Halide::Internal::IRMatcher::unwrap
HALIDE_ALWAYS_INLINE int64_t unwrap(IntLiteral t)
Definition IRMatch.h:559
Halide::Internal::IRMatcher::widening_mul
auto widening_mul(A &&a, B &&b) noexcept -> Intrin< Call::widening_mul, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:1537
Halide::Internal::IRMatcher::operator>
HALIDE_ALWAYS_INLINE auto operator>(A &&a, B &&b) noexcept -> CmpOp< GT, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:1129
Halide::Internal::IRMatcher::cast
HALIDE_ALWAYS_INLINE auto cast(halide_type_t t, A &&a) noexcept -> CastOp< decltype(pattern_arg(a))>
Definition IRMatch.h:2075
Halide::Internal::IRMatcher::overflows
HALIDE_ALWAYS_INLINE auto overflows(A &&a) noexcept -> Overflows< decltype(pattern_arg(a))>
Definition IRMatch.h:2271
Halide::Internal::IRMatcher::saturating_cast
auto saturating_cast(const Type &t, A &&a) noexcept -> Intrin< Call::saturating_cast, decltype(pattern_arg(a))>
Definition IRMatch.h:1549
Halide::Internal::IRMatcher::assert_is_lvalue_if_expr
HALIDE_ALWAYS_INLINE void assert_is_lvalue_if_expr()
Definition IRMatch.h:576
Halide::Internal::IRMatcher::operator%
HALIDE_ALWAYS_INLINE auto operator%(A &&a, B &&b) noexcept -> BinOp< Mod, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:1031
Halide::Internal::IRMatcher::constant_fold_bin_op< Sub >
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Sub >(halide_type_t &t, int64_t a, int64_t b) noexcept
Definition IRMatch.h:952
Halide::Internal::IRMatcher::rounding_shift_left
auto rounding_shift_left(A &&a, B &&b) noexcept -> Intrin< Call::rounding_shift_left, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:1575
Halide::Internal::IRMatcher::is_scalar
HALIDE_ALWAYS_INLINE auto is_scalar(A &&a) noexcept -> IsScalar< decltype(pattern_arg(a))>
Definition IRMatch.h:2567
Halide::Internal::IRMatcher::fold
HALIDE_ALWAYS_INLINE auto fold(A &&a) noexcept -> Fold< decltype(pattern_arg(a))>
Definition IRMatch.h:2234
Halide::Internal::IRMatcher::not_op
HALIDE_ALWAYS_INLINE auto not_op(A &&a) -> decltype(IRMatcher::operator!(a))
Definition IRMatch.h:1658
Halide::Internal::IRMatcher::likely
auto likely(A &&a) noexcept -> Intrin< Call::likely, decltype(pattern_arg(a))>
Definition IRMatch.h:1602
Halide::Internal::IRMatcher::constant_fold_bin_op< Max >
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Max >(halide_type_t &t, int64_t a, int64_t b) noexcept
Definition IRMatch.h:1089
Halide::Internal::IRMatcher::and_reduce
constexpr bool and_reduce()
Definition IRMatch.h:1314
Halide::Internal::IRMatcher::operator||
HALIDE_ALWAYS_INLINE auto operator||(A &&a, B &&b) noexcept -> BinOp< Or, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:1254
Halide::Internal::IRMatcher::max_wild
constexpr int max_wild
Definition IRMatch.h:74
Halide::Internal::IRMatcher::operator!=
HALIDE_ALWAYS_INLINE auto operator!=(A &&a, B &&b) noexcept -> CmpOp< NE, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:1229
Halide::Internal::IRMatcher::halving_add
auto halving_add(A &&a, B &&b) noexcept -> Intrin< Call::halving_add, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:1555
Halide::Internal::IRMatcher::mul_shift_right
auto mul_shift_right(A &&a, B &&b, C &&c) noexcept -> Intrin< Call::mul_shift_right, decltype(pattern_arg(a)), decltype(pattern_arg(b)), decltype(pattern_arg(c))>
Definition IRMatch.h:1583
Halide::Internal::IRMatcher::is_float
HALIDE_ALWAYS_INLINE auto is_float(A &&a) noexcept -> IsFloat< decltype(pattern_arg(a))>
Definition IRMatch.h:2438
Halide::Internal::IRMatcher::widening_sub
auto widening_sub(A &&a, B &&b) noexcept -> Intrin< Call::widening_sub, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:1533
Halide::Internal::IRMatcher::operator>=
HALIDE_ALWAYS_INLINE auto operator>=(A &&a, B &&b) noexcept -> CmpOp< GE, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:1179
Halide::Internal::IRMatcher::operator<
HALIDE_ALWAYS_INLINE auto operator<(A &&a, B &&b) noexcept -> CmpOp< LT, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:1104
Halide::Internal::IRMatcher::operator&&
HALIDE_ALWAYS_INLINE auto operator&&(A &&a, B &&b) noexcept -> BinOp< And, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:1280
Halide::Internal::IRMatcher::h_or
HALIDE_ALWAYS_INLINE auto h_or(A &&a, B lanes) noexcept -> VectorReduceOp< decltype(pattern_arg(a)), decltype(pattern_arg(lanes)), VectorReduce::Or >
Definition IRMatch.h:1947
Halide::Internal::IRMatcher::commutative
constexpr bool commutative(IRNodeType t)
Definition IRMatch.h:615
Halide::Internal::IRMatcher::sub
HALIDE_ALWAYS_INLINE auto sub(A &&a, B &&b) -> decltype(IRMatcher::operator-(a, b))
Definition IRMatch.h:945
Halide::Internal::IRMatcher::likely_if_innermost
auto likely_if_innermost(A &&a) noexcept -> Intrin< Call::likely_if_innermost, decltype(pattern_arg(a))>
Definition IRMatch.h:1607
Halide::Internal::IRMatcher::h_max
HALIDE_ALWAYS_INLINE auto h_max(A &&a, B lanes) noexcept -> VectorReduceOp< decltype(pattern_arg(a)), decltype(pattern_arg(lanes)), VectorReduce::Max >
Definition IRMatch.h:1935
Halide::Internal::IRMatcher::broadcast
HALIDE_ALWAYS_INLINE auto broadcast(A &&a, B lanes) noexcept -> BroadcastOp< decltype(pattern_arg(a)), decltype(pattern_arg(lanes))>
Definition IRMatch.h:1801
Halide::Internal::IRMatcher::select
HALIDE_ALWAYS_INLINE auto select(C &&c, T &&t, F &&f) noexcept -> SelectOp< decltype(pattern_arg(c)), decltype(pattern_arg(t)), decltype(pattern_arg(f))>
Definition IRMatch.h:1728
Halide::Internal::IRMatcher::saturating_sub
auto saturating_sub(A &&a, B &&b) noexcept -> Intrin< Call::saturating_sub, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:1545
Halide::Internal::IRMatcher::is_min_value
HALIDE_ALWAYS_INLINE auto is_min_value(A &&a) noexcept -> IsMinValue< decltype(pattern_arg(a))>
Definition IRMatch.h:2651
Halide::Internal::IRMatcher::constant_fold_bin_op< Min >
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Min >(halide_type_t &t, int64_t a, int64_t b) noexcept
Definition IRMatch.h:1067
Halide::Internal::IRMatcher::fuzz_test_rule
HALIDE_NEVER_INLINE void fuzz_test_rule(Before &&before, After &&after, Predicate &&pred, halide_type_t wildcard_type, halide_type_t output_type) noexcept
Definition IRMatch.h:2705
Halide::Internal::IRMatcher::constant_fold_cmp_op< GT >
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op< GT >(int64_t a, int64_t b) noexcept
Definition IRMatch.h:1139
Halide::Internal::IRMatcher::constant_fold_bin_op< Mul >
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Mul >(halide_type_t &t, int64_t a, int64_t b) noexcept
Definition IRMatch.h:985
Halide::Internal::IRMatcher::constant_fold_cmp_op< GE >
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op< GE >(int64_t a, int64_t b) noexcept
Definition IRMatch.h:1189
Halide::Internal::IRMatcher::operator-
HALIDE_ALWAYS_INLINE auto operator-(A &&a, B &&b) noexcept -> BinOp< Sub, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:938
Halide::Internal::IRMatcher::rounding_mul_shift_right
auto rounding_mul_shift_right(A &&a, B &&b, C &&c) noexcept -> Intrin< Call::rounding_mul_shift_right, decltype(pattern_arg(a)), decltype(pattern_arg(b)), decltype(pattern_arg(c))>
Definition IRMatch.h:1587
Halide::Internal::IRMatcher::saturating_add
auto saturating_add(A &&a, B &&b) noexcept -> Intrin< Call::saturating_add, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:1541
Halide::Internal::IRMatcher::le
HALIDE_ALWAYS_INLINE auto le(A &&a, B &&b) -> decltype(IRMatcher::operator<=(a, b))
Definition IRMatch.h:1159
Halide::Internal::IRMatcher::lt
HALIDE_ALWAYS_INLINE auto lt(A &&a, B &&b) -> decltype(IRMatcher::operator<(a, b))
Definition IRMatch.h:1109
Halide::Internal::IRMatcher::shift_left
auto shift_left(A &&a, B &&b) noexcept -> Intrin< Call::shift_left, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:1567
Halide::Internal::IRMatcher::lanes_of
HALIDE_ALWAYS_INLINE auto lanes_of(A &&a) noexcept -> LanesOf< decltype(pattern_arg(a))>
Definition IRMatch.h:2688
Halide::Internal::IRMatcher::rounding_shift_right
auto rounding_shift_right(A &&a, B &&b) noexcept -> Intrin< Call::rounding_shift_right, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:1579
Halide::Internal::IRMatcher::constant_fold_cmp_op< LT >
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op< LT >(int64_t a, int64_t b) noexcept
Definition IRMatch.h:1114
Halide::Internal::IRMatcher::h_min
HALIDE_ALWAYS_INLINE auto h_min(A &&a, B lanes) noexcept -> VectorReduceOp< decltype(pattern_arg(a)), decltype(pattern_arg(lanes)), VectorReduce::Min >
Definition IRMatch.h:1929
Halide::Internal::IRMatcher::h_add
HALIDE_ALWAYS_INLINE auto h_add(A &&a, B lanes) noexcept -> VectorReduceOp< decltype(pattern_arg(a)), decltype(pattern_arg(lanes)), VectorReduce::Add >
Definition IRMatch.h:1923
Halide::Internal::IRMatcher::constant_fold_bin_op< Or >
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Or >(halide_type_t &t, int64_t a, int64_t b) noexcept
Definition IRMatch.h:1264
Halide::Internal::IRMatcher::make_const_expr
HALIDE_ALWAYS_INLINE Expr make_const_expr(halide_scalar_value_t val, halide_type_t ty)
Definition IRMatch.h:160
Halide::Internal::IRMatcher::bitwise_or_reduce
constexpr uint32_t bitwise_or_reduce()
Definition IRMatch.h:1305
Halide::Internal::IRMatcher::constant_fold_bin_op
int64_t constant_fold_bin_op(halide_type_t &, int64_t, int64_t) noexcept
Halide::Internal::IRMatcher::constant_fold_cmp_op< EQ >
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op< EQ >(int64_t a, int64_t b) noexcept
Definition IRMatch.h:1214
Halide::Internal::IRMatcher::make_const_special_expr
HALIDE_NEVER_INLINE Expr make_const_special_expr(halide_type_t ty)
Definition IRMatch.h:149
Halide::Internal::IRMatcher::halving_sub
auto halving_sub(A &&a, B &&b) noexcept -> Intrin< Call::halving_sub, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:1559
Halide::Internal::IRMatcher::ge
HALIDE_ALWAYS_INLINE auto ge(A &&a, B &&b) -> decltype(IRMatcher::operator>=(a, b))
Definition IRMatch.h:1184
Halide::Internal::IRMatcher::constant_fold_cmp_op< NE >
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op< NE >(int64_t a, int64_t b) noexcept
Definition IRMatch.h:1239
Halide::Internal::IRMatcher::mod
HALIDE_ALWAYS_INLINE auto mod(A &&a, B &&b) -> decltype(IRMatcher::operator%(a, b))
Definition IRMatch.h:1038
Halide::Internal::IRMatcher::operator==
HALIDE_ALWAYS_INLINE auto operator==(A &&a, B &&b) noexcept -> CmpOp< EQ, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:1204
Halide::Internal::IRMatcher::widen_right_sub
auto widen_right_sub(A &&a, B &&b) noexcept -> Intrin< Call::widen_right_sub, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition IRMatch.h:1524
Halide::Internal::IRMatcher::constant_fold_bin_op< Add >
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Add >(halide_type_t &t, int64_t a, int64_t b) noexcept
Definition IRMatch.h:919
Halide::Internal::IRMatcher::can_prove
HALIDE_ALWAYS_INLINE auto can_prove(A &&a, Prover *p) noexcept -> CanProve< decltype(pattern_arg(a)), Prover >
Definition IRMatch.h:2401
Halide::Internal::IRMatcher::eq
HALIDE_ALWAYS_INLINE auto eq(A &&a, B &&b) -> decltype(IRMatcher::operator==(a, b))
Definition IRMatch.h:1209
Halide::Internal
Definition AbstractGenerator.h:24
Halide::Internal::div_imp
T div_imp(T a, T b)
Definition IROperator.h:278
Halide::Internal::is_const_zero
bool is_const_zero(const Expr &e)
Is the expression a const (as defined by is_const), and also equal to zero (in all lanes,...
Halide::Internal::make_zero
Expr make_zero(Type t)
Construct the representation of zero in the given type.
Halide::Internal::expr_match_test
void expr_match_test()
Halide::Internal::is_const_one
bool is_const_one(const Expr &e)
Is the expression a const (as defined by is_const), and also equal to one (in all lanes,...
Halide::Internal::SyntheticParamType::Type
@ Type
Definition Generator.h:2897
Halide::Internal::equal
bool equal(const RDom &bounds0, const RDom &bounds1)
Return true if bounds0 and bounds1 represent the same bounds.
Halide::Internal::StrongestExprNodeType
constexpr IRNodeType StrongestExprNodeType
Definition Expr.h:81
Halide::Internal::make_const
Expr make_const(Type t, int64_t val)
Construct an immediate of the given type from any numeric C++ type.
Halide::Internal::mod_imp
T mod_imp(T a, T b)
Implementations of division and mod that are specific to Halide.
Definition IROperator.h:257
Halide::Internal::sub_would_overflow
bool sub_would_overflow(int bits, int64_t a, int64_t b)
Halide::Internal::add_would_overflow
bool add_would_overflow(int bits, int64_t a, int64_t b)
Routines to test if math would overflow for signed integers with the given number of bits.
Halide::Internal::reinterpret_bits
DstType reinterpret_bits(const SrcType &src)
An aggressive form of reinterpret cast used for correct type-punning.
Definition Util.h:135
Halide::Internal::mul_would_overflow
bool mul_would_overflow(int bits, int64_t a, int64_t b)
Halide::Internal::with_lanes
Expr with_lanes(const Expr &x, int lanes)
Rewrite the expression x to have lanes lanes.
Halide::Internal::expr_match
bool expr_match(const Expr &pattern, const Expr &expr, std::vector< Expr > &result)
Does the first expression have the same structure as the second?
Halide::Internal::make_signed_integer_overflow
Expr make_signed_integer_overflow(Type type)
Construct a unique signed_integer_overflow Expr.
Halide::Internal::IRNodeType
IRNodeType
All our IR node types get unique IDs for the purposes of RTTI.
Definition Expr.h:25
Halide::Internal::IRNodeType::VectorReduce
@ VectorReduce
Definition Expr.h:60
Halide::Internal::IRNodeType::UIntImm
@ UIntImm
Definition Expr.h:31
Halide::Internal::IRNodeType::EQ
@ EQ
Definition Expr.h:45
Halide::Internal::IRNodeType::Or
@ Or
Definition Expr.h:52
Halide::Internal::IRNodeType::FloatImm
@ FloatImm
Definition Expr.h:32
Halide::Internal::IRNodeType::Cast
@ Cast
Definition Expr.h:35
Halide::Internal::IRNodeType::Mul
@ Mul
Definition Expr.h:41
Halide::Internal::IRNodeType::Max
@ Max
Definition Expr.h:44
Halide::Internal::IRNodeType::Min
@ Min
Definition Expr.h:43
Halide::Internal::IRNodeType::Ramp
@ Ramp
Definition Expr.h:56
Halide::Internal::IRNodeType::Not
@ Not
Definition Expr.h:53
Halide::Internal::IRNodeType::Broadcast
@ Broadcast
Definition Expr.h:34
Halide::Internal::IRNodeType::And
@ And
Definition Expr.h:51
Halide::Internal::IRNodeType::Call
@ Call
Definition Expr.h:57
Halide::Internal::IRNodeType::GT
@ GT
Definition Expr.h:49
Halide::Internal::IRNodeType::IntImm
@ IntImm
Definition Expr.h:30
Halide::Internal::IRNodeType::NE
@ NE
Definition Expr.h:46
Halide::Internal::IRNodeType::Select
@ Select
Definition Expr.h:54
Halide::Internal::IRNodeType::Sub
@ Sub
Definition Expr.h:39
Halide::Internal::IRNodeType::Shuffle
@ Shuffle
Definition Expr.h:59
Halide::Internal::IRNodeType::Add
@ Add
Definition Expr.h:38
Halide::Internal::IRNodeType::GE
@ GE
Definition Expr.h:50
Halide::Internal::is_const
bool is_const(const Expr &e)
Is the expression either an IntImm, a FloatImm, a StringImm, or a Cast of the same,...
Halide
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
Definition AbstractGenerator.h:19
Halide::TailStrategy::Predicate
@ Predicate
Guard the loads and stores in the loop with an if statement that prevents evaluation beyond the origi...
Definition Schedule.h:59
Halide::NameMangling::C
@ C
No name mangling.
Definition Function.h:28
uint64_t
unsigned __INT64_TYPE__ uint64_t
Definition runtime_internal.h:23
int64_t
signed __INT64_TYPE__ int64_t
Definition runtime_internal.h:22
int32_t
signed __INT32_TYPE__ int32_t
Definition runtime_internal.h:24
uint8_t
unsigned __INT8_TYPE__ uint8_t
Definition runtime_internal.h:29
uint16_t
unsigned __INT16_TYPE__ uint16_t
Definition runtime_internal.h:27
uint32_t
unsigned __INT32_TYPE__ uint32_t
Definition runtime_internal.h:25
Halide::Expr
A fragment of Halide syntax.
Definition Expr.h:258
Halide::Expr::type
HALIDE_ALWAYS_INLINE Type type() const
Get the type of this expression node.
Definition Expr.h:327
Halide::Expr::get
HALIDE_ALWAYS_INLINE const Internal::BaseExprNode * get() const
Override get() to return a BaseExprNode * instead of an IRNode *.
Definition Expr.h:321
Halide::Internal::Add
The sum of two expressions.
Definition IR.h:56
Halide::Internal::And
Logical and - are both expressions true.
Definition IR.h:175
Halide::Internal::BaseExprNode
A base class for expression nodes.
Definition Expr.h:143
Halide::Internal::BaseExprNode::type
Type type
Definition Expr.h:148
Halide::Internal::Broadcast
A vector with 'lanes' elements, in which every element is 'value'.
Definition IR.h:259
Halide::Internal::Broadcast::make
static Expr make(Expr value, int lanes)
Halide::Internal::Broadcast::value
Expr value
Definition IR.h:260
Halide::Internal::Broadcast::_node_type
static const IRNodeType _node_type
Definition IR.h:265
Halide::Internal::Broadcast::lanes
int lanes
Definition IR.h:261
Halide::Internal::Call
A function call.
Definition IR.h:490
Halide::Internal::Call::IntrinsicOp
IntrinsicOp
Definition IR.h:518
Halide::Internal::Call::signed_integer_overflow
@ signed_integer_overflow
Definition IR.h:595
Halide::Internal::Call::widen_right_mul
@ widen_right_mul
Definition IR.h:646
Halide::Internal::Call::widening_mul
@ widening_mul
Definition IR.h:651
Halide::Internal::Call::saturating_add
@ saturating_add
Definition IR.h:588
Halide::Internal::Call::abs
@ abs
Definition IR.h:519
Halide::Internal::Call::halving_add
@ halving_add
Definition IR.h:551
Halide::Internal::Call::shift_left
@ shift_left
Definition IR.h:593
Halide::Internal::Call::mul_shift_right
@ mul_shift_right
Definition IR.h:568
Halide::Internal::Call::saturating_sub
@ saturating_sub
Definition IR.h:589
Halide::Internal::Call::widening_add
@ widening_add
Definition IR.h:650
Halide::Internal::Call::widening_sub
@ widening_sub
Definition IR.h:654
Halide::Internal::Call::saturating_cast
@ saturating_cast
Definition IR.h:590
Halide::Internal::Call::rounding_mul_shift_right
@ rounding_mul_shift_right
Definition IR.h:585
Halide::Internal::Call::shift_right
@ shift_right
Definition IR.h:594
Halide::Internal::Call::halving_sub
@ halving_sub
Definition IR.h:552
Halide::Internal::Call::likely
@ likely
Definition IR.h:562
Halide::Internal::Call::widen_right_add
@ widen_right_add
Definition IR.h:644
Halide::Internal::Call::absd
@ absd
Definition IR.h:520
Halide::Internal::Call::widen_right_sub
@ widen_right_sub
Definition IR.h:648
Halide::Internal::Call::rounding_shift_right
@ rounding_shift_right
Definition IR.h:587
Halide::Internal::Call::rounding_halving_add
@ rounding_halving_add
Definition IR.h:584
Halide::Internal::Call::rounding_shift_left
@ rounding_shift_left
Definition IR.h:586
Halide::Internal::Call::likely_if_innermost
@ likely_if_innermost
Definition IR.h:563
Halide::Internal::Call::is_intrinsic
bool is_intrinsic() const
Definition IR.h:737
Halide::Internal::Call::_node_type
static const IRNodeType _node_type
Definition IR.h:795
Halide::Internal::Call::args
std::vector< Expr > args
Definition IR.h:492
Halide::Internal::Cast
The actual IR nodes begin here.
Definition IR.h:30
Halide::Internal::Cast::_node_type
static const IRNodeType _node_type
Definition IR.h:35
Halide::Internal::Cast::value
Expr value
Definition IR.h:31
Halide::Internal::Div
The ratio of two expressions.
Definition IR.h:83
Halide::Internal::EQ
Is the first expression equal to the second.
Definition IR.h:121
Halide::Internal::FloatImm
Floating point constants.
Definition Expr.h:236
Halide::Internal::FloatImm::make
static const FloatImm * make(Type t, double value)
Halide::Internal::GE
Is the first expression greater than or equal to the second.
Definition IR.h:166
Halide::Internal::GT
Is the first expression greater than the second.
Definition IR.h:157
Halide::Internal::IRMatcher::BinOp::pattern_tag
Definition IRMatch.h:629
Halide::Internal::IRMatcher::BinOp
Definition IRMatch.h:628
Halide::Internal::IRMatcher::BinOp::canonical
static constexpr bool canonical
Definition IRMatch.h:641
Halide::Internal::IRMatcher::BinOp::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition IRMatch.h:664
Halide::Internal::IRMatcher::BinOp::binds
static constexpr uint32_t binds
Definition IRMatch.h:633
Halide::Internal::IRMatcher::BinOp::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition IRMatch.h:645
Halide::Internal::IRMatcher::BinOp::foldable
static constexpr bool foldable
Definition IRMatch.h:661
Halide::Internal::IRMatcher::BinOp::a
A a
Definition IRMatch.h:630
Halide::Internal::IRMatcher::BinOp::b
B b
Definition IRMatch.h:631
Halide::Internal::IRMatcher::BinOp::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const noexcept
Definition IRMatch.h:707
Halide::Internal::IRMatcher::BinOp::match
HALIDE_ALWAYS_INLINE bool match(const BinOp< Op2, A2, B2 > &op, MatcherState &state) const noexcept
Definition IRMatch.h:655
Halide::Internal::IRMatcher::BinOp::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:636
Halide::Internal::IRMatcher::BinOp::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:635
Halide::Internal::IRMatcher::BroadcastOp::pattern_tag
Definition IRMatch.h:1737
Halide::Internal::IRMatcher::BroadcastOp
Definition IRMatch.h:1736
Halide::Internal::IRMatcher::BroadcastOp::a
A a
Definition IRMatch.h:1738
Halide::Internal::IRMatcher::BroadcastOp::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:1743
Halide::Internal::IRMatcher::BroadcastOp::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition IRMatch.h:1767
Halide::Internal::IRMatcher::BroadcastOp::lanes
B lanes
Definition IRMatch.h:1739
Halide::Internal::IRMatcher::BroadcastOp::match
HALIDE_ALWAYS_INLINE bool match(const BroadcastOp< A2, B2 > &op, MatcherState &state) const noexcept
Definition IRMatch.h:1761
Halide::Internal::IRMatcher::BroadcastOp::foldable
static constexpr bool foldable
Definition IRMatch.h:1781
Halide::Internal::IRMatcher::BroadcastOp::binds
static constexpr uint32_t binds
Definition IRMatch.h:1741
Halide::Internal::IRMatcher::BroadcastOp::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition IRMatch.h:1749
Halide::Internal::IRMatcher::BroadcastOp::canonical
static constexpr bool canonical
Definition IRMatch.h:1746
Halide::Internal::IRMatcher::BroadcastOp::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:1744
Halide::Internal::IRMatcher::BroadcastOp::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition IRMatch.h:1784
Halide::Internal::IRMatcher::CanProve::pattern_tag
Definition IRMatch.h:2376
Halide::Internal::IRMatcher::CanProve
Definition IRMatch.h:2375
Halide::Internal::IRMatcher::CanProve::make_folded_const
HALIDE_NEVER_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const
Definition IRMatch.h:2390
Halide::Internal::IRMatcher::CanProve::binds
static constexpr uint32_t binds
Definition IRMatch.h:2380
Halide::Internal::IRMatcher::CanProve::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:2383
Halide::Internal::IRMatcher::CanProve::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:2384
Halide::Internal::IRMatcher::CanProve::foldable
static constexpr bool foldable
Definition IRMatch.h:2387
Halide::Internal::IRMatcher::CanProve::canonical
static constexpr bool canonical
Definition IRMatch.h:2385
Halide::Internal::IRMatcher::CanProve::a
A a
Definition IRMatch.h:2377
Halide::Internal::IRMatcher::CanProve::prover
Prover * prover
Definition IRMatch.h:2378
Halide::Internal::IRMatcher::CastOp::pattern_tag
Definition IRMatch.h:2036
Halide::Internal::IRMatcher::CastOp
Definition IRMatch.h:2035
Halide::Internal::IRMatcher::CastOp::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:2043
Halide::Internal::IRMatcher::CastOp::foldable
static constexpr bool foldable
Definition IRMatch.h:2065
Halide::Internal::IRMatcher::CastOp::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition IRMatch.h:2047
Halide::Internal::IRMatcher::CastOp::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:2042
Halide::Internal::IRMatcher::CastOp::t
Type t
Definition IRMatch.h:2037
Halide::Internal::IRMatcher::CastOp::binds
static constexpr uint32_t binds
Definition IRMatch.h:2040
Halide::Internal::IRMatcher::CastOp::canonical
static constexpr bool canonical
Definition IRMatch.h:2044
Halide::Internal::IRMatcher::CastOp::match
HALIDE_ALWAYS_INLINE bool match(const CastOp< A2 > &op, MatcherState &state) const noexcept
Definition IRMatch.h:2056
Halide::Internal::IRMatcher::CastOp::a
A a
Definition IRMatch.h:2038
Halide::Internal::IRMatcher::CastOp::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition IRMatch.h:2061
Halide::Internal::IRMatcher::CmpOp::pattern_tag
Definition IRMatch.h:732
Halide::Internal::IRMatcher::CmpOp
Definition IRMatch.h:731
Halide::Internal::IRMatcher::CmpOp::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition IRMatch.h:800
Halide::Internal::IRMatcher::CmpOp::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:739
Halide::Internal::IRMatcher::CmpOp::binds
static constexpr uint32_t binds
Definition IRMatch.h:736
Halide::Internal::IRMatcher::CmpOp::canonical
static constexpr bool canonical
Definition IRMatch.h:740
Halide::Internal::IRMatcher::CmpOp::a
A a
Definition IRMatch.h:733
Halide::Internal::IRMatcher::CmpOp::foldable
static constexpr bool foldable
Definition IRMatch.h:763
Halide::Internal::IRMatcher::CmpOp::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition IRMatch.h:747
Halide::Internal::IRMatcher::CmpOp::b
B b
Definition IRMatch.h:734
Halide::Internal::IRMatcher::CmpOp::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:738
Halide::Internal::IRMatcher::CmpOp::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition IRMatch.h:766
Halide::Internal::IRMatcher::CmpOp::match
HALIDE_ALWAYS_INLINE bool match(const CmpOp< Op2, A2, B2 > &op, MatcherState &state) const noexcept
Definition IRMatch.h:757
Halide::Internal::IRMatcher::Fold::pattern_tag
Definition IRMatch.h:2193
Halide::Internal::IRMatcher::Fold
Definition IRMatch.h:2192
Halide::Internal::IRMatcher::Fold::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:2199
Halide::Internal::IRMatcher::Fold::binds
static constexpr uint32_t binds
Definition IRMatch.h:2196
Halide::Internal::IRMatcher::Fold::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:2198
Halide::Internal::IRMatcher::Fold::a
A a
Definition IRMatch.h:2194
Halide::Internal::IRMatcher::Fold::canonical
static constexpr bool canonical
Definition IRMatch.h:2200
Halide::Internal::IRMatcher::Fold::foldable
static constexpr bool foldable
Definition IRMatch.h:2225
Halide::Internal::IRMatcher::Fold::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const noexcept
Definition IRMatch.h:2203
Halide::Internal::IRMatcher::Fold::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition IRMatch.h:2228
Halide::Internal::IRMatcher::IntLiteral::pattern_tag
Definition IRMatch.h:489
Halide::Internal::IRMatcher::IntLiteral
Definition IRMatch.h:488
Halide::Internal::IRMatcher::IntLiteral::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:495
Halide::Internal::IRMatcher::IntLiteral::v
int64_t v
Definition IRMatch.h:490
Halide::Internal::IRMatcher::IntLiteral::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition IRMatch.h:504
Halide::Internal::IRMatcher::IntLiteral::IntLiteral
HALIDE_ALWAYS_INLINE IntLiteral(int64_t v)
Definition IRMatch.h:499
Halide::Internal::IRMatcher::IntLiteral::match
HALIDE_ALWAYS_INLINE bool match(const IntLiteral &b, MatcherState &state) const noexcept
Definition IRMatch.h:527
Halide::Internal::IRMatcher::IntLiteral::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition IRMatch.h:539
Halide::Internal::IRMatcher::IntLiteral::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:494
Halide::Internal::IRMatcher::IntLiteral::canonical
static constexpr bool canonical
Definition IRMatch.h:496
Halide::Internal::IRMatcher::IntLiteral::foldable
static constexpr bool foldable
Definition IRMatch.h:536
Halide::Internal::IRMatcher::IntLiteral::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition IRMatch.h:532
Halide::Internal::IRMatcher::IntLiteral::match
HALIDE_ALWAYS_INLINE bool match(int64_t val, MatcherState &state) const noexcept
Definition IRMatch.h:522
Halide::Internal::IRMatcher::IntLiteral::binds
static constexpr uint32_t binds
Definition IRMatch.h:492
Halide::Internal::IRMatcher::Intrin::pattern_tag
Definition IRMatch.h:1340
Halide::Internal::IRMatcher::Intrin
Definition IRMatch.h:1339
Halide::Internal::IRMatcher::Intrin::Intrin
HALIDE_ALWAYS_INLINE Intrin(Args... args) noexcept
Definition IRMatch.h:1502
Halide::Internal::IRMatcher::Intrin::print_args
HALIDE_ALWAYS_INLINE void print_args(std::ostream &s) const
Definition IRMatch.h:1394
Halide::Internal::IRMatcher::Intrin::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition IRMatch.h:1369
Halide::Internal::IRMatcher::Intrin::foldable
static constexpr bool foldable
Definition IRMatch.h:1457
Halide::Internal::IRMatcher::Intrin::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition IRMatch.h:1459
Halide::Internal::IRMatcher::Intrin::args
std::tuple< Args... > args
Definition IRMatch.h:1341
Halide::Internal::IRMatcher::Intrin::binds
static constexpr uint32_t binds
Definition IRMatch.h:1348
Halide::Internal::IRMatcher::Intrin::print_args
HALIDE_ALWAYS_INLINE void print_args(double, std::ostream &s) const
Definition IRMatch.h:1390
Halide::Internal::IRMatcher::Intrin::match_args
HALIDE_ALWAYS_INLINE bool match_args(int, const Call &c, MatcherState &state) const noexcept
Definition IRMatch.h:1357
Halide::Internal::IRMatcher::Intrin::canonical
static constexpr bool canonical
Definition IRMatch.h:1352
Halide::Internal::IRMatcher::Intrin::match_args
HALIDE_ALWAYS_INLINE bool match_args(double, const Call &c, MatcherState &state) const noexcept
Definition IRMatch.h:1364
Halide::Internal::IRMatcher::Intrin::print_args
HALIDE_ALWAYS_INLINE void print_args(int, std::ostream &s) const
Definition IRMatch.h:1381
Halide::Internal::IRMatcher::Intrin::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:1351
Halide::Internal::IRMatcher::Intrin::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition IRMatch.h:1399
Halide::Internal::IRMatcher::Intrin::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:1350
Halide::Internal::IRMatcher::Intrin::optional_type_hint
OptionalIntrinType< intrin > optional_type_hint
Definition IRMatch.h:1346
Halide::Internal::IRMatcher::IsConst::pattern_tag
Definition IRMatch.h:2323
Halide::Internal::IRMatcher::IsConst
Definition IRMatch.h:2322
Halide::Internal::IRMatcher::IsConst::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:2328
Halide::Internal::IRMatcher::IsConst::check_v
bool check_v
Definition IRMatch.h:2333
Halide::Internal::IRMatcher::IsConst::canonical
static constexpr bool canonical
Definition IRMatch.h:2330
Halide::Internal::IRMatcher::IsConst::foldable
static constexpr bool foldable
Definition IRMatch.h:2336
Halide::Internal::IRMatcher::IsConst::v
int64_t v
Definition IRMatch.h:2334
Halide::Internal::IRMatcher::IsConst::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:2329
Halide::Internal::IRMatcher::IsConst::binds
static constexpr uint32_t binds
Definition IRMatch.h:2325
Halide::Internal::IRMatcher::IsConst::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition IRMatch.h:2339
Halide::Internal::IRMatcher::IsConst::a
A a
Definition IRMatch.h:2332
Halide::Internal::IRMatcher::IsFloat::pattern_tag
Definition IRMatch.h:2414
Halide::Internal::IRMatcher::IsFloat
Definition IRMatch.h:2413
Halide::Internal::IRMatcher::IsFloat::foldable
static constexpr bool foldable
Definition IRMatch.h:2424
Halide::Internal::IRMatcher::IsFloat::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const
Definition IRMatch.h:2427
Halide::Internal::IRMatcher::IsFloat::a
A a
Definition IRMatch.h:2415
Halide::Internal::IRMatcher::IsFloat::canonical
static constexpr bool canonical
Definition IRMatch.h:2422
Halide::Internal::IRMatcher::IsFloat::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:2420
Halide::Internal::IRMatcher::IsFloat::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:2421
Halide::Internal::IRMatcher::IsFloat::binds
static constexpr uint32_t binds
Definition IRMatch.h:2417
Halide::Internal::IRMatcher::IsInt::pattern_tag
Definition IRMatch.h:2451
Halide::Internal::IRMatcher::IsInt
Definition IRMatch.h:2450
Halide::Internal::IRMatcher::IsInt::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:2460
Halide::Internal::IRMatcher::IsInt::lanes
uint16_t lanes
Definition IRMatch.h:2454
Halide::Internal::IRMatcher::IsInt::a
A a
Definition IRMatch.h:2452
Halide::Internal::IRMatcher::IsInt::bits
uint8_t bits
Definition IRMatch.h:2453
Halide::Internal::IRMatcher::IsInt::foldable
static constexpr bool foldable
Definition IRMatch.h:2463
Halide::Internal::IRMatcher::IsInt::binds
static constexpr uint32_t binds
Definition IRMatch.h:2456
Halide::Internal::IRMatcher::IsInt::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const
Definition IRMatch.h:2466
Halide::Internal::IRMatcher::IsInt::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:2459
Halide::Internal::IRMatcher::IsInt::canonical
static constexpr bool canonical
Definition IRMatch.h:2461
Halide::Internal::IRMatcher::IsMaxValue::pattern_tag
Definition IRMatch.h:2580
Halide::Internal::IRMatcher::IsMaxValue
Definition IRMatch.h:2579
Halide::Internal::IRMatcher::IsMaxValue::a
A a
Definition IRMatch.h:2581
Halide::Internal::IRMatcher::IsMaxValue::foldable
static constexpr bool foldable
Definition IRMatch.h:2590
Halide::Internal::IRMatcher::IsMaxValue::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:2586
Halide::Internal::IRMatcher::IsMaxValue::canonical
static constexpr bool canonical
Definition IRMatch.h:2588
Halide::Internal::IRMatcher::IsMaxValue::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:2587
Halide::Internal::IRMatcher::IsMaxValue::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const
Definition IRMatch.h:2593
Halide::Internal::IRMatcher::IsMaxValue::binds
static constexpr uint32_t binds
Definition IRMatch.h:2583
Halide::Internal::IRMatcher::IsMinValue::pattern_tag
Definition IRMatch.h:2621
Halide::Internal::IRMatcher::IsMinValue
Definition IRMatch.h:2620
Halide::Internal::IRMatcher::IsMinValue::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:2627
Halide::Internal::IRMatcher::IsMinValue::canonical
static constexpr bool canonical
Definition IRMatch.h:2629
Halide::Internal::IRMatcher::IsMinValue::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const
Definition IRMatch.h:2634
Halide::Internal::IRMatcher::IsMinValue::a
A a
Definition IRMatch.h:2622
Halide::Internal::IRMatcher::IsMinValue::foldable
static constexpr bool foldable
Definition IRMatch.h:2631
Halide::Internal::IRMatcher::IsMinValue::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:2628
Halide::Internal::IRMatcher::IsMinValue::binds
static constexpr uint32_t binds
Definition IRMatch.h:2624
Halide::Internal::IRMatcher::IsScalar::pattern_tag
Definition IRMatch.h:2543
Halide::Internal::IRMatcher::IsScalar
Definition IRMatch.h:2542
Halide::Internal::IRMatcher::IsScalar::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:2550
Halide::Internal::IRMatcher::IsScalar::binds
static constexpr uint32_t binds
Definition IRMatch.h:2546
Halide::Internal::IRMatcher::IsScalar::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const
Definition IRMatch.h:2556
Halide::Internal::IRMatcher::IsScalar::a
A a
Definition IRMatch.h:2544
Halide::Internal::IRMatcher::IsScalar::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:2549
Halide::Internal::IRMatcher::IsScalar::foldable
static constexpr bool foldable
Definition IRMatch.h:2553
Halide::Internal::IRMatcher::IsScalar::canonical
static constexpr bool canonical
Definition IRMatch.h:2551
Halide::Internal::IRMatcher::IsUInt::pattern_tag
Definition IRMatch.h:2497
Halide::Internal::IRMatcher::IsUInt
Definition IRMatch.h:2496
Halide::Internal::IRMatcher::IsUInt::a
A a
Definition IRMatch.h:2498
Halide::Internal::IRMatcher::IsUInt::bits
uint8_t bits
Definition IRMatch.h:2499
Halide::Internal::IRMatcher::IsUInt::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const
Definition IRMatch.h:2512
Halide::Internal::IRMatcher::IsUInt::foldable
static constexpr bool foldable
Definition IRMatch.h:2509
Halide::Internal::IRMatcher::IsUInt::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:2505
Halide::Internal::IRMatcher::IsUInt::canonical
static constexpr bool canonical
Definition IRMatch.h:2507
Halide::Internal::IRMatcher::IsUInt::binds
static constexpr uint32_t binds
Definition IRMatch.h:2502
Halide::Internal::IRMatcher::IsUInt::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:2506
Halide::Internal::IRMatcher::IsUInt::lanes
uint16_t lanes
Definition IRMatch.h:2500
Halide::Internal::IRMatcher::LanesOf::pattern_tag
Definition IRMatch.h:2664
Halide::Internal::IRMatcher::LanesOf
Definition IRMatch.h:2663
Halide::Internal::IRMatcher::LanesOf::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:2671
Halide::Internal::IRMatcher::LanesOf::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const
Definition IRMatch.h:2677
Halide::Internal::IRMatcher::LanesOf::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:2670
Halide::Internal::IRMatcher::LanesOf::foldable
static constexpr bool foldable
Definition IRMatch.h:2674
Halide::Internal::IRMatcher::LanesOf::binds
static constexpr uint32_t binds
Definition IRMatch.h:2667
Halide::Internal::IRMatcher::LanesOf::canonical
static constexpr bool canonical
Definition IRMatch.h:2672
Halide::Internal::IRMatcher::LanesOf::a
A a
Definition IRMatch.h:2665
Halide::Internal::IRMatcher::MatcherState
To save stack space, the matcher objects are largely stateless and immutable.
Definition IRMatch.h:82
Halide::Internal::IRMatcher::MatcherState::get_bound_const
HALIDE_ALWAYS_INLINE void get_bound_const(int i, halide_scalar_value_t &val, halide_type_t &type) const noexcept
Definition IRMatch.h:127
Halide::Internal::IRMatcher::MatcherState::set_bound_const
HALIDE_ALWAYS_INLINE void set_bound_const(int i, int64_t s, halide_type_t t) noexcept
Definition IRMatch.h:103
Halide::Internal::IRMatcher::MatcherState::set_bound_const
HALIDE_ALWAYS_INLINE void set_bound_const(int i, double f, halide_type_t t) noexcept
Definition IRMatch.h:115
Halide::Internal::IRMatcher::MatcherState::special_values_mask
static constexpr uint16_t special_values_mask
Definition IRMatch.h:88
Halide::Internal::IRMatcher::MatcherState::set_bound_const
HALIDE_ALWAYS_INLINE void set_bound_const(int i, halide_scalar_value_t val, halide_type_t t) noexcept
Definition IRMatch.h:121
Halide::Internal::IRMatcher::MatcherState::bound_const_type
halide_type_t bound_const_type[max_wild]
Definition IRMatch.h:90
Halide::Internal::IRMatcher::MatcherState::set_binding
HALIDE_ALWAYS_INLINE void set_binding(int i, const BaseExprNode &n) noexcept
Definition IRMatch.h:93
Halide::Internal::IRMatcher::MatcherState::MatcherState
HALIDE_ALWAYS_INLINE MatcherState() noexcept
Definition IRMatch.h:134
Halide::Internal::IRMatcher::MatcherState::get_binding
HALIDE_ALWAYS_INLINE const BaseExprNode * get_binding(int i) const noexcept
Definition IRMatch.h:98
Halide::Internal::IRMatcher::MatcherState::bound_const
halide_scalar_value_t bound_const[max_wild]
Definition IRMatch.h:84
Halide::Internal::IRMatcher::MatcherState::bindings
const BaseExprNode * bindings[max_wild]
Definition IRMatch.h:83
Halide::Internal::IRMatcher::MatcherState::set_bound_const
HALIDE_ALWAYS_INLINE void set_bound_const(int i, uint64_t u, halide_type_t t) noexcept
Definition IRMatch.h:109
Halide::Internal::IRMatcher::MatcherState::signed_integer_overflow
static constexpr uint16_t signed_integer_overflow
Definition IRMatch.h:87
Halide::Internal::IRMatcher::NegateOp::pattern_tag
Definition IRMatch.h:1954
Halide::Internal::IRMatcher::NegateOp
Definition IRMatch.h:1953
Halide::Internal::IRMatcher::NegateOp::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition IRMatch.h:1965
Halide::Internal::IRMatcher::NegateOp::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition IRMatch.h:1980
Halide::Internal::IRMatcher::NegateOp::match
HALIDE_ALWAYS_INLINE bool match(NegateOp< A2 > &&p, MatcherState &state) const noexcept
Definition IRMatch.h:1975
Halide::Internal::IRMatcher::NegateOp::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition IRMatch.h:1989
Halide::Internal::IRMatcher::NegateOp::binds
static constexpr uint32_t binds
Definition IRMatch.h:1957
Halide::Internal::IRMatcher::NegateOp::canonical
static constexpr bool canonical
Definition IRMatch.h:1962
Halide::Internal::IRMatcher::NegateOp::foldable
static constexpr bool foldable
Definition IRMatch.h:1986
Halide::Internal::IRMatcher::NegateOp::a
A a
Definition IRMatch.h:1955
Halide::Internal::IRMatcher::NegateOp::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:1960
Halide::Internal::IRMatcher::NegateOp::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:1959
Halide::Internal::IRMatcher::NotOp::pattern_tag
Definition IRMatch.h:1613
Halide::Internal::IRMatcher::NotOp
Definition IRMatch.h:1612
Halide::Internal::IRMatcher::NotOp::a
A a
Definition IRMatch.h:1614
Halide::Internal::IRMatcher::NotOp::binds
static constexpr uint32_t binds
Definition IRMatch.h:1616
Halide::Internal::IRMatcher::NotOp::foldable
static constexpr bool foldable
Definition IRMatch.h:1641
Halide::Internal::IRMatcher::NotOp::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition IRMatch.h:1623
Halide::Internal::IRMatcher::NotOp::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:1619
Halide::Internal::IRMatcher::NotOp::canonical
static constexpr bool canonical
Definition IRMatch.h:1620
Halide::Internal::IRMatcher::NotOp::match
HALIDE_ALWAYS_INLINE bool match(const NotOp< A2 > &op, MatcherState &state) const noexcept
Definition IRMatch.h:1632
Halide::Internal::IRMatcher::NotOp::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition IRMatch.h:1637
Halide::Internal::IRMatcher::NotOp::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition IRMatch.h:1644
Halide::Internal::IRMatcher::NotOp::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:1618
Halide::Internal::IRMatcher::OptionalIntrinType< Call::saturating_cast >::check
bool check(const Type &t) const
Definition IRMatch.h:1333
Halide::Internal::IRMatcher::OptionalIntrinType< Call::saturating_cast >::type
halide_type_t type
Definition IRMatch.h:1332
Halide::Internal::IRMatcher::OptionalIntrinType
Definition IRMatch.h:1324
Halide::Internal::IRMatcher::OptionalIntrinType::check
bool check(const Type &) const
Definition IRMatch.h:1325
Halide::Internal::IRMatcher::Overflow::pattern_tag
Definition IRMatch.h:2283
Halide::Internal::IRMatcher::Overflow
Definition IRMatch.h:2282
Halide::Internal::IRMatcher::Overflow::binds
static constexpr uint32_t binds
Definition IRMatch.h:2285
Halide::Internal::IRMatcher::Overflow::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:2289
Halide::Internal::IRMatcher::Overflow::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition IRMatch.h:2293
Halide::Internal::IRMatcher::Overflow::canonical
static constexpr bool canonical
Definition IRMatch.h:2290
Halide::Internal::IRMatcher::Overflow::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition IRMatch.h:2302
Halide::Internal::IRMatcher::Overflow::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition IRMatch.h:2310
Halide::Internal::IRMatcher::Overflow::foldable
static constexpr bool foldable
Definition IRMatch.h:2307
Halide::Internal::IRMatcher::Overflow::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:2288
Halide::Internal::IRMatcher::Overflows::pattern_tag
Definition IRMatch.h:2247
Halide::Internal::IRMatcher::Overflows
Definition IRMatch.h:2246
Halide::Internal::IRMatcher::Overflows::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition IRMatch.h:2261
Halide::Internal::IRMatcher::Overflows::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:2254
Halide::Internal::IRMatcher::Overflows::binds
static constexpr uint32_t binds
Definition IRMatch.h:2250
Halide::Internal::IRMatcher::Overflows::canonical
static constexpr bool canonical
Definition IRMatch.h:2256
Halide::Internal::IRMatcher::Overflows::a
A a
Definition IRMatch.h:2248
Halide::Internal::IRMatcher::Overflows::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:2255
Halide::Internal::IRMatcher::Overflows::foldable
static constexpr bool foldable
Definition IRMatch.h:2258
Halide::Internal::IRMatcher::RampOp::pattern_tag
Definition IRMatch.h:1808
Halide::Internal::IRMatcher::RampOp
Definition IRMatch.h:1807
Halide::Internal::IRMatcher::RampOp::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition IRMatch.h:1843
Halide::Internal::IRMatcher::RampOp::canonical
static constexpr bool canonical
Definition IRMatch.h:1818
Halide::Internal::IRMatcher::RampOp::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:1816
Halide::Internal::IRMatcher::RampOp::lanes
C lanes
Definition IRMatch.h:1811
Halide::Internal::IRMatcher::RampOp::b
B b
Definition IRMatch.h:1810
Halide::Internal::IRMatcher::RampOp::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:1815
Halide::Internal::IRMatcher::RampOp::binds
static constexpr uint32_t binds
Definition IRMatch.h:1813
Halide::Internal::IRMatcher::RampOp::match
HALIDE_ALWAYS_INLINE bool match(const RampOp< A2, B2, C2 > &op, MatcherState &state) const noexcept
Definition IRMatch.h:1836
Halide::Internal::IRMatcher::RampOp::foldable
static constexpr bool foldable
Definition IRMatch.h:1855
Halide::Internal::IRMatcher::RampOp::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition IRMatch.h:1821
Halide::Internal::IRMatcher::RampOp::a
A a
Definition IRMatch.h:1809
Halide::Internal::IRMatcher::Rewriter
Definition IRMatch.h:2863
Halide::Internal::IRMatcher::Rewriter::build_replacement
HALIDE_NEVER_INLINE void build_replacement(After after)
Definition IRMatch.h:2876
Halide::Internal::IRMatcher::Rewriter::validate
bool validate
Definition IRMatch.h:2868
Halide::Internal::IRMatcher::Rewriter::operator()
HALIDE_ALWAYS_INLINE bool operator()(Before before, After after, Predicate pred)
Definition IRMatch.h:2953
Halide::Internal::IRMatcher::Rewriter::operator()
HALIDE_ALWAYS_INLINE bool operator()(Before before, int64_t after) noexcept
Definition IRMatch.h:2928
Halide::Internal::IRMatcher::Rewriter::Rewriter
HALIDE_ALWAYS_INLINE Rewriter(Instance instance, halide_type_t ot, halide_type_t wt)
Definition IRMatch.h:2871
Halide::Internal::IRMatcher::Rewriter::operator()
HALIDE_ALWAYS_INLINE bool operator()(Before before, const Expr &after, Predicate pred)
Definition IRMatch.h:2982
Halide::Internal::IRMatcher::Rewriter::result
Expr result
Definition IRMatch.h:2865
Halide::Internal::IRMatcher::Rewriter::operator()
HALIDE_ALWAYS_INLINE bool operator()(Before before, const Expr &after) noexcept
Definition IRMatch.h:2910
Halide::Internal::IRMatcher::Rewriter::operator()
HALIDE_ALWAYS_INLINE bool operator()(Before before, int64_t after, Predicate pred)
Definition IRMatch.h:3005
Halide::Internal::IRMatcher::Rewriter::state
MatcherState state
Definition IRMatch.h:2866
Halide::Internal::IRMatcher::Rewriter::instance
Instance instance
Definition IRMatch.h:2864
Halide::Internal::IRMatcher::Rewriter::operator()
HALIDE_ALWAYS_INLINE bool operator()(Before before, After after)
Definition IRMatch.h:2887
Halide::Internal::IRMatcher::Rewriter::wildcard_type
halide_type_t wildcard_type
Definition IRMatch.h:2867
Halide::Internal::IRMatcher::Rewriter::output_type
halide_type_t output_type
Definition IRMatch.h:2867
Halide::Internal::IRMatcher::SelectOp::pattern_tag
Definition IRMatch.h:1671
Halide::Internal::IRMatcher::SelectOp
Definition IRMatch.h:1670
Halide::Internal::IRMatcher::SelectOp::t
T t
Definition IRMatch.h:1673
Halide::Internal::IRMatcher::SelectOp::binds
static constexpr uint32_t binds
Definition IRMatch.h:1676
Halide::Internal::IRMatcher::SelectOp::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition IRMatch.h:1708
Halide::Internal::IRMatcher::SelectOp::foldable
static constexpr bool foldable
Definition IRMatch.h:1705
Halide::Internal::IRMatcher::SelectOp::canonical
static constexpr bool canonical
Definition IRMatch.h:1681
Halide::Internal::IRMatcher::SelectOp::f
F f
Definition IRMatch.h:1674
Halide::Internal::IRMatcher::SelectOp::c
C c
Definition IRMatch.h:1672
Halide::Internal::IRMatcher::SelectOp::match
HALIDE_ALWAYS_INLINE bool match(const SelectOp< C2, T2, F2 > &instance, MatcherState &state) const noexcept
Definition IRMatch.h:1694
Halide::Internal::IRMatcher::SelectOp::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition IRMatch.h:1684
Halide::Internal::IRMatcher::SelectOp::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition IRMatch.h:1701
Halide::Internal::IRMatcher::SelectOp::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:1679
Halide::Internal::IRMatcher::SelectOp::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:1678
Halide::Internal::IRMatcher::SliceOp::pattern_tag
Definition IRMatch.h:2129
Halide::Internal::IRMatcher::SliceOp
Definition IRMatch.h:2128
Halide::Internal::IRMatcher::SliceOp::canonical
static constexpr bool canonical
Definition IRMatch.h:2139
Halide::Internal::IRMatcher::SliceOp::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:2138
Halide::Internal::IRMatcher::SliceOp::base
Base base
Definition IRMatch.h:2131
Halide::Internal::IRMatcher::SliceOp::stride
Stride stride
Definition IRMatch.h:2132
Halide::Internal::IRMatcher::SliceOp::foldable
static constexpr bool foldable
Definition IRMatch.h:2168
Halide::Internal::IRMatcher::SliceOp::SliceOp
HALIDE_ALWAYS_INLINE SliceOp(Vec v, Base b, Stride s, Lanes l)
Definition IRMatch.h:2171
Halide::Internal::IRMatcher::SliceOp::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:2137
Halide::Internal::IRMatcher::SliceOp::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition IRMatch.h:2142
Halide::Internal::IRMatcher::SliceOp::lanes
Lanes lanes
Definition IRMatch.h:2133
Halide::Internal::IRMatcher::SliceOp::binds
static constexpr uint32_t binds
Definition IRMatch.h:2135
Halide::Internal::IRMatcher::SliceOp::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition IRMatch.h:2156
Halide::Internal::IRMatcher::SliceOp::vec
Vec vec
Definition IRMatch.h:2130
Halide::Internal::IRMatcher::SpecificExpr::pattern_tag
Definition IRMatch.h:193
Halide::Internal::IRMatcher::SpecificExpr
Definition IRMatch.h:192
Halide::Internal::IRMatcher::SpecificExpr::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:198
Halide::Internal::IRMatcher::SpecificExpr::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition IRMatch.h:205
Halide::Internal::IRMatcher::SpecificExpr::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition IRMatch.h:210
Halide::Internal::IRMatcher::SpecificExpr::canonical
static constexpr bool canonical
Definition IRMatch.h:200
Halide::Internal::IRMatcher::SpecificExpr::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:199
Halide::Internal::IRMatcher::SpecificExpr::expr
const BaseExprNode & expr
Definition IRMatch.h:202
Halide::Internal::IRMatcher::SpecificExpr::binds
static constexpr uint32_t binds
Definition IRMatch.h:195
Halide::Internal::IRMatcher::SpecificExpr::foldable
static constexpr bool foldable
Definition IRMatch.h:214
Halide::Internal::IRMatcher::VectorReduceOp::pattern_tag
Definition IRMatch.h:1874
Halide::Internal::IRMatcher::VectorReduceOp
Definition IRMatch.h:1873
Halide::Internal::IRMatcher::VectorReduceOp::canonical
static constexpr bool canonical
Definition IRMatch.h:1882
Halide::Internal::IRMatcher::VectorReduceOp::match
HALIDE_ALWAYS_INLINE bool match(const VectorReduceOp< A2, B2, reduce_op_2 > &op, MatcherState &state) const noexcept
Definition IRMatch.h:1898
Halide::Internal::IRMatcher::VectorReduceOp::binds
static constexpr uint32_t binds
Definition IRMatch.h:1878
Halide::Internal::IRMatcher::VectorReduceOp::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:1880
Halide::Internal::IRMatcher::VectorReduceOp::a
A a
Definition IRMatch.h:1875
Halide::Internal::IRMatcher::VectorReduceOp::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition IRMatch.h:1885
Halide::Internal::IRMatcher::VectorReduceOp::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:1881
Halide::Internal::IRMatcher::VectorReduceOp::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition IRMatch.h:1905
Halide::Internal::IRMatcher::VectorReduceOp::foldable
static constexpr bool foldable
Definition IRMatch.h:1913
Halide::Internal::IRMatcher::VectorReduceOp::lanes
B lanes
Definition IRMatch.h:1876
Halide::Internal::IRMatcher::WidenOp::pattern_tag
Definition IRMatch.h:2082
Halide::Internal::IRMatcher::WidenOp
Definition IRMatch.h:2081
Halide::Internal::IRMatcher::WidenOp::binds
static constexpr uint32_t binds
Definition IRMatch.h:2085
Halide::Internal::IRMatcher::WidenOp::canonical
static constexpr bool canonical
Definition IRMatch.h:2089
Halide::Internal::IRMatcher::WidenOp::foldable
static constexpr bool foldable
Definition IRMatch.h:2112
Halide::Internal::IRMatcher::WidenOp::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition IRMatch.h:2106
Halide::Internal::IRMatcher::WidenOp::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:2088
Halide::Internal::IRMatcher::WidenOp::match
HALIDE_ALWAYS_INLINE bool match(const WidenOp< A2 > &op, MatcherState &state) const noexcept
Definition IRMatch.h:2101
Halide::Internal::IRMatcher::WidenOp::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:2087
Halide::Internal::IRMatcher::WidenOp::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition IRMatch.h:2092
Halide::Internal::IRMatcher::WidenOp::a
A a
Definition IRMatch.h:2083
Halide::Internal::IRMatcher::Wild::pattern_tag
Definition IRMatch.h:455
Halide::Internal::IRMatcher::WildConst::pattern_tag
Definition IRMatch.h:397
Halide::Internal::IRMatcher::WildConstFloat::pattern_tag
Definition IRMatch.h:343
Halide::Internal::IRMatcher::WildConstFloat
Definition IRMatch.h:342
Halide::Internal::IRMatcher::WildConstFloat::canonical
static constexpr bool canonical
Definition IRMatch.h:349
Halide::Internal::IRMatcher::WildConstFloat::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition IRMatch.h:352
Halide::Internal::IRMatcher::WildConstFloat::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition IRMatch.h:373
Halide::Internal::IRMatcher::WildConstFloat::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:348
Halide::Internal::IRMatcher::WildConstFloat::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:347
Halide::Internal::IRMatcher::WildConstFloat::binds
static constexpr uint32_t binds
Definition IRMatch.h:345
Halide::Internal::IRMatcher::WildConstFloat::foldable
static constexpr bool foldable
Definition IRMatch.h:380
Halide::Internal::IRMatcher::WildConstFloat::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition IRMatch.h:383
Halide::Internal::IRMatcher::WildConst
Definition IRMatch.h:396
Halide::Internal::IRMatcher::WildConst::canonical
static constexpr bool canonical
Definition IRMatch.h:403
Halide::Internal::IRMatcher::WildConst::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:402
Halide::Internal::IRMatcher::WildConst::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition IRMatch.h:431
Halide::Internal::IRMatcher::WildConst::binds
static constexpr uint32_t binds
Definition IRMatch.h:399
Halide::Internal::IRMatcher::WildConst::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition IRMatch.h:406
Halide::Internal::IRMatcher::WildConst::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition IRMatch.h:441
Halide::Internal::IRMatcher::WildConst::match
HALIDE_ALWAYS_INLINE bool match(int64_t e, MatcherState &state) const noexcept
Definition IRMatch.h:425
Halide::Internal::IRMatcher::WildConst::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:401
Halide::Internal::IRMatcher::WildConst::foldable
static constexpr bool foldable
Definition IRMatch.h:438
Halide::Internal::IRMatcher::WildConstInt::pattern_tag
Definition IRMatch.h:224
Halide::Internal::IRMatcher::WildConstInt
Definition IRMatch.h:223
Halide::Internal::IRMatcher::WildConstInt::canonical
static constexpr bool canonical
Definition IRMatch.h:230
Halide::Internal::IRMatcher::WildConstInt::foldable
static constexpr bool foldable
Definition IRMatch.h:274
Halide::Internal::IRMatcher::WildConstInt::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition IRMatch.h:267
Halide::Internal::IRMatcher::WildConstInt::binds
static constexpr uint32_t binds
Definition IRMatch.h:226
Halide::Internal::IRMatcher::WildConstInt::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const
Definition IRMatch.h:277
Halide::Internal::IRMatcher::WildConstInt::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition IRMatch.h:233
Halide::Internal::IRMatcher::WildConstInt::match
HALIDE_ALWAYS_INLINE bool match(int64_t value, MatcherState &state) const noexcept
Definition IRMatch.h:254
Halide::Internal::IRMatcher::WildConstInt::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:228
Halide::Internal::IRMatcher::WildConstInt::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:229
Halide::Internal::IRMatcher::WildConstUInt::pattern_tag
Definition IRMatch.h:290
Halide::Internal::IRMatcher::WildConstUInt
Definition IRMatch.h:289
Halide::Internal::IRMatcher::WildConstUInt::binds
static constexpr uint32_t binds
Definition IRMatch.h:292
Halide::Internal::IRMatcher::WildConstUInt::foldable
static constexpr bool foldable
Definition IRMatch.h:327
Halide::Internal::IRMatcher::WildConstUInt::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:295
Halide::Internal::IRMatcher::WildConstUInt::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition IRMatch.h:299
Halide::Internal::IRMatcher::WildConstUInt::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition IRMatch.h:330
Halide::Internal::IRMatcher::WildConstUInt::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:294
Halide::Internal::IRMatcher::WildConstUInt::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition IRMatch.h:320
Halide::Internal::IRMatcher::WildConstUInt::canonical
static constexpr bool canonical
Definition IRMatch.h:296
Halide::Internal::IRMatcher::Wild
Definition IRMatch.h:454
Halide::Internal::IRMatcher::Wild::min_node_type
static constexpr IRNodeType min_node_type
Definition IRMatch.h:459
Halide::Internal::IRMatcher::Wild::binds
static constexpr uint32_t binds
Definition IRMatch.h:457
Halide::Internal::IRMatcher::Wild::max_node_type
static constexpr IRNodeType max_node_type
Definition IRMatch.h:460
Halide::Internal::IRMatcher::Wild::canonical
static constexpr bool canonical
Definition IRMatch.h:461
Halide::Internal::IRMatcher::Wild::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition IRMatch.h:473
Halide::Internal::IRMatcher::Wild::foldable
static constexpr bool foldable
Definition IRMatch.h:477
Halide::Internal::IRMatcher::Wild::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition IRMatch.h:464
Halide::Internal::IRMatcher::bindings
Definition IRMatch.h:145
Halide::Internal::IRMatcher::bindings::mask
static constexpr uint32_t mask
Definition IRMatch.h:146
Halide::Internal::IRMatcher::enable_if_pattern::type
Definition IRMatch.h:141
Halide::Internal::IRMatcher::enable_if_pattern
Definition IRMatch.h:140
Halide::Internal::IRNode::node_type
IRNodeType node_type
Each IR node subclass has a unique identifier.
Definition Expr.h:113
Halide::Internal::IntImm
Integer constants.
Definition Expr.h:218
Halide::Internal::IntImm::make
static const IntImm * make(Type t, int64_t value)
Halide::Internal::LE
Is the first expression less than or equal to the second.
Definition IR.h:148
Halide::Internal::LT
Is the first expression less than the second.
Definition IR.h:139
Halide::Internal::Max
The greater of two values.
Definition IR.h:112
Halide::Internal::Min
The lesser of two values.
Definition IR.h:103
Halide::Internal::Mod
The remainder of a / b.
Definition IR.h:94
Halide::Internal::Mul
The product of two expressions.
Definition IR.h:74
Halide::Internal::NE
Is the first expression not equal to the second.
Definition IR.h:130
Halide::Internal::Not
Logical not - true if the expression false.
Definition IR.h:193
Halide::Internal::Not::make
static Expr make(Expr a)
Halide::Internal::Not::a
Expr a
Definition IR.h:194
Halide::Internal::Or
Logical or - is at least one of the expression true.
Definition IR.h:184
Halide::Internal::Ramp
A linear ramp vector node.
Definition IR.h:247
Halide::Internal::Ramp::stride
Expr stride
Definition IR.h:248
Halide::Internal::Ramp::_node_type
static const IRNodeType _node_type
Definition IR.h:253
Halide::Internal::Ramp::lanes
int lanes
Definition IR.h:249
Halide::Internal::Ramp::make
static Expr make(Expr base, Expr stride, int lanes)
Halide::Internal::Ramp::base
Expr base
Definition IR.h:248
Halide::Internal::Select
A ternary operator.
Definition IR.h:204
Halide::Internal::Select::make
static Expr make(Expr condition, Expr true_value, Expr false_value)
Halide::Internal::Select::false_value
Expr false_value
Definition IR.h:205
Halide::Internal::Select::true_value
Expr true_value
Definition IR.h:205
Halide::Internal::Select::condition
Expr condition
Definition IR.h:205
Halide::Internal::Select::_node_type
static const IRNodeType _node_type
Definition IR.h:209
Halide::Internal::Shuffle
Construct a new vector by taking elements from another sequence of vectors.
Definition IR.h:884
Halide::Internal::Shuffle::make_slice
static Expr make_slice(Expr vector, int begin, int stride, int size)
Convenience constructor for making a shuffle representing a contiguous subset of a vector.
Halide::Internal::Shuffle::vectors
std::vector< Expr > vectors
Definition IR.h:885
Halide::Internal::Shuffle::is_slice
bool is_slice() const
Check if this shuffle is a contiguous strict subset of the vector arguments, and if so,...
Halide::Internal::Shuffle::slice_stride
int slice_stride() const
Check if this shuffle is a contiguous strict subset of the vector arguments, and if so,...
Definition IR.h:938
Halide::Internal::Shuffle::slice_begin
int slice_begin() const
Check if this shuffle is a contiguous strict subset of the vector arguments, and if so,...
Definition IR.h:935
Halide::Internal::Sub
The difference of two expressions.
Definition IR.h:65
Halide::Internal::Sub::_node_type
static const IRNodeType _node_type
Definition IR.h:70
Halide::Internal::Sub::a
Expr a
Definition IR.h:66
Halide::Internal::Sub::make
static Expr make(Expr a, Expr b)
Halide::Internal::Sub::b
Expr b
Definition IR.h:66
Halide::Internal::UIntImm
Unsigned integer constants.
Definition Expr.h:227
Halide::Internal::UIntImm::make
static const UIntImm * make(Type t, uint64_t value)
Halide::Internal::VectorReduce
Horizontally reduce a vector to a scalar or narrower vector using the given commutative and associati...
Definition IR.h:1012
Halide::Internal::VectorReduce::_node_type
static const IRNodeType _node_type
Definition IR.h:1031
Halide::Internal::VectorReduce::make
static Expr make(Operator op, Expr vec, int lanes)
Halide::Internal::VectorReduce::value
Expr value
Definition IR.h:1026
Halide::Internal::VectorReduce::op
Operator op
Definition IR.h:1027
Halide::Internal::VectorReduce::Min
@ Min
Definition IR.h:1020
Halide::Internal::VectorReduce::Add
@ Add
Definition IR.h:1017
Halide::Internal::VectorReduce::And
@ And
Definition IR.h:1022
Halide::Internal::VectorReduce::Or
@ Or
Definition IR.h:1023
Halide::Internal::VectorReduce::Max
@ Max
Definition IR.h:1021
Halide::Type
Types in the halide type system.
Definition Type.h:283
Halide::Type::widen
Type widen() const
Return Type with the same type code and number of lanes, but with at least twice as many bits.
Definition Type.h:378
Halide::Type::is_int
HALIDE_ALWAYS_INLINE bool is_int() const
Is this type a signed integer type?
Definition Type.h:435
Halide::Type::lanes
HALIDE_ALWAYS_INLINE int lanes() const
Return the number of vector elements in this type.
Definition Type.h:355
Halide::Type::is_uint
HALIDE_ALWAYS_INLINE bool is_uint() const
Is this type an unsigned integer type?
Definition Type.h:441
Halide::Type::bits
HALIDE_ALWAYS_INLINE int bits() const
Return the bit size of a single element of this type.
Definition Type.h:349
Halide::Type::is_scalar
HALIDE_ALWAYS_INLINE bool is_scalar() const
Is this type a scalar type?
Definition Type.h:417
Halide::Type::is_float
HALIDE_ALWAYS_INLINE bool is_float() const
Is this type a floating point type (float or double).
Definition Type.h:423
halide_scalar_value_t
halide_scalar_value_t is a simple union able to represent all the well-known scalar values in a filte...
Definition HalideRuntime.h:1741
halide_scalar_value_t::u
union halide_scalar_value_t::@314322015121151262135054202130057122113055355347 u
halide_scalar_value_t::i64
int64_t i64
Definition HalideRuntime.h:1747
halide_scalar_value_t::u64
uint64_t u64
Definition HalideRuntime.h:1751
halide_scalar_value_t::f64
double f64
Definition HalideRuntime.h:1753
halide_type_t
A runtime tag for a type in the halide type system.
Definition HalideRuntime.h:503
halide_type_t::bits
uint8_t bits
The number of bits of precision of a single scalar value of this type.
Definition HalideRuntime.h:515
halide_type_t::lanes
uint16_t lanes
How many elements in a vector.
Definition HalideRuntime.h:519
halide_type_t::code
uint8_t code
The basic type code: signed integer, unsigned integer, or floating point.
Definition HalideRuntime.h:510