11#ifndef O2_FRAMEWORK_EXPRESSIONS_H_
12#define O2_FRAMEWORK_EXPRESSIONS_H_
21#include <arrow/type_fwd.h>
22#include <gandiva/gandiva_aliases.h>
23#include <arrow/type.h>
24#include <gandiva/arrow.h>
25#if !defined(__CLING__) && !defined(__ROOTCLING__)
26#include <arrow/table.h>
27#include <gandiva/selection_vector.h>
28#include <gandiva/node.h>
29#include <gandiva/filter.h>
30#include <gandiva/projector.h>
46using Selection = std::shared_ptr<gandiva::SelectionVector>;
52 ExpressionInfo(
int ai,
size_t hash, std::set<uint32_t>&& hs, gandiva::SchemaPtr sc)
63 gandiva::NodePtr
tree =
nullptr;
74template <
typename... T>
80using LiteralValue =
LiteralStorage<int, bool, float, double, uint8_t, int64_t, int16_t, uint16_t, int8_t, uint32_t, uint64_t>;
88#define SELECT_ARROW_TYPE(_Ctype_, _Atype_) \
89 template <typename T> \
90 requires std::same_as<T, _Ctype_> \
91 constexpr auto selectArrowType() \
93 return atype::_Atype_; \
118 template <
typename T>
154 template <
typename T>
161 template <
typename T,
typename AT>
168 template <
typename T>
187template <
typename AT,
typename T>
201 template <
typename T>
202 void reset(T value_,
int index_ = -1)
219concept is_literal_like = std::same_as<T, LiteralNode> || std::same_as<T, PlaceholderNode> || std::same_as<T, ParameterNode>;
291 if (
other.left !=
nullptr) {
292 left = std::make_unique<Node>(*
other.left);
294 if (
other.right !=
nullptr) {
297 if (
other.condition !=
nullptr) {
307 using self_t = std::variant<LiteralNode, BindingNode, OpNode, PlaceholderNode, ConditionalNode, ParameterNode>;
311 std::unique_ptr<Node>
left =
nullptr;
312 std::unique_ptr<Node>
right =
nullptr;
327 return this->node_ptr != rhs.node_ptr;
335 std::stack<NodeRecord>
path;
336 path.emplace(head, 0);
337 while (!
path.empty()) {
341 auto* leftp =
top.node_ptr->left.get();
342 auto* rightp =
top.node_ptr->right.get();
343 auto* condp =
top.node_ptr->condition.get();
346 if (leftp !=
nullptr) {
347 path.emplace(leftp, 0);
349 if (rightp !=
nullptr) {
350 path.emplace(rightp, 0);
352 if (condp !=
nullptr) {
353 path.emplace(condp, 0);
364#define BINARY_OP_NODES(_operator_, _operation_) \
365 inline Node operator _operator_(Node&& left, Node&& right) \
367 return Node{OpNode{BasicOp::_operation_}, std::forward<Node>(left), std::forward<Node>(right)}; \
369 template <arithmetic T> \
370 inline Node operator _operator_(Node&& left, T right) \
372 return Node{OpNode{BasicOp::_operation_}, std::forward<Node>(left), LiteralNode{right}}; \
374 template <arithmetic T> \
375 inline Node operator _operator_(T left, Node&& right) \
377 return Node{OpNode{BasicOp::_operation_}, LiteralNode{left}, std::forward<Node>(right)}; \
379 template <typename T> \
380 inline Node operator _operator_(Node&& left, Configurable<T> const& right) \
382 return Node{OpNode{BasicOp::_operation_}, std::forward<Node>(left), PlaceholderNode{right}}; \
384 template <typename T> \
385 inline Node operator _operator_(Configurable<T> const& left, Node&& right) \
387 return Node{OpNode{BasicOp::_operation_}, PlaceholderNode{left}, std::forward<Node>(right)}; \
389 inline Node operator _operator_(BindingNode const& left, BindingNode const& right) \
391 return Node{OpNode{BasicOp::_operation_}, left, right}; \
393 inline Node operator _operator_(BindingNode const& left, Node&& right) \
395 return Node{OpNode{BasicOp::_operation_}, left, std::forward<Node>(right)}; \
397 inline Node operator _operator_(Node&& left, BindingNode const& right) \
399 return Node{OpNode{BasicOp::_operation_}, std::forward<Node>(left), right}; \
401 template <typename T> \
402 inline Node operator _operator_(Configurable<T> const& left, BindingNode const& right) \
404 return Node{OpNode{BasicOp::_operation_}, PlaceholderNode{left}, right}; \
406 template <typename T> \
407 inline Node operator _operator_(BindingNode const& left, Configurable<T> const& right) \
409 return Node{OpNode{BasicOp::_operation_}, left, PlaceholderNode{right}}; \
429template <arithmetic T>
435#define BINARY_FUNC_NODES(_func_, _node_) \
436 template <arithmetic L, arithmetic R> \
437 inline Node _node_(L left, R right) \
439 return Node{OpNode{BasicOp::_func_}, LiteralNode{left}, LiteralNode{right}}; \
442 inline Node _node_(Node&& left, Node&& right) \
444 return Node{OpNode{BasicOp::_func_}, std::forward<Node>(left), std::forward<Node>(right)}; \
447 inline Node _node_(Node&& left, BindingNode const& right) \
449 return Node{OpNode{BasicOp::_func_}, std::forward<Node>(left), right}; \
452 inline Node _node_(BindingNode const& left, BindingNode const& right) \
454 return Node{OpNode{BasicOp::_func_}, left, right}; \
457 inline Node _node_(BindingNode const& left, Node&& right) \
459 return Node{OpNode{BasicOp::_func_}, left, std::forward<Node>(right)}; \
462 template <typename T> \
463 inline Node _node_(Node&& left, Configurable<T> const& right) \
465 return Node{OpNode{BasicOp::_func_}, std::forward<Node>(left), PlaceholderNode{right}}; \
468 template <typename T> \
469 inline Node _node_(Configurable<T> const& left, Node&& right) \
471 return Node{OpNode{BasicOp::_func_}, PlaceholderNode{left}, std::forward<Node>(right)}; \
474 template <typename T> \
475 inline Node _node_(BindingNode const& left, Configurable<T> const& right) \
477 return Node{OpNode{BasicOp::_func_}, left, PlaceholderNode{right}}; \
480 template <typename T> \
481 inline Node _node_(Configurable<T> const& left, BindingNode const& right) \
483 return Node{OpNode{BasicOp::_func_}, PlaceholderNode{left}, right}; \
487#define ncheckbit(_node_, _bit_) ((_node_ & _bit_) == _bit_)
490#define UNARY_FUNC_NODES(_func_, _node_) \
491 inline Node _node_(Node&& arg) \
493 return Node{OpNode{BasicOp::_func_}, std::forward<Node>(arg)}; \
513 return Node{
ConditionalNode{}, std::forward<Node>(then_), std::forward<Node>(else_), std::forward<Node>(condition_)};
516template <arithmetic L>
522template <arithmetic L>
528template <arithmetic L1, arithmetic L2>
552template <
typename L1,
typename L2>
569 const auto binned_copy =
binned;
570 const auto out_copy = out;
572 auto* current = &root;
573 for (
auto i = 0;
i <
bins; ++
i) {
575 current = current->right.get();
577 current->right = std::make_unique<Node>(out);
586 if (node->
self.index() == 5) {
587 auto* n = std::get_if<5>(&node->self);
588 n->reset(parameters[n->index * bins + bin]);
642 if (
node !=
nullptr) {
648 std::unique_ptr<Node>
node =
nullptr;
675 gandiva::SchemaPtr
const& Schema);
677std::shared_ptr<gandiva::Filter>
createFilter(gandiva::SchemaPtr
const& Schema,
678 gandiva::ConditionPtr condition);
680std::shared_ptr<gandiva::Filter>
createFilter(gandiva::SchemaPtr
const& Schema,
683std::shared_ptr<gandiva::Projector>
createProjector(gandiva::SchemaPtr
const& Schema,
685 gandiva::FieldPtr
result);
687std::shared_ptr<gandiva::Projector>
createProjector(gandiva::SchemaPtr
const& Schema,
689 gandiva::FieldPtr
result);
700 std::shared_ptr<arrow::Schema> schema,
701 std::vector<std::shared_ptr<arrow::Field>>
const& fields);
703template <
typename... C>
762 Tokenizer(std::string
const& input =
"");
763 void reset(std::string
const& input);
770 static Node parse(std::string
const& input);
#define SELECT_ARROW_TYPE(_Ctype_, _Atype_)
#define BINARY_FUNC_NODES(_func_, _node_)
#define UNARY_FUNC_NODES(_func_, _node_)
unary functions on nodes
T get(const char *key) const
ConfigParamRegistry const & options()
GLsizei const GLchar *const * string
GLdouble GLdouble GLdouble GLdouble top
GLuint const GLchar * name
GLsizei const GLfloat * value
GLint GLint GLsizei GLint GLenum GLenum type
typedef void(APIENTRYP PFNGLCULLFACEPROC)(GLenum mode)
GLint GLint GLint GLint GLint GLint GLint GLbitfield GLenum filter
GLsizei const GLchar *const * path
std::shared_ptr< gandiva::Filter > FilterPtr
std::shared_ptr< gandiva::SelectionVector > Selection
std::shared_ptr< arrow::DataType > concreteArrowType(atype::type type)
std::shared_ptr< gandiva::Filter > createFilter(gandiva::SchemaPtr const &Schema, gandiva::ConditionPtr condition)
Function to create gandiva filter from gandiva condition.
gandiva::ExpressionPtr makeExpression(gandiva::NodePtr node, gandiva::FieldPtr result)
Function to create gandiva projecting expression from generic gandiva expression tree.
BINARY_OP_NODES && LogicalAnd
BINARY_OP_NODES & BitwiseAnd
std::shared_ptr< gandiva::Projector > createProjectorHelper(size_t nColumns, expressions::Projector *projectors, std::shared_ptr< arrow::Schema > schema, std::vector< std::shared_ptr< arrow::Field > > const &fields)
gandiva::Selection createSelection(std::shared_ptr< arrow::Table > const &table, Filter const &expression)
Function for creating gandiva selection from our internal filter tree.
PlaceholderNode as(Configurable< T > const &v)
const char * stringType(atype::type t)
std::vector< ColumnOperationSpec > Operations
void updateExpressionInfos(expressions::Filter const &filter, std::vector< ExpressionInfo > &eInfos)
Function for attaching gandiva filters to to compatible task inputs.
constexpr auto selectArrowType()
Node par(int index)
Parameters.
Operations createOperations(Filter const &expression)
Function to create an internal operation sequence from a filter tree.
Node updateParameters(Node const &pexp, int bins, std::vector< T > const ¶meters, int bin)
gandiva::ConditionPtr makeCondition(gandiva::NodePtr node)
Function to create gandiva condition expression from generic gandiva expression tree.
bool isTableCompatible(std::set< uint32_t > const &hashes, Operations const &specs)
Function to check compatibility of a given arrow schema with operation sequence.
void unknownParameterUsed(const char *name)
void walk(Node *head, L &&pred)
Tree-walker helper.
BINARY_OP_NODES(<, LessThan)
Node ifnode(Node &&condition_, Node &&then_, Node &&else_)
conditionals
Node npow(Node &&left, T right)
functions
gandiva::NodePtr createExpressionTree(Operations const &opSpecs, gandiva::SchemaPtr const &Schema)
Function to create gandiva expression tree from operation sequence.
Node binned(std::vector< T > const &binning, std::vector< T > const ¶meters, Node &&binned, Node &&pexp, Node &&out)
binned functional
void updatePlaceholders(Filter &filter, InitContext &context)
Update placeholder nodes from context.
BINARY_OP_NODES^ BitwiseXor
BINARY_OP_NODES * Multiplication
Node protect0(Node &&expr)
division by 0 protector
Node ncfg(T defaultValue, std::string path)
context-independent configurable
std::string upcastTo(atype::type f)
std::shared_ptr< gandiva::Projector > createProjector(gandiva::SchemaPtr const &Schema, Operations const &opSpecs, gandiva::FieldPtr result)
Function to create gandiva projector from operation sequence.
std::shared_ptr< gandiva::Projector > createProjectors(framework::pack< C... >, std::vector< std::shared_ptr< arrow::Field > > const &fields, gandiva::SchemaPtr schema)
void updateFilterInfo(ExpressionInfo &info, std::shared_ptr< arrow::Table > &table)
constexpr VariantType variant_trait_v
std::vector< int64_t > SelectionVector
Defining DataPointCompositeObject explicitly as copiable.
gandiva::Selection selection
gandiva::SchemaPtr schema
ExpressionInfo(int ai, size_t hash, std::set< uint32_t > &&hs, gandiva::SchemaPtr sc)
std::set< uint32_t > hashes
An expression tree node corresponding to a column binding.
constexpr BindingNode(const char *name_, uint32_t hash_, atype::type type_)
BindingNode(BindingNode const &)=default
constexpr BindingNode(uint32_t hash_, atype::type type_)
BindingNode(BindingNode &&)=delete
A struct, containing the root of the expression tree.
Filter & operator=(std::string const &input_)
Filter(std::string const &input_)
Filter & operator=(Filter &&other) noexcept
size_t designateSubtrees(Node *node, size_t index=0)
std::unique_ptr< Node > node
An expression tree node corresponding to a literal value.
LiteralValue::stored_type var_t
std::variant< T... > stored_type
helper struct used to parse trees
bool operator!=(NodeRecord const &rhs)
Node * node_ptr
pointer to the actual tree node
NodeRecord(Node *node_, size_t index_)
Node(PlaceholderNode &&v)
Node(OpNode op, Node &&l)
std::unique_ptr< Node > condition
Node(ConditionalNode op, Node &&then_, std::unique_ptr< Node > &&else_, Node &&condition_)
std::unique_ptr< Node > right
std::variant< LiteralNode, BindingNode, OpNode, PlaceholderNode, ConditionalNode, ParameterNode > self_t
variant with possible nodes
Node(BindingNode const &n)
Node(OpNode op, Node &&l, Node &&r)
Node(OpNode op, std::unique_ptr< Node > &&l, Node &&r)
std::string binding
buffer for dynamic binding
Node(BindingNode const &n, std::string binding_)
Node(ConditionalNode op, Node &&then_, Node &&else_, Node &&condition_)
An expression tree node corresponding to binary or unary operation.
A placeholder node for parameters taken from an array.
void reset(T value_, int index_=-1)
ParameterNode(int index_=-1)
static std::unique_ptr< Node > parseBase(Tokenizer &tk)
static std::unique_ptr< Node > parseTier8(Tokenizer &tk)
static std::unique_ptr< Node > parseTier3(Tokenizer &tk)
static std::unique_ptr< Node > parseTier1(Tokenizer &tk)
static std::unique_ptr< Node > parseTier6(Tokenizer &tk)
static std::unique_ptr< Node > parseTier7(Tokenizer &tk)
static std::unique_ptr< Node > parsePrimary(Tokenizer &tk)
static std::unique_ptr< Node > parseTier4(Tokenizer &tk)
static std::unique_ptr< Node > parseTier2(Tokenizer &tk)
static Node parse(std::string const &input)
static OpNode opFromToken(std::string const &token)
static std::unique_ptr< Node > parseTier5(Tokenizer &tk)
A placeholder node for simple type configurable.
void reset(InitContext &context)
LiteralNode::var_t(* retrieve)(InitContext &, char const *)
PlaceholderNode(Configurable< T > const &v)
PlaceholderNode(Configurable< T > const &v, AT *)
PlaceholderNode(T defaultValue, std::string &&path)
void reset(std::string const &input)
std::variant< uint32_t, int32_t, uint64_t, int64_t > IntegerValue
std::string::iterator current
std::variant< float, double > FloatValue
std::string IdentifierStr
VectorOfTObjectPtrs other