From 64285a1f25e2c7b85ed1f00b7947403e92baea00 Mon Sep 17 00:00:00 2001 From: Grant Watson Date: Wed, 16 Nov 2022 15:32:39 +0000 Subject: Extend reference model API with eager operator execution entrypoints - Adds a script to generate operators.h and operators.cc - Adds jinja2 templates for generating operators.h and operators.cc - Adds unit tests for a subset of the operators generated - Includes the TOSA specification as a submodule - Adds supporting C++ and header files Signed-off-by: Grant Watson Change-Id: I5b60db4c56113110d8e75fe1152525d258233f9c --- .gitmodules | 3 + reference_model/CMakeLists.txt | 1 + reference_model/include/model_runner.h | 14 + reference_model/include/operators.h | 337 ++++ reference_model/src/array_proxy.h | 98 + reference_model/src/model_runner.cc | 12 +- reference_model/src/model_runner_impl.cc | 178 +- reference_model/src/model_runner_impl.h | 7 +- reference_model/src/operators.cc | 2325 ++++++++++++++++++++++++ reference_model/src/tensor.cc | 21 +- reference_model/src/tensor.h | 23 +- reference_model/test/model_runner_tests.cpp | 397 +++- scripts/operator_api/README.md | 19 + scripts/operator_api/generate_api.py | 349 ++++ scripts/operator_api/templates/operators_cc.j2 | 176 ++ scripts/operator_api/templates/operators_h.j2 | 74 + thirdparty/specification | 1 + 17 files changed, 3882 insertions(+), 153 deletions(-) create mode 100644 reference_model/include/operators.h create mode 100644 reference_model/src/array_proxy.h create mode 100644 reference_model/src/operators.cc create mode 100644 scripts/operator_api/README.md create mode 100644 scripts/operator_api/generate_api.py create mode 100644 scripts/operator_api/templates/operators_cc.j2 create mode 100644 scripts/operator_api/templates/operators_h.j2 create mode 160000 thirdparty/specification diff --git a/.gitmodules b/.gitmodules index 87ce1ef..5ed5edb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "thirdparty/doctest"] path = thirdparty/doctest url = https://github.com/doctest/doctest.git +[submodule "thirdparty/specification"] + path = thirdparty/specification + url = https://review.mlplatform.org/tosa/specification diff --git a/reference_model/CMakeLists.txt b/reference_model/CMakeLists.txt index 04b0db5..6494225 100644 --- a/reference_model/CMakeLists.txt +++ b/reference_model/CMakeLists.txt @@ -66,6 +66,7 @@ set(CXX_SOURCE src/graph_node.cc src/subgraph_traverser.cc src/func_debug.cc + src/operators.cc src/ops/op_factory.cc src/ops/tensor_ops.cc src/ops/activation_funcs.cc diff --git a/reference_model/include/model_runner.h b/reference_model/include/model_runner.h index 4335794..86d0056 100644 --- a/reference_model/include/model_runner.h +++ b/reference_model/include/model_runner.h @@ -70,6 +70,13 @@ public: template int setInput(std::string input_name, std::vector& vals); + /* + * Set the input tensors for the model through a raw byte buffer. + * The input_name much match the input tensor name in the model. + * NOTE: setInput() must be called for each input tensor before run() is called. + */ + int setInput(std::string input_name, uint8_t* raw_ptr, size_t size); + /* * Retrieve the output tensors from the graph after running. * The output_name much match the output tensor name in the model. @@ -78,6 +85,13 @@ public: template std::vector getOutput(std::string output_name); + /* + * Retrieve the output tensors from the graph after running in a raw byte buffer. + * The output_name much match the output tensor name in the model. + * NOTE: run() must be called before outputs are retrieved. + */ + int getOutput(std::string output_name, uint8_t* raw_ptr, size_t size); + private: std::unique_ptr model_runner_impl; }; diff --git a/reference_model/include/operators.h b/reference_model/include/operators.h new file mode 100644 index 0000000..6e21e95 --- /dev/null +++ b/reference_model/include/operators.h @@ -0,0 +1,337 @@ + +// Copyright (c) 2022, ARM Limited. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// THIS FILE IS GENERATED. DO NOT EDIT! +// See scripts/operator_api/generate_api.py + +#ifndef OPERATORS_H_ +#define OPERATORS_H_ + +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif /* __cplusplus */ + + // Note status needs to be aligned with graph_status + enum tosa_status_t + { + tosa_status_valid = 0, + tosa_status_unpredictable = 1, + tosa_status_error = 2 + }; + + enum tosa_mode_t + { + tosa_mode_unknown = 0, + tosa_mode_nearest = 1, + tosa_mode_bilinear = 2, + tosa_mode_min = 3, + tosa_mode_max = 4 + }; + + enum tosa_datatype_t + { + tosa_datatype_bf16_t = 0, + tosa_datatype_bool_t = 1, + tosa_datatype_fp16_t = 2, + tosa_datatype_fp32_t = 3, + tosa_datatype_int16_t = 4, + tosa_datatype_int32_t = 5, + tosa_datatype_int48_t = 6, + tosa_datatype_int4_t = 7, + tosa_datatype_int8_t = 8, + tosa_datatype_uint16_t = 9, + tosa_datatype_uint8_t = 10, + }; + + struct tosa_tensor_t + { + int32_t* shape; + int32_t num_dims; + tosa_datatype_t data_type; + uint8_t* data; + size_t size; + }; + + tosa_status_t tosa_run_argmax(tosa_tensor_t client_input, const int32_t client_axis, tosa_tensor_t client_output); + + tosa_status_t tosa_run_avg_pool2d(tosa_tensor_t client_input, + const int32_t client_kernel[2], + const int32_t client_stride[2], + const int32_t client_pad[4], + const int32_t client_input_zp, + const int32_t client_output_zp, + tosa_tensor_t client_output); + + tosa_status_t tosa_run_conv2d(tosa_tensor_t client_input, + tosa_tensor_t client_weight, + tosa_tensor_t client_bias, + const int32_t client_pad[4], + const int32_t client_stride[2], + const int32_t client_dilation[2], + const int32_t client_input_zp, + const int32_t client_weight_zp, + tosa_tensor_t client_output); + + tosa_status_t tosa_run_conv3d(tosa_tensor_t client_input, + tosa_tensor_t client_weight, + tosa_tensor_t client_bias, + const int32_t client_pad[6], + const int32_t client_stride[3], + const int32_t client_dilation[3], + const int32_t client_input_zp, + const int32_t client_weight_zp, + tosa_tensor_t client_output); + + tosa_status_t tosa_run_depthwise_conv2d(tosa_tensor_t client_input, + tosa_tensor_t client_weight, + tosa_tensor_t client_bias, + const int32_t client_pad[4], + const int32_t client_stride[2], + const int32_t client_dilation[2], + const int32_t client_input_zp, + const int32_t client_weight_zp, + tosa_tensor_t client_output); + + tosa_status_t tosa_run_fully_connected(tosa_tensor_t client_input, + const int32_t client_input_zp, + const int32_t client_weight_zp, + tosa_tensor_t client_output); + + tosa_status_t tosa_run_matmul(tosa_tensor_t client_a, + tosa_tensor_t client_b, + const int32_t client_a_zp, + const int32_t client_b_zp, + tosa_tensor_t client_output); + + tosa_status_t tosa_run_max_pool2d(tosa_tensor_t client_input, + const int32_t client_kernel[2], + const int32_t client_stride[2], + const int32_t client_pad[4], + const int32_t client_input_zp, + const int32_t client_output_zp, + tosa_tensor_t client_output); + + tosa_status_t tosa_run_transpose_conv2d(tosa_tensor_t client_input, + tosa_tensor_t client_weight, + tosa_tensor_t client_bias, + const int32_t client_out_pad[4], + const int32_t client_stride[2], + const int32_t client_out_shape[4], + const int32_t client_input_zp, + const int32_t client_weight_zp, + const int32_t client_pad_len, + const int32_t client_pad[], + const int32_t client_dilation_len, + const int32_t client_dilation[], + tosa_tensor_t client_output); + + tosa_status_t tosa_run_clamp(tosa_tensor_t client_input, + const int32_t client_min_int, + const int32_t client_max_int, + const float client_min_fp, + const float client_max_fp, + tosa_tensor_t client_output); + + tosa_status_t tosa_run_sigmoid(tosa_tensor_t client_input, tosa_tensor_t client_output); + + tosa_status_t tosa_run_tanh(tosa_tensor_t client_input, tosa_tensor_t client_output); + + tosa_status_t tosa_run_add(tosa_tensor_t client_input1, tosa_tensor_t client_input2, tosa_tensor_t client_output); + + tosa_status_t tosa_run_arithmetic_right_shift(tosa_tensor_t client_input1, + tosa_tensor_t client_input2, + const bool client_round, + tosa_tensor_t client_output); + + tosa_status_t + tosa_run_bitwise_and(tosa_tensor_t client_input1, tosa_tensor_t client_input2, tosa_tensor_t client_output); + + tosa_status_t + tosa_run_bitwise_or(tosa_tensor_t client_input1, tosa_tensor_t client_input2, tosa_tensor_t client_output); + + tosa_status_t + tosa_run_bitwise_xor(tosa_tensor_t client_input1, tosa_tensor_t client_input2, tosa_tensor_t client_output); + + tosa_status_t + tosa_run_intdiv(tosa_tensor_t client_input1, tosa_tensor_t client_input2, tosa_tensor_t client_output); + + tosa_status_t + tosa_run_logical_and(tosa_tensor_t client_input1, tosa_tensor_t client_input2, tosa_tensor_t client_output); + + tosa_status_t tosa_run_logical_left_shift(tosa_tensor_t client_input1, + tosa_tensor_t client_input2, + tosa_tensor_t client_output); + + tosa_status_t tosa_run_logical_right_shift(tosa_tensor_t client_input1, + tosa_tensor_t client_input2, + tosa_tensor_t client_output); + + tosa_status_t + tosa_run_logical_or(tosa_tensor_t client_input1, tosa_tensor_t client_input2, tosa_tensor_t client_output); + + tosa_status_t + tosa_run_logical_xor(tosa_tensor_t client_input1, tosa_tensor_t client_input2, tosa_tensor_t client_output); + + tosa_status_t + tosa_run_maximum(tosa_tensor_t client_input1, tosa_tensor_t client_input2, tosa_tensor_t client_output); + + tosa_status_t + tosa_run_minimum(tosa_tensor_t client_input1, tosa_tensor_t client_input2, tosa_tensor_t client_output); + + tosa_status_t tosa_run_mul(tosa_tensor_t client_input1, + tosa_tensor_t client_input2, + const uint8_t client_shift, + tosa_tensor_t client_output); + + tosa_status_t tosa_run_pow(tosa_tensor_t client_input1, tosa_tensor_t client_input2, tosa_tensor_t client_output); + + tosa_status_t tosa_run_sub(tosa_tensor_t client_input1, tosa_tensor_t client_input2, tosa_tensor_t client_output); + + tosa_status_t tosa_run_table(tosa_tensor_t client_input, + const int32_t client_table_len, + const int16_t client_table[], + tosa_tensor_t client_output); + + tosa_status_t tosa_run_abs(tosa_tensor_t client_input1, tosa_tensor_t client_output); + + tosa_status_t tosa_run_bitwise_not(tosa_tensor_t client_input1, tosa_tensor_t client_output); + + tosa_status_t tosa_run_ceil(tosa_tensor_t client_input1, tosa_tensor_t client_output); + + tosa_status_t tosa_run_clz(tosa_tensor_t client_input1, tosa_tensor_t client_output); + + tosa_status_t tosa_run_exp(tosa_tensor_t client_input1, tosa_tensor_t client_output); + + tosa_status_t tosa_run_floor(tosa_tensor_t client_input1, tosa_tensor_t client_output); + + tosa_status_t tosa_run_log(tosa_tensor_t client_input1, tosa_tensor_t client_output); + + tosa_status_t tosa_run_logical_not(tosa_tensor_t client_input1, tosa_tensor_t client_output); + + tosa_status_t tosa_run_negate(tosa_tensor_t client_input1, + const int32_t client_input1_zp, + const int32_t client_output_zp, + tosa_tensor_t client_output); + + tosa_status_t tosa_run_reciprocal(tosa_tensor_t client_input1, tosa_tensor_t client_output); + + tosa_status_t tosa_run_rsqrt(tosa_tensor_t client_input1, tosa_tensor_t client_output); + + tosa_status_t tosa_run_select(tosa_tensor_t client_input1, + tosa_tensor_t client_input2, + tosa_tensor_t client_input3, + tosa_tensor_t client_output); + + tosa_status_t tosa_run_equal(tosa_tensor_t client_input1, tosa_tensor_t client_input2, tosa_tensor_t client_output); + + tosa_status_t + tosa_run_greater(tosa_tensor_t client_input1, tosa_tensor_t client_input2, tosa_tensor_t client_output); + + tosa_status_t + tosa_run_greater_equal(tosa_tensor_t client_input1, tosa_tensor_t client_input2, tosa_tensor_t client_output); + + tosa_status_t + tosa_run_reduce_all(tosa_tensor_t client_input, const int32_t client_axis, tosa_tensor_t client_output); + + tosa_status_t + tosa_run_reduce_any(tosa_tensor_t client_input, const int32_t client_axis, tosa_tensor_t client_output); + + tosa_status_t + tosa_run_reduce_max(tosa_tensor_t client_input, const int32_t client_axis, tosa_tensor_t client_output); + + tosa_status_t + tosa_run_reduce_min(tosa_tensor_t client_input, const int32_t client_axis, tosa_tensor_t client_output); + + tosa_status_t + tosa_run_reduce_product(tosa_tensor_t client_input, const int32_t client_axis, tosa_tensor_t client_output); + + tosa_status_t + tosa_run_reduce_sum(tosa_tensor_t client_input, const int32_t client_axis, tosa_tensor_t client_output); + + tosa_status_t tosa_run_concat(tosa_tensor_t client_input1, const int32_t client_axis, tosa_tensor_t client_output); + + tosa_status_t tosa_run_pad(tosa_tensor_t client_input1, + const int32_t client_padding_len, + const int32_t client_padding[], + const int32_t client_pad_const_int, + const float client_pad_const_fp, + tosa_tensor_t client_output); + + tosa_status_t tosa_run_reshape(tosa_tensor_t client_input1, + const int32_t client_new_shape_len, + const int32_t client_new_shape[], + tosa_tensor_t client_output); + + tosa_status_t tosa_run_reverse(tosa_tensor_t client_input, const int32_t client_axis, tosa_tensor_t client_output); + + tosa_status_t tosa_run_slice(tosa_tensor_t client_input1, + const int32_t client_start_len, + const int32_t client_start[], + const int32_t client_size_len, + const int32_t client_size[], + tosa_tensor_t client_output); + + tosa_status_t tosa_run_tile(tosa_tensor_t client_input1, + const int32_t client_multiplies_len, + const int32_t client_multiplies[], + const int32_t client_multiples_len, + const int32_t client_multiples[], + tosa_tensor_t client_output); + + tosa_status_t tosa_run_transpose(tosa_tensor_t client_input1, + const int32_t client_perms_len, + const int32_t client_perms[], + tosa_tensor_t client_output); + + tosa_status_t + tosa_run_gather(tosa_tensor_t client_values, tosa_tensor_t client_indices, tosa_tensor_t client_output); + + tosa_status_t tosa_run_scatter(tosa_tensor_t client_values_in, + tosa_tensor_t client_indices, + tosa_tensor_t client_input, + tosa_tensor_t client_values_out); + + tosa_status_t tosa_run_resize(tosa_tensor_t client_input, + const int16_t client_scale[4], + const int16_t client_offset[2], + const int16_t client_border[2], + const tosa_mode_t client_mode, + tosa_tensor_t client_output); + + tosa_status_t tosa_run_cast(tosa_tensor_t client_input, tosa_tensor_t client_output); + + tosa_status_t tosa_run_rescale(tosa_tensor_t client_input, + tosa_tensor_t client_output, + const int32_t client_input_zp, + const int32_t client_output_zp, + const int32_t client_multiplier_len, + const int32_t client_multiplier[], + const int32_t client_shift_len, + const uint8_t client_shift[], + const bool client_scale32, + const bool client_double_round, + const bool client_per_channel); + + tosa_status_t tosa_run_identity(tosa_tensor_t client_input1, tosa_tensor_t client_output); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif // OPERATORS_H_ \ No newline at end of file diff --git a/reference_model/src/array_proxy.h b/reference_model/src/array_proxy.h new file mode 100644 index 0000000..f6b3105 --- /dev/null +++ b/reference_model/src/array_proxy.h @@ -0,0 +1,98 @@ + +// Copyright (c) 2022, ARM Limited. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ARRAY_PROXY_H_ +#define ARRAY_PROXY_H_ + +#include +#include + +template +class ArrayProxy +{ +public: + ArrayProxy(size_t n, T* ptr) noexcept + : _n(n) + , _ptr(ptr) + {} + + template ::value, int> = 0> + ArrayProxy(size_t n, typename std::remove_const_t* ptr) noexcept + : _n(n) + , _ptr(ptr) + {} + + template + ArrayProxy(T (&ptr)[S]) noexcept + : _n(S) + , _ptr(ptr) + {} + + template ::value, int> = 0> + ArrayProxy(typename std::remove_const_t (&ptr)[S]) noexcept + : _n(S) + , _ptr(ptr) + {} + + template ().data()), T*> && + std::is_convertible_v().size()), std::size_t>, + int> = 0> + ArrayProxy(O& obj) noexcept + : _n(obj.size()) + , _ptr(obj.data()) + {} + + size_t size() const noexcept + { + return _n; + } + + T* data() const noexcept + { + return _ptr; + } + + bool empty() const noexcept + { + return _n == 0; + } + + const T* begin() const noexcept + { + return _ptr; + } + + const T* end() const noexcept + { + return _ptr + _n; + } + + T& operator[](size_t idx) noexcept + { + return *(_ptr + idx); + } + + const T& operator[](size_t idx) const noexcept + { + return *(_ptr + idx); + } + +private: + size_t _n; + T* _ptr; +}; + +#endif // ARRAY_PROXY_H_ diff --git a/reference_model/src/model_runner.cc b/reference_model/src/model_runner.cc index 5c086e6..62d6ce6 100644 --- a/reference_model/src/model_runner.cc +++ b/reference_model/src/model_runner.cc @@ -55,7 +55,12 @@ GraphStatus IModelRunner::run() template int IModelRunner::setInput(std::string input_name, std::vector& vals) { - return model_runner_impl->setInput(input_name, vals); + return model_runner_impl->setInput(input_name, ArrayProxy(vals.size(), vals.data())); +} + +int IModelRunner::setInput(std::string input_name, uint8_t* raw_ptr, size_t size) +{ + return model_runner_impl->setInput(input_name, raw_ptr, size); } template @@ -64,6 +69,11 @@ std::vector IModelRunner::getOutput(std::string output_name) return model_runner_impl->getOutput(output_name); } +int IModelRunner::getOutput(std::string output_name, uint8_t* raw_ptr, size_t size) +{ + return model_runner_impl->getOutput(output_name, raw_ptr, size); +} + // Template explicit specialization template int IModelRunner::setInput(std::string input_name, std::vector& vals); template int IModelRunner::setInput(std::string input_name, std::vector& vals); diff --git a/reference_model/src/model_runner_impl.cc b/reference_model/src/model_runner_impl.cc index 8427150..1109dd6 100644 --- a/reference_model/src/model_runner_impl.cc +++ b/reference_model/src/model_runner_impl.cc @@ -45,42 +45,12 @@ void ModelRunnerImpl::setFuncDebug(func_debug_t& func_debug) GraphStatus ModelRunnerImpl::initialize(TosaSerializationHandler& serialization_handler) { validateTosaVersion(serialization_handler); + return initialize(serialization_handler.GetMainBlock(), &serialization_handler); +} - // Make nullptr in case ModelRunnerImpl is being initialized again with a different graph. - _main_gt = nullptr; - _main_gt = new SubgraphTraverser(serialization_handler.GetMainBlock(), &serialization_handler); - - if(_main_gt == nullptr) - { - WARNING("An error occurred when generating main graph traverser."); - return GraphStatus::TOSA_ERROR; - } - - if (_main_gt->initializeGraph()) - { - WARNING("Unable to initialize main graph traverser."); - return _main_gt->getGraphStatus(); - } - - if (_main_gt->linkTensorsAndNodes()) - { - WARNING("Failed to link tensors and nodes"); - return _main_gt->getGraphStatus(); - } - - if (_main_gt->validateGraph()) - { - WARNING("Failed to validate graph."); - return _main_gt->getGraphStatus(); - } - - if (_main_gt->allocateTensor()) - { - WARNING("Failed to allocate tensor."); - return _main_gt->getGraphStatus(); - } - - return _main_gt->getGraphStatus(); +GraphStatus ModelRunnerImpl::initialize(TosaSerializationBasicBlock& bb) +{ + return initialize(&bb, nullptr); } GraphStatus ModelRunnerImpl::run() @@ -156,7 +126,7 @@ done: } template -int ModelRunnerImpl::setInput(std::string input_name, std::vector& vals) +int ModelRunnerImpl::setInput(std::string input_name, ArrayProxy vals) { if (_main_gt == nullptr) { @@ -197,6 +167,44 @@ int ModelRunnerImpl::setInput(std::string input_name, std::vector& vals) return 0; } +int ModelRunnerImpl::setInput(std::string input_name, uint8_t* raw_ptr, size_t size) +{ + if (_main_gt == nullptr) + { + FATAL_ERROR("ModelRunner hasn't been initialized, please invoke initialize() before setInput()"); + } + + Tensor* tensor; + tensor = _main_gt->getInputTensorByName(input_name); + + if (!tensor) + { + WARNING("Unable to find input tensor %s", input_name.c_str()); + return 1; + } + + int status = 0; + switch (tensor->getDtype()) + { + case DType::DType_FP16: { + auto typed_ptr = reinterpret_cast(raw_ptr); + const int elements = size / sizeof(half_float::half); + status = setInput(input_name, ArrayProxy(elements, typed_ptr)); + break; + } + case DType::DType_FP32: { + auto typed_ptr = reinterpret_cast(raw_ptr); + const int elements = size / sizeof(float); + status = setInput(input_name, ArrayProxy(elements, typed_ptr)); + break; + } + default: + status = 1; + } + + return status; +} + template std::vector ModelRunnerImpl::getOutput(std::string output_name) { @@ -216,7 +224,7 @@ std::vector ModelRunnerImpl::getOutput(std::string output_name) std::vector outputs(tensor->getElementCount()); - if (tensor->writeToVector(outputs)) + if (tensor->writeToVector(ArrayProxy(outputs))) { WARNING("Unable to convert output tensor %s to vector", tensor->getName().c_str()); return std::vector(); @@ -225,6 +233,92 @@ std::vector ModelRunnerImpl::getOutput(std::string output_name) return outputs; } +int ModelRunnerImpl::getOutput(std::string output_name, uint8_t* raw_ptr, size_t size) +{ + if (_main_gt == nullptr) + { + FATAL_ERROR("ModelRunner hasn't been initialized, please invoke initialize() and run() before getOutput()"); + } + + Tensor* tensor; + tensor = _main_gt->getOutputTensorByName(output_name); + + if (!tensor) + { + WARNING("Unable to find output tensor %s", output_name.c_str()); + return 1; + } + + int status = 0; + switch (tensor->getDtype()) + { + case DType::DType_FP16: { + auto typed_ptr = reinterpret_cast(raw_ptr); + const int elements = size / sizeof(half_float::half); + status = tensor->writeToVector(ArrayProxy(elements, typed_ptr)); + break; + } + case DType::DType_FP32: { + auto typed_ptr = reinterpret_cast(raw_ptr); + const int elements = size / sizeof(float); + status = tensor->writeToVector(ArrayProxy(elements, typed_ptr)); + break; + } + default: + status = 1; + } + if (status) + { + WARNING("Unable to convert output tensor %s to vector", tensor->getName().c_str()); + return 1; + } + + return 0; +} + +GraphStatus ModelRunnerImpl::initialize(TosaSerializationBasicBlock* bb, + TosaSerializationHandler* serialization_handler) +{ + if (serialization_handler != nullptr) + validateTosaVersion(*serialization_handler); + + // Make nullptr in case ModelRunnerImpl is being initialized again with a different graph. + _main_gt = nullptr; + _main_gt = new SubgraphTraverser(bb, serialization_handler); + + if (_main_gt == nullptr) + { + WARNING("An error occurred when generating main graph traverser."); + return GraphStatus::TOSA_ERROR; + } + + if (_main_gt->initializeGraph()) + { + WARNING("Unable to initialize main graph traverser."); + return _main_gt->getGraphStatus(); + } + + if (_main_gt->linkTensorsAndNodes()) + { + WARNING("Failed to link tensors and nodes"); + return _main_gt->getGraphStatus(); + } + + if (_main_gt->validateGraph()) + { + WARNING("Failed to validate graph."); + return _main_gt->getGraphStatus(); + } + + if (_main_gt->allocateTensor()) + { + WARNING("Failed to allocate tensor."); + return _main_gt->getGraphStatus(); + } + + return _main_gt->getGraphStatus(); +} + void ModelRunnerImpl::validateTosaVersion(TosaSerializationHandler& serialization_handler) { TosaVersion model_version(TOSA_REFERENCE_MODEL_VERSION_MAJOR, @@ -266,11 +360,11 @@ void ModelRunnerImpl::checkGraphStatus(SubgraphTraverser& main_gt) } // Template explicit specialization -template int ModelRunnerImpl::setInput(std::string input_name, std::vector& vals); -template int ModelRunnerImpl::setInput(std::string input_name, std::vector& vals); -template int ModelRunnerImpl::setInput(std::string input_name, std::vector& vals); -template int ModelRunnerImpl::setInput(std::string input_name, std::vector& vals); -template int ModelRunnerImpl::setInput(std::string input_name, std::vector& vals); +template int ModelRunnerImpl::setInput(std::string input_name, ArrayProxy vals); +template int ModelRunnerImpl::setInput(std::string input_name, ArrayProxy vals); +template int ModelRunnerImpl::setInput(std::string input_name, ArrayProxy vals); +template int ModelRunnerImpl::setInput(std::string input_name, ArrayProxy vals); +template int ModelRunnerImpl::setInput(std::string input_name, ArrayProxy vals); template std::vector ModelRunnerImpl::getOutput(std::string output_name); template std::vector ModelRunnerImpl::getOutput(std::string output_name); diff --git a/reference_model/src/model_runner_impl.h b/reference_model/src/model_runner_impl.h index f26c484..b43370c 100644 --- a/reference_model/src/model_runner_impl.h +++ b/reference_model/src/model_runner_impl.h @@ -20,6 +20,7 @@ #include "graph_status.h" #include "version.h" +#include "array_proxy.h" #include "ops/op_factory.h" #include "subgraph_traverser.h" #include "tosa_serialization_handler.h" @@ -42,14 +43,17 @@ public: void setFuncConfig(func_config_t& func_config); void setFuncDebug(func_debug_t& func_debug); + GraphStatus initialize(TosaSerializationBasicBlock& bb); GraphStatus initialize(TosaSerializationHandler& serialization_handler); GraphStatus run(); template - int setInput(std::string input_name, std::vector& vals); + int setInput(std::string input_name, ArrayProxy vals); + int setInput(std::string input_name, uint8_t* raw_ptr, size_t size); template std::vector getOutput(std::string output_name); + int getOutput(std::string output_name, uint8_t* ptr, size_t size); private: SubgraphTraverser* _main_gt = nullptr; @@ -57,6 +61,7 @@ private: // Used to determine if all input tensors have been set correctly. uint32_t n_input_tensors = 0; + GraphStatus initialize(TosaSerializationBasicBlock* bb, TosaSerializationHandler* serialization_handler); void validateTosaVersion(TosaSerializationHandler& serialization_handler); void checkGraphStatus(SubgraphTraverser& main_gt); }; diff --git a/reference_model/src/operators.cc b/reference_model/src/operators.cc new file mode 100644 index 0000000..dfad9b8 --- /dev/null +++ b/reference_model/src/operators.cc @@ -0,0 +1,2325 @@ + +// Copyright (c) 2022, ARM Limited. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// THIS FILE IS GENERATED. DO NOT EDIT! +// See scripts/operator_api/generate_api.py + +#include "operators.h" +#include "model_runner_impl.h" +#include "ops/op_factory.h" + +#define TOSA_RETURN_ON_ERROR(status) \ + do \ + { \ + if (status != 0) \ + { \ + return tosa_status_error; \ + } \ + } while (false) + +#define TOSA_RETURN_ON_GRAPH_STATUS_ERROR(status) \ + do \ + { \ + if (status != GraphStatus::TOSA_VALID) \ + { \ + auto ustatus = static_cast>(status); \ + return static_cast(ustatus); \ + } \ + } while (false) + +namespace +{ + +tosa::DType translate_client_datatype(tosa_datatype_t type) +{ + switch (type) + { + case tosa_datatype_fp16_t: + return tosa::DType::DType_FP16; + case tosa_datatype_fp32_t: + return tosa::DType::DType_FP32; + default: + return tosa::DType::DType_UNKNOWN; + } +}; + +tosa::TosaSerializationTensor* translate_client_tensor(tosa_tensor_t& tensor, const std::string& name) +{ + std::vector shape(tensor.shape, tensor.shape + tensor.num_dims); + return new tosa::TosaSerializationTensor(name, shape, translate_client_datatype(tensor.data_type), {}); +} + +tosa::ResizeMode translate_client_tosa_mode(tosa_mode_t mode) +{ + switch (mode) + { + case tosa_mode_nearest: + return tosa::ResizeMode_NEAREST; + case tosa_mode_max: + case tosa_mode_bilinear: + return tosa::ResizeMode_BILINEAR; + default: + return tosa::ResizeMode_UNKNOWN; + } +} + +} // namespace + +extern "C" +{ + + tosa_status_t tosa_run_argmax(tosa_tensor_t client_input, const int32_t client_axis, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input = translate_client_tensor(client_input, "input"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_ARGMAX, tosa::Attribute::Attribute_NONE, &attr, + { input->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("argmax", { op }, { input, output }, { input->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input->GetName(), client_input.data, client_input.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_avg_pool2d(tosa_tensor_t client_input, + const int32_t client_kernel[2], + const int32_t client_stride[2], + const int32_t client_pad[4], + const int32_t client_input_zp, + const int32_t client_output_zp, + tosa_tensor_t client_output) + { + // Create operator attributes + const std::vector pad(&client_pad[0], &client_pad[4]); + const std::vector kernel(&client_kernel[0], &client_kernel[2]); + const std::vector stride(&client_stride[0], &client_stride[2]); + const int32_t input_zp = client_input_zp; + const int32_t output_zp = client_output_zp; + const tosa::DType accum_dtype = tosa::DType::DType_FP32; + TosaPoolAttribute attr(pad, kernel, stride, input_zp, output_zp, accum_dtype); + + // Create tensors + tosa::TosaSerializationTensor* input = translate_client_tensor(client_input, "input"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_AVG_POOL2D, tosa::Attribute::Attribute_PoolAttribute, + &attr, { input->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("avg_pool2d", { op }, { input, output }, { input->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input->GetName(), client_input.data, client_input.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_conv2d(tosa_tensor_t client_input, + tosa_tensor_t client_weight, + tosa_tensor_t client_bias, + const int32_t client_pad[4], + const int32_t client_stride[2], + const int32_t client_dilation[2], + const int32_t client_input_zp, + const int32_t client_weight_zp, + tosa_tensor_t client_output) + { + // Create operator attributes + const std::vector pad(&client_pad[0], &client_pad[4]); + const std::vector stride(&client_stride[0], &client_stride[2]); + const std::vector dilation(&client_dilation[0], &client_dilation[2]); + const int32_t input_zp = client_input_zp; + const int32_t weight_zp = client_weight_zp; + const tosa::DType accum_dtype = tosa::DType::DType_FP32; + TosaConvAttribute attr(pad, stride, dilation, input_zp, weight_zp, accum_dtype); + + // Create tensors + tosa::TosaSerializationTensor* input = translate_client_tensor(client_input, "input"); + tosa::TosaSerializationTensor* weight = translate_client_tensor(client_weight, "weight"); + tosa::TosaSerializationTensor* bias = translate_client_tensor(client_bias, "bias"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_CONV2D, tosa::Attribute::Attribute_ConvAttribute, + &attr, { input->GetName(), weight->GetName(), bias->GetName() }, + { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("conv2d", { op }, { input, weight, bias, output }, + { input->GetName(), weight->GetName(), bias->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input->GetName(), client_input.data, client_input.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(weight->GetName(), client_weight.data, client_weight.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(bias->GetName(), client_bias.data, client_bias.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_conv3d(tosa_tensor_t client_input, + tosa_tensor_t client_weight, + tosa_tensor_t client_bias, + const int32_t client_pad[6], + const int32_t client_stride[3], + const int32_t client_dilation[3], + const int32_t client_input_zp, + const int32_t client_weight_zp, + tosa_tensor_t client_output) + { + // Create operator attributes + const std::vector pad(&client_pad[0], &client_pad[6]); + const std::vector stride(&client_stride[0], &client_stride[3]); + const std::vector dilation(&client_dilation[0], &client_dilation[3]); + const int32_t input_zp = client_input_zp; + const int32_t weight_zp = client_weight_zp; + const tosa::DType accum_dtype = tosa::DType::DType_FP32; + TosaConvAttribute attr(pad, stride, dilation, input_zp, weight_zp, accum_dtype); + + // Create tensors + tosa::TosaSerializationTensor* input = translate_client_tensor(client_input, "input"); + tosa::TosaSerializationTensor* weight = translate_client_tensor(client_weight, "weight"); + tosa::TosaSerializationTensor* bias = translate_client_tensor(client_bias, "bias"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_CONV3D, tosa::Attribute::Attribute_ConvAttribute, + &attr, { input->GetName(), weight->GetName(), bias->GetName() }, + { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("conv3d", { op }, { input, weight, bias, output }, + { input->GetName(), weight->GetName(), bias->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input->GetName(), client_input.data, client_input.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(weight->GetName(), client_weight.data, client_weight.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(bias->GetName(), client_bias.data, client_bias.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_depthwise_conv2d(tosa_tensor_t client_input, + tosa_tensor_t client_weight, + tosa_tensor_t client_bias, + const int32_t client_pad[4], + const int32_t client_stride[2], + const int32_t client_dilation[2], + const int32_t client_input_zp, + const int32_t client_weight_zp, + tosa_tensor_t client_output) + { + // Create operator attributes + const std::vector pad(&client_pad[0], &client_pad[4]); + const std::vector stride(&client_stride[0], &client_stride[2]); + const std::vector dilation(&client_dilation[0], &client_dilation[2]); + const int32_t input_zp = client_input_zp; + const int32_t weight_zp = client_weight_zp; + const tosa::DType accum_dtype = tosa::DType::DType_FP32; + TosaConvAttribute attr(pad, stride, dilation, input_zp, weight_zp, accum_dtype); + + // Create tensors + tosa::TosaSerializationTensor* input = translate_client_tensor(client_input, "input"); + tosa::TosaSerializationTensor* weight = translate_client_tensor(client_weight, "weight"); + tosa::TosaSerializationTensor* bias = translate_client_tensor(client_bias, "bias"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator( + tosa::Op::Op_DEPTHWISE_CONV2D, tosa::Attribute::Attribute_ConvAttribute, &attr, + { input->GetName(), weight->GetName(), bias->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("depthwise_conv2d", { op }, { input, weight, bias, output }, + { input->GetName(), weight->GetName(), bias->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input->GetName(), client_input.data, client_input.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(weight->GetName(), client_weight.data, client_weight.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(bias->GetName(), client_bias.data, client_bias.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_fully_connected(tosa_tensor_t client_input, + const int32_t client_input_zp, + const int32_t client_weight_zp, + tosa_tensor_t client_output) + { + // Create operator attributes + const int32_t input_zp = client_input_zp; + const int32_t weight_zp = client_weight_zp; + const tosa::DType accum_dtype = tosa::DType::DType_FP32; + TosaFullyConnectedAttribute attr(input_zp, weight_zp, accum_dtype); + + // Create tensors + tosa::TosaSerializationTensor* input = translate_client_tensor(client_input, "input"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_FULLY_CONNECTED, + tosa::Attribute::Attribute_FullyConnectedAttribute, &attr, + { input->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("fully_connected", { op }, { input, output }, { input->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input->GetName(), client_input.data, client_input.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_matmul(tosa_tensor_t client_a, + tosa_tensor_t client_b, + const int32_t client_a_zp, + const int32_t client_b_zp, + tosa_tensor_t client_output) + { + // Create operator attributes + const int32_t a_zp = client_a_zp; + const int32_t b_zp = client_b_zp; + const tosa::DType accum_dtype = tosa::DType::DType_FP32; + TosaMatMulAttribute attr(a_zp, b_zp, accum_dtype); + + // Create tensors + tosa::TosaSerializationTensor* a = translate_client_tensor(client_a, "a"); + tosa::TosaSerializationTensor* b = translate_client_tensor(client_b, "b"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_MATMUL, tosa::Attribute::Attribute_MatMulAttribute, + &attr, { a->GetName(), b->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("matmul", { op }, { a, b, output }, { a->GetName(), b->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(a->GetName(), client_a.data, client_a.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(b->GetName(), client_b.data, client_b.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_max_pool2d(tosa_tensor_t client_input, + const int32_t client_kernel[2], + const int32_t client_stride[2], + const int32_t client_pad[4], + const int32_t client_input_zp, + const int32_t client_output_zp, + tosa_tensor_t client_output) + { + // Create operator attributes + const std::vector pad(&client_pad[0], &client_pad[4]); + const std::vector kernel(&client_kernel[0], &client_kernel[2]); + const std::vector stride(&client_stride[0], &client_stride[2]); + const int32_t input_zp = client_input_zp; + const int32_t output_zp = client_output_zp; + const tosa::DType accum_dtype = tosa::DType::DType_FP32; + TosaPoolAttribute attr(pad, kernel, stride, input_zp, output_zp, accum_dtype); + + // Create tensors + tosa::TosaSerializationTensor* input = translate_client_tensor(client_input, "input"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_MAX_POOL2D, tosa::Attribute::Attribute_PoolAttribute, + &attr, { input->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("max_pool2d", { op }, { input, output }, { input->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input->GetName(), client_input.data, client_input.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_transpose_conv2d(tosa_tensor_t client_input, + tosa_tensor_t client_weight, + tosa_tensor_t client_bias, + const int32_t client_out_pad[4], + const int32_t client_stride[2], + const int32_t client_out_shape[4], + const int32_t client_input_zp, + const int32_t client_weight_zp, + const int32_t client_pad_len, + const int32_t client_pad[], + const int32_t client_dilation_len, + const int32_t client_dilation[], + tosa_tensor_t client_output) + { + // Create operator attributes + const std::vector pad(&client_pad[0], &client_pad[0] + client_pad_len); + const std::vector stride(&client_stride[0], &client_stride[2]); + const std::vector dilation(&client_dilation[0], &client_dilation[0] + client_dilation_len); + const int32_t input_zp = client_input_zp; + const int32_t weight_zp = client_weight_zp; + const tosa::DType accum_dtype = tosa::DType::DType_FP32; + TosaConvAttribute attr(pad, stride, dilation, input_zp, weight_zp, accum_dtype); + + // Create tensors + tosa::TosaSerializationTensor* input = translate_client_tensor(client_input, "input"); + tosa::TosaSerializationTensor* weight = translate_client_tensor(client_weight, "weight"); + tosa::TosaSerializationTensor* bias = translate_client_tensor(client_bias, "bias"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator( + tosa::Op::Op_TRANSPOSE_CONV2D, tosa::Attribute::Attribute_ConvAttribute, &attr, + { input->GetName(), weight->GetName(), bias->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("transpose_conv2d", { op }, { input, weight, bias, output }, + { input->GetName(), weight->GetName(), bias->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input->GetName(), client_input.data, client_input.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(weight->GetName(), client_weight.data, client_weight.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(bias->GetName(), client_bias.data, client_bias.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_clamp(tosa_tensor_t client_input, + const int32_t client_min_int, + const int32_t client_max_int, + const float client_min_fp, + const float client_max_fp, + tosa_tensor_t client_output) + { + // Create operator attributes + const int32_t min_int = client_min_int; + const int32_t max_int = client_max_int; + const float min_fp = client_min_fp; + const float max_fp = client_max_fp; + TosaClampAttribute attr(min_int, max_int, min_fp, max_fp); + + // Create tensors + tosa::TosaSerializationTensor* input = translate_client_tensor(client_input, "input"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_CLAMP, tosa::Attribute::Attribute_ClampAttribute, + &attr, { input->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("clamp", { op }, { input, output }, { input->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input->GetName(), client_input.data, client_input.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_sigmoid(tosa_tensor_t client_input, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input = translate_client_tensor(client_input, "input"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_SIGMOID, tosa::Attribute::Attribute_NONE, &attr, + { input->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("sigmoid", { op }, { input, output }, { input->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input->GetName(), client_input.data, client_input.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_tanh(tosa_tensor_t client_input, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input = translate_client_tensor(client_input, "input"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_TANH, tosa::Attribute::Attribute_NONE, &attr, + { input->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("tanh", { op }, { input, output }, { input->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input->GetName(), client_input.data, client_input.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_add(tosa_tensor_t client_input1, tosa_tensor_t client_input2, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* input2 = translate_client_tensor(client_input2, "input2"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_ADD, tosa::Attribute::Attribute_NONE, &attr, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("add", { op }, { input1, input2, output }, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(input2->GetName(), client_input2.data, client_input2.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_arithmetic_right_shift(tosa_tensor_t client_input1, + tosa_tensor_t client_input2, + const bool client_round, + tosa_tensor_t client_output) + { + // Create operator attributes + const bool round = client_round; + TosaArithmeticRightShiftAttribute attr(round); + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* input2 = translate_client_tensor(client_input2, "input2"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_ARITHMETIC_RIGHT_SHIFT, + tosa::Attribute::Attribute_ArithmeticRightShiftAttribute, &attr, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("arithmetic_right_shift", { op }, { input1, input2, output }, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(input2->GetName(), client_input2.data, client_input2.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t + tosa_run_bitwise_and(tosa_tensor_t client_input1, tosa_tensor_t client_input2, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* input2 = translate_client_tensor(client_input2, "input2"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_BITWISE_AND, tosa::Attribute::Attribute_NONE, &attr, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("bitwise_and", { op }, { input1, input2, output }, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(input2->GetName(), client_input2.data, client_input2.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t + tosa_run_bitwise_or(tosa_tensor_t client_input1, tosa_tensor_t client_input2, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* input2 = translate_client_tensor(client_input2, "input2"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_BITWISE_OR, tosa::Attribute::Attribute_NONE, &attr, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("bitwise_or", { op }, { input1, input2, output }, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(input2->GetName(), client_input2.data, client_input2.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t + tosa_run_bitwise_xor(tosa_tensor_t client_input1, tosa_tensor_t client_input2, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* input2 = translate_client_tensor(client_input2, "input2"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_BITWISE_XOR, tosa::Attribute::Attribute_NONE, &attr, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("bitwise_xor", { op }, { input1, input2, output }, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(input2->GetName(), client_input2.data, client_input2.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_intdiv(tosa_tensor_t client_input1, tosa_tensor_t client_input2, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* input2 = translate_client_tensor(client_input2, "input2"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_INTDIV, tosa::Attribute::Attribute_NONE, &attr, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("intdiv", { op }, { input1, input2, output }, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(input2->GetName(), client_input2.data, client_input2.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t + tosa_run_logical_and(tosa_tensor_t client_input1, tosa_tensor_t client_input2, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* input2 = translate_client_tensor(client_input2, "input2"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_LOGICAL_AND, tosa::Attribute::Attribute_NONE, &attr, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("logical_and", { op }, { input1, input2, output }, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(input2->GetName(), client_input2.data, client_input2.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_logical_left_shift(tosa_tensor_t client_input1, + tosa_tensor_t client_input2, + tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* input2 = translate_client_tensor(client_input2, "input2"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = + new tosa::TosaSerializationOperator(tosa::Op::Op_LOGICAL_LEFT_SHIFT, tosa::Attribute::Attribute_NONE, &attr, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("logical_left_shift", { op }, { input1, input2, output }, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(input2->GetName(), client_input2.data, client_input2.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_logical_right_shift(tosa_tensor_t client_input1, + tosa_tensor_t client_input2, + tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* input2 = translate_client_tensor(client_input2, "input2"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = + new tosa::TosaSerializationOperator(tosa::Op::Op_LOGICAL_RIGHT_SHIFT, tosa::Attribute::Attribute_NONE, + &attr, { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("logical_right_shift", { op }, { input1, input2, output }, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(input2->GetName(), client_input2.data, client_input2.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t + tosa_run_logical_or(tosa_tensor_t client_input1, tosa_tensor_t client_input2, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* input2 = translate_client_tensor(client_input2, "input2"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_LOGICAL_OR, tosa::Attribute::Attribute_NONE, &attr, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("logical_or", { op }, { input1, input2, output }, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(input2->GetName(), client_input2.data, client_input2.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t + tosa_run_logical_xor(tosa_tensor_t client_input1, tosa_tensor_t client_input2, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* input2 = translate_client_tensor(client_input2, "input2"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_LOGICAL_XOR, tosa::Attribute::Attribute_NONE, &attr, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("logical_xor", { op }, { input1, input2, output }, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(input2->GetName(), client_input2.data, client_input2.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t + tosa_run_maximum(tosa_tensor_t client_input1, tosa_tensor_t client_input2, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* input2 = translate_client_tensor(client_input2, "input2"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_MAXIMUM, tosa::Attribute::Attribute_NONE, &attr, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("maximum", { op }, { input1, input2, output }, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(input2->GetName(), client_input2.data, client_input2.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t + tosa_run_minimum(tosa_tensor_t client_input1, tosa_tensor_t client_input2, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* input2 = translate_client_tensor(client_input2, "input2"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_MINIMUM, tosa::Attribute::Attribute_NONE, &attr, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("minimum", { op }, { input1, input2, output }, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(input2->GetName(), client_input2.data, client_input2.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_mul(tosa_tensor_t client_input1, + tosa_tensor_t client_input2, + const uint8_t client_shift, + tosa_tensor_t client_output) + { + // Create operator attributes + const int32_t shift = client_shift; + TosaMulAttribute attr(shift); + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* input2 = translate_client_tensor(client_input2, "input2"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_MUL, tosa::Attribute::Attribute_MulAttribute, &attr, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("mul", { op }, { input1, input2, output }, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(input2->GetName(), client_input2.data, client_input2.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_pow(tosa_tensor_t client_input1, tosa_tensor_t client_input2, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* input2 = translate_client_tensor(client_input2, "input2"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_POW, tosa::Attribute::Attribute_NONE, &attr, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("pow", { op }, { input1, input2, output }, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(input2->GetName(), client_input2.data, client_input2.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_sub(tosa_tensor_t client_input1, tosa_tensor_t client_input2, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* input2 = translate_client_tensor(client_input2, "input2"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_SUB, tosa::Attribute::Attribute_NONE, &attr, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("sub", { op }, { input1, input2, output }, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(input2->GetName(), client_input2.data, client_input2.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_table(tosa_tensor_t client_input, + const int32_t client_table_len, + const int16_t client_table[], + tosa_tensor_t client_output) + { + // Create operator attributes + const std::vector table(&client_table[0], &client_table[0] + client_table_len); + TosaTableAttribute attr(table); + + // Create tensors + tosa::TosaSerializationTensor* input = translate_client_tensor(client_input, "input"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_TABLE, tosa::Attribute::Attribute_TableAttribute, + &attr, { input->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("table", { op }, { input, output }, { input->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input->GetName(), client_input.data, client_input.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_abs(tosa_tensor_t client_input1, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_ABS, tosa::Attribute::Attribute_NONE, &attr, + { input1->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("abs", { op }, { input1, output }, { input1->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_bitwise_not(tosa_tensor_t client_input1, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_BITWISE_NOT, tosa::Attribute::Attribute_NONE, &attr, + { input1->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("bitwise_not", { op }, { input1, output }, { input1->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_ceil(tosa_tensor_t client_input1, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_CEIL, tosa::Attribute::Attribute_NONE, &attr, + { input1->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("ceil", { op }, { input1, output }, { input1->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_clz(tosa_tensor_t client_input1, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_CLZ, tosa::Attribute::Attribute_NONE, &attr, + { input1->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("clz", { op }, { input1, output }, { input1->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_exp(tosa_tensor_t client_input1, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_EXP, tosa::Attribute::Attribute_NONE, &attr, + { input1->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("exp", { op }, { input1, output }, { input1->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_floor(tosa_tensor_t client_input1, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_FLOOR, tosa::Attribute::Attribute_NONE, &attr, + { input1->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("floor", { op }, { input1, output }, { input1->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_log(tosa_tensor_t client_input1, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_LOG, tosa::Attribute::Attribute_NONE, &attr, + { input1->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("log", { op }, { input1, output }, { input1->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_logical_not(tosa_tensor_t client_input1, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_LOGICAL_NOT, tosa::Attribute::Attribute_NONE, &attr, + { input1->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("logical_not", { op }, { input1, output }, { input1->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_negate(tosa_tensor_t client_input1, + const int32_t client_input1_zp, + const int32_t client_output_zp, + tosa_tensor_t client_output) + { + // Create operator attributes + const int32_t input1_zp = client_input1_zp; + const int32_t output_zp = client_output_zp; + TosaNegateAttribute attr(input1_zp, output_zp); + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_NEGATE, tosa::Attribute::Attribute_NegateAttribute, + &attr, { input1->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("negate", { op }, { input1, output }, { input1->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_reciprocal(tosa_tensor_t client_input1, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_RECIPROCAL, tosa::Attribute::Attribute_NONE, &attr, + { input1->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("reciprocal", { op }, { input1, output }, { input1->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_rsqrt(tosa_tensor_t client_input1, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_RSQRT, tosa::Attribute::Attribute_NONE, &attr, + { input1->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("rsqrt", { op }, { input1, output }, { input1->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_select(tosa_tensor_t client_input1, + tosa_tensor_t client_input2, + tosa_tensor_t client_input3, + tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* input2 = translate_client_tensor(client_input2, "input2"); + tosa::TosaSerializationTensor* input3 = translate_client_tensor(client_input3, "input3"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_SELECT, tosa::Attribute::Attribute_NONE, &attr, + { input1->GetName(), input2->GetName(), input3->GetName() }, + { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("select", { op }, { input1, input2, input3, output }, + { input1->GetName(), input2->GetName(), input3->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(input2->GetName(), client_input2.data, client_input2.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(input3->GetName(), client_input3.data, client_input3.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_equal(tosa_tensor_t client_input1, tosa_tensor_t client_input2, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* input2 = translate_client_tensor(client_input2, "input2"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_EQUAL, tosa::Attribute::Attribute_NONE, &attr, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("equal", { op }, { input1, input2, output }, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(input2->GetName(), client_input2.data, client_input2.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t + tosa_run_greater(tosa_tensor_t client_input1, tosa_tensor_t client_input2, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* input2 = translate_client_tensor(client_input2, "input2"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_GREATER, tosa::Attribute::Attribute_NONE, &attr, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("greater", { op }, { input1, input2, output }, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(input2->GetName(), client_input2.data, client_input2.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t + tosa_run_greater_equal(tosa_tensor_t client_input1, tosa_tensor_t client_input2, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* input2 = translate_client_tensor(client_input2, "input2"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = + new tosa::TosaSerializationOperator(tosa::Op::Op_GREATER_EQUAL, tosa::Attribute::Attribute_NONE, &attr, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("greater_equal", { op }, { input1, input2, output }, + { input1->GetName(), input2->GetName() }, { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(input2->GetName(), client_input2.data, client_input2.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t + tosa_run_reduce_all(tosa_tensor_t client_input, const int32_t client_axis, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input = translate_client_tensor(client_input, "input"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_REDUCE_ALL, tosa::Attribute::Attribute_NONE, &attr, + { input->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("reduce_all", { op }, { input, output }, { input->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input->GetName(), client_input.data, client_input.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t + tosa_run_reduce_any(tosa_tensor_t client_input, const int32_t client_axis, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input = translate_client_tensor(client_input, "input"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_REDUCE_ANY, tosa::Attribute::Attribute_NONE, &attr, + { input->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("reduce_any", { op }, { input, output }, { input->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input->GetName(), client_input.data, client_input.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t + tosa_run_reduce_max(tosa_tensor_t client_input, const int32_t client_axis, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input = translate_client_tensor(client_input, "input"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_REDUCE_MAX, tosa::Attribute::Attribute_NONE, &attr, + { input->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("reduce_max", { op }, { input, output }, { input->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input->GetName(), client_input.data, client_input.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t + tosa_run_reduce_min(tosa_tensor_t client_input, const int32_t client_axis, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input = translate_client_tensor(client_input, "input"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_REDUCE_MIN, tosa::Attribute::Attribute_NONE, &attr, + { input->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("reduce_min", { op }, { input, output }, { input->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input->GetName(), client_input.data, client_input.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t + tosa_run_reduce_product(tosa_tensor_t client_input, const int32_t client_axis, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input = translate_client_tensor(client_input, "input"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_REDUCE_PRODUCT, tosa::Attribute::Attribute_NONE, + &attr, { input->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("reduce_product", { op }, { input, output }, { input->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input->GetName(), client_input.data, client_input.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t + tosa_run_reduce_sum(tosa_tensor_t client_input, const int32_t client_axis, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input = translate_client_tensor(client_input, "input"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_REDUCE_SUM, tosa::Attribute::Attribute_NONE, &attr, + { input->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("reduce_sum", { op }, { input, output }, { input->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input->GetName(), client_input.data, client_input.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_concat(tosa_tensor_t client_input1, const int32_t client_axis, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_CONCAT, tosa::Attribute::Attribute_NONE, &attr, + { input1->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("concat", { op }, { input1, output }, { input1->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_pad(tosa_tensor_t client_input1, + const int32_t client_padding_len, + const int32_t client_padding[], + const int32_t client_pad_const_int, + const float client_pad_const_fp, + tosa_tensor_t client_output) + { + // Create operator attributes + const std::vector padding(&client_padding[0], &client_padding[0] + client_padding_len); + const int32_t pad_const_int = client_pad_const_int; + const float pad_const_fp = client_pad_const_fp; + TosaPadAttribute attr(padding, pad_const_int, pad_const_fp); + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_PAD, tosa::Attribute::Attribute_PadAttribute, &attr, + { input1->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("pad", { op }, { input1, output }, { input1->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_reshape(tosa_tensor_t client_input1, + const int32_t client_new_shape_len, + const int32_t client_new_shape[], + tosa_tensor_t client_output) + { + // Create operator attributes + const std::vector new_shape(&client_new_shape[0], &client_new_shape[0] + client_new_shape_len); + TosaReshapeAttribute attr(new_shape); + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_RESHAPE, tosa::Attribute::Attribute_ReshapeAttribute, + &attr, { input1->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("reshape", { op }, { input1, output }, { input1->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_reverse(tosa_tensor_t client_input, const int32_t client_axis, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input = translate_client_tensor(client_input, "input"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_REVERSE, tosa::Attribute::Attribute_NONE, &attr, + { input->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("reverse", { op }, { input, output }, { input->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input->GetName(), client_input.data, client_input.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_slice(tosa_tensor_t client_input1, + const int32_t client_start_len, + const int32_t client_start[], + const int32_t client_size_len, + const int32_t client_size[], + tosa_tensor_t client_output) + { + // Create operator attributes + const std::vector start(&client_start[0], &client_start[0] + client_start_len); + const std::vector size(&client_size[0], &client_size[0] + client_size_len); + TosaSliceAttribute attr(start, size); + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_SLICE, tosa::Attribute::Attribute_SliceAttribute, + &attr, { input1->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("slice", { op }, { input1, output }, { input1->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_tile(tosa_tensor_t client_input1, + const int32_t client_multiplies_len, + const int32_t client_multiplies[], + const int32_t client_multiples_len, + const int32_t client_multiples[], + tosa_tensor_t client_output) + { + // Create operator attributes + const std::vector multiples(&client_multiples[0], &client_multiples[0] + client_multiples_len); + TosaTileAttribute attr(multiples); + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_TILE, tosa::Attribute::Attribute_TileAttribute, + &attr, { input1->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("tile", { op }, { input1, output }, { input1->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_transpose(tosa_tensor_t client_input1, + const int32_t client_perms_len, + const int32_t client_perms[], + tosa_tensor_t client_output) + { + // Create operator attributes + const std::vector perms(&client_perms[0], &client_perms[0] + client_perms_len); + TosaTransposeAttribute attr(perms); + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = + new tosa::TosaSerializationOperator(tosa::Op::Op_TRANSPOSE, tosa::Attribute::Attribute_TransposeAttribute, + &attr, { input1->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("transpose", { op }, { input1, output }, { input1->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t + tosa_run_gather(tosa_tensor_t client_values, tosa_tensor_t client_indices, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* values = translate_client_tensor(client_values, "values"); + tosa::TosaSerializationTensor* indices = translate_client_tensor(client_indices, "indices"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_GATHER, tosa::Attribute::Attribute_NONE, &attr, + { values->GetName(), indices->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("gather", { op }, { values, indices, output }, + { values->GetName(), indices->GetName() }, { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(values->GetName(), client_values.data, client_values.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(indices->GetName(), client_indices.data, client_indices.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_scatter(tosa_tensor_t client_values_in, + tosa_tensor_t client_indices, + tosa_tensor_t client_input, + tosa_tensor_t client_values_out) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* values_in = translate_client_tensor(client_values_in, "values_in"); + tosa::TosaSerializationTensor* indices = translate_client_tensor(client_indices, "indices"); + tosa::TosaSerializationTensor* input = translate_client_tensor(client_input, "input"); + tosa::TosaSerializationTensor* values_out = translate_client_tensor(client_values_out, "values_out"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_SCATTER, tosa::Attribute::Attribute_NONE, &attr, + { values_in->GetName(), indices->GetName(), input->GetName() }, + { values_out->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("scatter", { op }, { values_in, indices, input, values_out }, + { values_in->GetName(), indices->GetName(), input->GetName() }, + { values_out->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(values_in->GetName(), client_values_in.data, client_values_in.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(indices->GetName(), client_indices.data, client_indices.size)); + TOSA_RETURN_ON_ERROR(runner.setInput(input->GetName(), client_input.data, client_input.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(values_out->GetName(), client_values_out.data, client_values_out.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_resize(tosa_tensor_t client_input, + const int16_t client_scale[4], + const int16_t client_offset[2], + const int16_t client_border[2], + const tosa_mode_t client_mode, + tosa_tensor_t client_output) + { + // Create operator attributes + const std::vector scale(&client_scale[0], &client_scale[4]); + const std::vector offset(&client_offset[0], &client_offset[2]); + const std::vector border(&client_border[0], &client_border[2]); + const ResizeMode mode = translate_client_tosa_mode(client_mode); + TosaResizeAttribute attr(scale, offset, border, mode); + + // Create tensors + tosa::TosaSerializationTensor* input = translate_client_tensor(client_input, "input"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_RESIZE, tosa::Attribute::Attribute_ResizeAttribute, + &attr, { input->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("resize", { op }, { input, output }, { input->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input->GetName(), client_input.data, client_input.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_cast(tosa_tensor_t client_input, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input = translate_client_tensor(client_input, "input"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_CAST, tosa::Attribute::Attribute_NONE, &attr, + { input->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("cast", { op }, { input, output }, { input->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input->GetName(), client_input.data, client_input.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_rescale(tosa_tensor_t client_input, + tosa_tensor_t client_output, + const int32_t client_input_zp, + const int32_t client_output_zp, + const int32_t client_multiplier_len, + const int32_t client_multiplier[], + const int32_t client_shift_len, + const uint8_t client_shift[], + const bool client_scale32, + const bool client_double_round, + const bool client_per_channel) + { + // Create operator attributes + const int32_t input_zp = client_input_zp; + const int32_t output_zp = client_output_zp; + const std::vector multiplier(&client_multiplier[0], &client_multiplier[0] + client_multiplier_len); + const std::vector shift(&client_shift[0], &client_shift[0] + client_shift_len); + const bool scale32 = client_scale32; + const bool double_round = client_double_round; + const bool per_channel = client_per_channel; + TosaRescaleAttribute attr(input_zp, output_zp, multiplier, shift, scale32, double_round, per_channel); + + // Create tensors + tosa::TosaSerializationTensor* input = translate_client_tensor(client_input, "input"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_RESCALE, tosa::Attribute::Attribute_RescaleAttribute, + &attr, { input->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("rescale", { op }, { input, output }, { input->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input->GetName(), client_input.data, client_input.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + + tosa_status_t tosa_run_identity(tosa_tensor_t client_input1, tosa_tensor_t client_output) + { + // Create operator attributes + TosaNoneAttribute attr; + + // Create tensors + tosa::TosaSerializationTensor* input1 = translate_client_tensor(client_input1, "input1"); + tosa::TosaSerializationTensor* output = translate_client_tensor(client_output, "output"); + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_IDENTITY, tosa::Attribute::Attribute_NONE, &attr, + { input1->GetName() }, { output->GetName() }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("identity", { op }, { input1, output }, { input1->GetName() }, + { output->GetName() }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + TOSA_RETURN_ON_ERROR(runner.setInput(input1->GetName(), client_input1.data, client_input1.size)); + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + TOSA_RETURN_ON_ERROR(runner.getOutput(output->GetName(), client_output.data, client_output.size)); + + return tosa_status_valid; + } + +} // extern "C" \ No newline at end of file diff --git a/reference_model/src/tensor.cc b/reference_model/src/tensor.cc index e9598c4..3cf4aa0 100644 --- a/reference_model/src/tensor.cc +++ b/reference_model/src/tensor.cc @@ -15,6 +15,7 @@ #include "tensor.h" #include "arith_util.h" +#include "array_proxy.h" #include "half.hpp" using namespace TosaReference; @@ -445,7 +446,7 @@ DEF_CTENSOR_COPY_VALUE_FROM(6, bool) #undef DEF_CTENSOR_COPY_VALUE_FROM -int TosaReference::Tensor::readfromVector(const std::vector& vals) +int TosaReference::Tensor::readfromVector(const ArrayProxy vals) { uint32_t elements = getElementCount(); switch (getDtype()) @@ -470,7 +471,7 @@ int TosaReference::Tensor::readfromVector(const std::vector& vals) return 0; } -int TosaReference::Tensor::readfromVector(const std::vector& vals) +int TosaReference::Tensor::readfromVector(const ArrayProxy vals) { uint32_t elements = getElementCount(); std::vector tensor(elements); @@ -502,7 +503,7 @@ int TosaReference::Tensor::readfromVector(const std::vector& v return 0; } -int TosaReference::Tensor::readfromVector(const std::vector& vals) +int TosaReference::Tensor::readfromVector(const ArrayProxy vals) { uint32_t elements = getElementCount(); switch (getDtype()) @@ -531,7 +532,7 @@ int TosaReference::Tensor::readfromVector(const std::vector& vals) return 0; } -int TosaReference::Tensor::readfromVector(const std::vector& vals) +int TosaReference::Tensor::readfromVector(const ArrayProxy vals) { uint32_t elements = getElementCount(); switch (getDtype()) @@ -555,7 +556,7 @@ int TosaReference::Tensor::readfromVector(const std::vector& vals) return 0; } -int TosaReference::Tensor::readfromVector(const std::vector& vals) +int TosaReference::Tensor::readfromVector(const ArrayProxy vals) { uint32_t elements = getElementCount(); @@ -580,7 +581,7 @@ int TosaReference::Tensor::readfromVector(const std::vector& vals return 0; } -int TosaReference::Tensor::writeToVector(std::vector& vals) +int TosaReference::Tensor::writeToVector(ArrayProxy vals) { uint32_t elements = getElementCount(); @@ -605,7 +606,7 @@ int TosaReference::Tensor::writeToVector(std::vector& vals) return 0; } -int TosaReference::Tensor::writeToVector(std::vector& vals) +int TosaReference::Tensor::writeToVector(ArrayProxy vals) { uint32_t elements = getElementCount(); std::vector tensor(elements); @@ -636,7 +637,7 @@ int TosaReference::Tensor::writeToVector(std::vector& vals) return 0; } -int TosaReference::Tensor::writeToVector(std::vector& vals) +int TosaReference::Tensor::writeToVector(ArrayProxy vals) { uint32_t elements = getElementCount(); @@ -665,7 +666,7 @@ int TosaReference::Tensor::writeToVector(std::vector& vals) return 0; } -int TosaReference::Tensor::writeToVector(std::vector& vals) +int TosaReference::Tensor::writeToVector(ArrayProxy vals) { uint32_t elements = getElementCount(); @@ -689,7 +690,7 @@ int TosaReference::Tensor::writeToVector(std::vector& vals) return 0; } -int TosaReference::Tensor::writeToVector(std::vector& vals) +int TosaReference::Tensor::writeToVector(ArrayProxy vals) { uint32_t elements = getElementCount(); diff --git a/reference_model/src/tensor.h b/reference_model/src/tensor.h index a3ce4bb..08e865a 100644 --- a/reference_model/src/tensor.h +++ b/reference_model/src/tensor.h @@ -16,6 +16,7 @@ #ifndef TOSA_REFERENCE_TENSOR_H #define TOSA_REFERENCE_TENSOR_H +#include "array_proxy.h" #include "model_common.h" #include "ops/template_types.h" #include "tosa_generated.h" @@ -228,17 +229,17 @@ public: virtual int writeToNpyFile(const char* filename) const; virtual int copyValueFrom(Tensor* tensor) = 0; - virtual int readfromVector(const std::vector& vals); - virtual int readfromVector(const std::vector& vals); - virtual int readfromVector(const std::vector& vals); - virtual int readfromVector(const std::vector& vals); - virtual int readfromVector(const std::vector& vals); - - virtual int writeToVector(std::vector& vals); - virtual int writeToVector(std::vector& vals); - virtual int writeToVector(std::vector& vals); - virtual int writeToVector(std::vector& vals); - virtual int writeToVector(std::vector& vals); + virtual int readfromVector(const ArrayProxy vals); + virtual int readfromVector(const ArrayProxy vals); + virtual int readfromVector(const ArrayProxy vals); + virtual int readfromVector(const ArrayProxy vals); + virtual int readfromVector(const ArrayProxy vals); + + virtual int writeToVector(ArrayProxy vals); + virtual int writeToVector(ArrayProxy vals); + virtual int writeToVector(ArrayProxy vals); + virtual int writeToVector(ArrayProxy vals); + virtual int writeToVector(ArrayProxy vals); const char* bool_to_str(bool in) const { diff --git a/reference_model/test/model_runner_tests.cpp b/reference_model/test/model_runner_tests.cpp index 8304bc7..bb57657 100644 --- a/reference_model/test/model_runner_tests.cpp +++ b/reference_model/test/model_runner_tests.cpp @@ -17,8 +17,11 @@ #define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN #endif -#include "model_runner.h" #include "general_utils.h" +#include "model_runner.h" +#include "operators.h" + +#include // Remove conflicting REQUIRE definition between doctest and reference_model #undef REQUIRE @@ -33,125 +36,343 @@ void compareOutput(std::vector& tensor1, std::vector& tensor2, size_t size { for (size_t i = 0; i < size; ++i) { - CHECK((tensor1[i] == tensor2[i])); + CHECK_MESSAGE(tensor1[i] == doctest::Approx(tensor2[i]), ""); } } TEST_SUITE("model_runner") { -TEST_CASE("simple_add_f32_test") -{ - std::string test_root(std::string(PROJECT_ROOT) + "../examples/test_add_1x4x4x4_f32/"); - std::string tosa_model_file(test_root + "flatbuffer-tflite/test_add_1x4x4x4_f32.tosa"); - std::string input0_file(test_root + "placeholder_0.npy"); - std::string input1_file(test_root + "placeholder_1.npy"); - std::string expected_output_file(test_root + "tflite_result.npy"); + TEST_CASE("op_entry_add") + { + // Inputs/Outputs + tosa_datatype_t dt = tosa_datatype_fp32_t; + std::vector input_shape = { 2, 4, 4, 1 }; + std::vector output_shape = { 2, 4, 4, 1 }; + std::vector srcData1(32, 4.0f); + std::vector srcData2(32, 3.0f); + std::vector dstData(32, 0.0f); + + tosa_tensor_t input1; + input1.shape = input_shape.data(); + input1.num_dims = input_shape.size(); + input1.data_type = dt; + input1.data = reinterpret_cast(srcData1.data()); + input1.size = srcData1.size() * sizeof(float); + + tosa_tensor_t input2; + input2.shape = input_shape.data(); + input2.num_dims = input_shape.size(); + input2.data_type = dt; + input2.data = reinterpret_cast(srcData2.data()); + input2.size = srcData2.size() * sizeof(float); + + tosa_tensor_t output; + output.shape = output_shape.data(); + output.num_dims = output_shape.size(); + output.data_type = dt; + output.data = reinterpret_cast(dstData.data()); + output.size = dstData.size() * sizeof(float); + + // Execution + auto status = tosa_run_add(input1, input2, output); + CHECK((status == tosa_status_valid)); + + // Compare results + std::vector expectedData(8, 7.0f); + compareOutput(dstData, expectedData, expectedData.size()); + } - std::vector input_names = { "TosaInput_0", "TosaInput_1" }; - std::string output_name = "TosaOutput_0"; + TEST_CASE("op_entry_avg_pool2d") + { + // Pool parameters + const int32_t kernel[2] = { 2, 2 }; + const int32_t stride[2] = { 2, 2 }; + const int32_t pad[4] = { 0, 0, 0, 0 }; + + // Inputs/Outputs + tosa_datatype_t dt = tosa_datatype_fp32_t; + std::vector input_shape = { 2, 4, 4, 1 }; + std::vector output_shape = { 2, 2, 2, 1 }; + std::vector srcData(32, 7.0f); + std::vector dstData(8, 0.f); + + tosa_tensor_t input; + input.shape = input_shape.data(); + input.num_dims = input_shape.size(); + input.data_type = dt; + input.data = reinterpret_cast(srcData.data()); + input.size = srcData.size() * sizeof(float); + + tosa_tensor_t output; + output.shape = output_shape.data(); + output.num_dims = output_shape.size(); + output.data_type = dt; + output.data = reinterpret_cast(dstData.data()); + output.size = dstData.size() * sizeof(float); + + // Execution + auto status = tosa_run_avg_pool2d(input, kernel, stride, pad, 0, 0, output); + CHECK((status == tosa_status_valid)); + + // Compare results + std::vector expectedData(8, 7.0f); + compareOutput(dstData, expectedData, expectedData.size()); + } - std::vector input0_shape = { 1, 4, 4, 1 }; - std::vector input1_shape = { 1, 4, 4, 4 }; - std::vector output_shape = { 1, 4, 4, 4 }; + TEST_CASE("op_entry_conv2d") + { + // Conv parameters + const int32_t stride[2] = { 1, 1 }; + const int32_t pad[4] = { 0, 0, 0, 0 }; + const int32_t dilation[2] = { 1, 1 }; + + // Inputs/Outputs + tosa_datatype_t dt = tosa_datatype_fp32_t; + std::vector input_shape = { 1, 32, 32, 8 }; + std::vector output_shape = { 1, 32, 32, 16 }; + std::vector weight_shape = { 16, 1, 1, 8 }; + std::vector bias_shape = { 16 }; + std::vector srcData(32 * 32 * 8, 1.0f); + std::vector dstData(32 * 32 * 16, 0.f); + std::vector biasData(16, 0.f); + std::vector weightData(16 * 8, 1.0f); + + tosa_tensor_t input; + input.shape = input_shape.data(); + input.num_dims = input_shape.size(); + input.data_type = dt; + input.data = reinterpret_cast(srcData.data()); + input.size = srcData.size() * sizeof(float); + + tosa_tensor_t weight; + weight.shape = weight_shape.data(); + weight.num_dims = weight_shape.size(); + weight.data_type = dt; + weight.data = reinterpret_cast(weightData.data()); + weight.size = weightData.size() * sizeof(float); + + tosa_tensor_t bias; + bias.shape = bias_shape.data(); + bias.num_dims = bias_shape.size(); + bias.data_type = dt; + bias.data = reinterpret_cast(biasData.data()); + bias.size = biasData.size() * sizeof(float); + + tosa_tensor_t output; + output.shape = output_shape.data(); + output.num_dims = output_shape.size(); + output.data_type = dt; + output.data = reinterpret_cast(dstData.data()); + output.size = dstData.size() * sizeof(float); + + const int32_t input_zp = 0; + const int32_t weight_zp = 0; + + // Execution + auto status = tosa_run_conv2d(input, weight, bias, pad, stride, dilation, input_zp, weight_zp, output); + CHECK((status == tosa_status_valid)); + + // Compare results + std::vector expectedData(32 * 32 * 16, 8.0f); + compareOutput(dstData, expectedData, expectedData.size()); + } - std::vector> inputs(input_names.size()); - std::vector actual_outputs = { }; - std::vector expected_outputs = { }; + TEST_CASE("op_entry_max_pool2d") + { + // Pool parameters + const int32_t kernel[2] = { 2, 2 }; + const int32_t stride[2] = { 2, 2 }; + const int32_t pad[4] = { 0, 0, 0, 0 }; + + // Inputs/Outputs + tosa_datatype_t dt = tosa_datatype_fp32_t; + std::vector input_shape = { 2, 4, 4, 1 }; + std::vector output_shape = { 2, 2, 2, 1 }; + std::vector srcData(32); + std::vector dstData(8, 0.f); + std::iota(std::begin(srcData), std::end(srcData), 1); + + tosa_tensor_t input; + input.shape = input_shape.data(); + input.num_dims = input_shape.size(); + input.data_type = dt; + input.data = reinterpret_cast(srcData.data()); + input.size = srcData.size() * sizeof(float); + + tosa_tensor_t output; + output.shape = output_shape.data(); + output.num_dims = output_shape.size(); + output.data_type = dt; + output.data = reinterpret_cast(dstData.data()); + output.size = dstData.size() * sizeof(float); + + // Execution + auto status = tosa_run_max_pool2d(input, kernel, stride, pad, 0, 0, output); + CHECK((status == tosa_status_valid)); + + // Compare results + std::vector expectedData = { 6, 8, 14, 16, 22, 24, 30, 32 }; + compareOutput(dstData, expectedData, expectedData.size()); + } - // Read in inputs and expected outputs. - inputs[0] = readFromNpyFile(input0_file.c_str(), input0_shape); - inputs[1] = readFromNpyFile(input1_file.c_str(), input1_shape); - expected_outputs = readFromNpyFile(expected_output_file.c_str(), output_shape); + TEST_CASE("op_entry_pad") + { + // Inputs/Outputs + tosa_datatype_t dt = tosa_datatype_fp32_t; + std::vector input_shape = { 2, 2 }; + std::vector output_shape = { 4, 4 }; + std::vector srcData1(4, 4.0f); + std::vector dstData(16, 0.0f); + + tosa_tensor_t input1; + input1.shape = input_shape.data(); + input1.num_dims = input_shape.size(); + input1.data_type = dt; + input1.data = reinterpret_cast(srcData1.data()); + input1.size = srcData1.size() * sizeof(float); + + tosa_tensor_t output; + output.shape = output_shape.data(); + output.num_dims = output_shape.size(); + output.data_type = dt; + output.data = reinterpret_cast(dstData.data()); + output.size = dstData.size() * sizeof(float); + + // Execution + int32_t padding[4] = { 1, 1, 1, 1 }; + int32_t padding_len = 4; + int32_t pad_const_int = 0; + float pad_const_fp = 5.0f; + auto status = tosa_run_pad(input1, padding_len, padding, pad_const_int, pad_const_fp, output); + CHECK((status == tosa_status_valid)); + + // Compare results + // Expect a 4x4 array with a border of 5's and inner 2x2 of 4's + std::vector expectedData(16, 5.0f); + expectedData[5] = 4.0f; + expectedData[6] = 4.0f; + expectedData[9] = 4.0f; + expectedData[10] = 4.0f; + compareOutput(dstData, expectedData, expectedData.size()); + } - TosaSerializationHandler handler; - tosa_err_t error = handler.LoadFileTosaFlatbuffer(tosa_model_file.c_str()); - CHECK((error == tosa::TOSA_OK)); + TEST_CASE("simple_add_f32_test") + { + std::string test_root(std::string(PROJECT_ROOT) + "../examples/test_add_1x4x4x4_f32/"); + std::string tosa_model_file(test_root + "flatbuffer-tflite/test_add_1x4x4x4_f32.tosa"); + std::string input0_file(test_root + "placeholder_0.npy"); + std::string input1_file(test_root + "placeholder_1.npy"); + std::string expected_output_file(test_root + "tflite_result.npy"); - GraphStatus status; + std::vector input_names = { "TosaInput_0", "TosaInput_1" }; + std::string output_name = "TosaOutput_0"; - // Initialize the ModelRunner with configurations. - IModelRunner runner; - status = runner.initialize(handler); - CHECK((status == GraphStatus::TOSA_VALID)); + std::vector input0_shape = { 1, 4, 4, 1 }; + std::vector input1_shape = { 1, 4, 4, 4 }; + std::vector output_shape = { 1, 4, 4, 4 }; - runner.setInput(input_names[0], inputs[0]); - runner.setInput(input_names[1], inputs[1]); + std::vector> inputs(input_names.size()); + std::vector actual_outputs = {}; + std::vector expected_outputs = {}; - // Run the ModelRunner using test inputs. - status = runner.run(); - CHECK((status == GraphStatus::TOSA_VALID)); + // Read in inputs and expected outputs. + inputs[0] = readFromNpyFile(input0_file.c_str(), input0_shape); + inputs[1] = readFromNpyFile(input1_file.c_str(), input1_shape); + expected_outputs = readFromNpyFile(expected_output_file.c_str(), output_shape); - actual_outputs = runner.getOutput(output_name); - CHECK(!actual_outputs.empty()); + TosaSerializationHandler handler; + tosa_err_t error = handler.LoadFileTosaFlatbuffer(tosa_model_file.c_str()); + CHECK((error == tosa::TOSA_OK)); - compareOutput(expected_outputs, actual_outputs, expected_outputs.size()); -} + GraphStatus status; -TEST_CASE("conv2d_f32_test") -{ - std::string test_root(std::string(PROJECT_ROOT) + "../examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/"); - std::string tosa_model_file(test_root + "flatbuffer-tflite/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11.tosa"); - std::string input_file(test_root + "placeholder_0.npy"); - std::string expected_output_file(test_root + "tflite_result.npy"); + // Initialize the ModelRunner with configurations. + IModelRunner runner; + status = runner.initialize(handler); + CHECK((status == GraphStatus::TOSA_VALID)); - std::string input_name = "TosaInput_0"; - std::string output_name = "TosaOutput_0"; + runner.setInput(input_names[0], inputs[0]); + runner.setInput(input_names[1], inputs[1]); - std::vector input_shape = { 1, 32, 32, 8 }; - std::vector output_shape = { 1, 32, 32, 16 }; + // Run the ModelRunner using test inputs. + status = runner.run(); + CHECK((status == GraphStatus::TOSA_VALID)); - // Read in inputs and expected outputs. - std::vector inputs = readFromNpyFile(input_file.c_str(), input_shape); - std::vector expected_outputs = readFromNpyFile(expected_output_file.c_str(), output_shape); + actual_outputs = runner.getOutput(output_name); + CHECK(!actual_outputs.empty()); - TosaSerializationHandler handler; - tosa_err_t error = handler.LoadFileTosaFlatbuffer(tosa_model_file.c_str()); - CHECK((error == tosa::TOSA_OK)); + compareOutput(expected_outputs, actual_outputs, expected_outputs.size()); + } - GraphStatus status; + TEST_CASE("conv2d_f32_test") + { + std::string test_root(std::string(PROJECT_ROOT) + + "../examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/"); + std::string tosa_model_file(test_root + + "flatbuffer-tflite/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11.tosa"); + std::string input_file(test_root + "placeholder_0.npy"); + std::string expected_output_file(test_root + "tflite_result.npy"); - // Initialize the ModelRunner with configurations. - IModelRunner runner; - status = runner.initialize(handler); - CHECK((status == GraphStatus::TOSA_VALID)); + std::string input_name = "TosaInput_0"; + std::string output_name = "TosaOutput_0"; - runner.setInput(input_name, inputs); + std::vector input_shape = { 1, 32, 32, 8 }; + std::vector output_shape = { 1, 32, 32, 16 }; - // Run the ModelRunner using test inputs. - status = runner.run(); - CHECK((status == GraphStatus::TOSA_VALID)); + // Read in inputs and expected outputs. + std::vector inputs = readFromNpyFile(input_file.c_str(), input_shape); + std::vector expected_outputs = readFromNpyFile(expected_output_file.c_str(), output_shape); - std::vector actual_outputs = runner.getOutput(output_name); - CHECK(!actual_outputs.empty()); + TosaSerializationHandler handler; + tosa_err_t error = handler.LoadFileTosaFlatbuffer(tosa_model_file.c_str()); + CHECK((error == tosa::TOSA_OK)); - compareOutput(expected_outputs, actual_outputs, expected_outputs.size()); -} + GraphStatus status; -TEST_CASE("conv2d_f32_validate_only_test") -{ - std::string test_root(std::string(PROJECT_ROOT) + "../examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/"); - std::string tosa_model_file(test_root + "flatbuffer-tflite/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11.tosa"); + // Initialize the ModelRunner with configurations. + IModelRunner runner; + status = runner.initialize(handler); + CHECK((status == GraphStatus::TOSA_VALID)); - TosaSerializationHandler handler; - tosa_err_t error = handler.LoadFileTosaFlatbuffer(tosa_model_file.c_str()); - CHECK((error == tosa::TOSA_OK)); + runner.setInput(input_name, inputs); - GraphStatus status; - func_debug_t funcDebug; + // Run the ModelRunner using test inputs. + status = runner.run(); + CHECK((status == GraphStatus::TOSA_VALID)); - func_config_t funcConfig; - funcConfig.validate_only = 1; + std::vector actual_outputs = runner.getOutput(output_name); + CHECK(!actual_outputs.empty()); - // Initialize the ModelRunner with configurations. - IModelRunner runner = IModelRunner(funcConfig, funcDebug); - runner.setFuncConfig(funcConfig); - status = runner.initialize(handler); - CHECK((status == GraphStatus::TOSA_VALID)); + compareOutput(expected_outputs, actual_outputs, expected_outputs.size()); + } - // Run the ModelRunner using no inputs, as validate_only is specified run() should still work. - status = runner.run(); - CHECK((status == GraphStatus::TOSA_VALID)); -} + TEST_CASE("conv2d_f32_validate_only_test") + { + std::string test_root(std::string(PROJECT_ROOT) + + "../examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/"); + std::string tosa_model_file(test_root + + "flatbuffer-tflite/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11.tosa"); + + TosaSerializationHandler handler; + tosa_err_t error = handler.LoadFileTosaFlatbuffer(tosa_model_file.c_str()); + CHECK((error == tosa::TOSA_OK)); + + GraphStatus status; + func_debug_t funcDebug; + + func_config_t funcConfig; + funcConfig.validate_only = 1; + + // Initialize the ModelRunner with configurations. + IModelRunner runner = IModelRunner(funcConfig, funcDebug); + runner.setFuncConfig(funcConfig); + status = runner.initialize(handler); + CHECK((status == GraphStatus::TOSA_VALID)); + + // Run the ModelRunner using no inputs, as validate_only is specified run() should still work. + status = runner.run(); + CHECK((status == GraphStatus::TOSA_VALID)); + } -} +} // TEST_SUITE(model_runner) diff --git a/scripts/operator_api/README.md b/scripts/operator_api/README.md new file mode 100644 index 0000000..381d90c --- /dev/null +++ b/scripts/operator_api/README.md @@ -0,0 +1,19 @@ +# Generate eager operator execution entrypoints + +## Introduction + +The generate_api.py script will generate an extended reference model API with eager operator execution entrypoints. +The following files will be generated: include/operators.h and src/operators.cc + +## Requirements + +* Python 3.6 or later +* Jinja2 (install with ```pip install Jinja2```) + +## Running from the command line + +The script can be run from the scripts/operator-api directory as follows: + +```bash +python generate_api.py +``` diff --git a/scripts/operator_api/generate_api.py b/scripts/operator_api/generate_api.py new file mode 100644 index 0000000..1f89f74 --- /dev/null +++ b/scripts/operator_api/generate_api.py @@ -0,0 +1,349 @@ +"""Generate extended reference model API with eager operator execution entrypoints""" +# Copyright (c) 2021-2022, ARM Limited. +# SPDX-License-Identifier: Apache-2.0 +import copy +import os +import subprocess +from xml.dom import minidom + +from jinja2 import Environment +from jinja2 import FileSystemLoader + + +def getTosaArgTypes(tosaXml): + """ + Returns a list of the TOSA argument types from tosa.xml. + """ + argTypes = {"in_t", "out_t", "mul_t", "weight_t", "in_out_t"} + argTypesXml = tosaXml.getElementsByTagName("type") + for argTypeXml in argTypesXml: + argTypes.add(argTypeXml.getAttribute("name")) + argTypes.remove("TABLE_SIZE") + return argTypes + + +def getTosaDataTypes(tosaXml): + """ + Returns a list of the TOSA data types from tosa.xml. + """ + argTypes = getTosaArgTypes(tosaXml) + dataTypes = set() + dataTypesXml = tosaXml.getElementsByTagName("typesupport") + for dataTypeXml in dataTypesXml: + for argType in argTypes: + dataType = dataTypeXml.getAttribute(argType) + if dataType != "": + dataTypes.add(f"tosa_datatype_{dataType}") + return sorted(dataTypes) + + +def getSerializeOpType(tosaOpName): + """ + Returns the Serialization library operator that matches the TOSA operator specified. + """ + map = { + "avg_pool2d": "Pool", + "conv2d": "Conv", + "conv3d": "Conv", + "depthwise_conv2d": "Conv", + "fully_connected": "FullyConnected", + "matmul": "MatMul", + "max_pool2d": "Pool", + "transpose_conv2d": "Conv", + "clamp": "Clamp", + "arithmetic_right_shift": "ArithmeticRightShift", + "mul": "Mul", + "table": "Table", + "negate": "Negate", + "pad": "Pad", + "reshape": "Reshape", + "slice": "Slice", + "tile": "Tile", + "transpose": "Transpose", + "resize": "Resize", + "rescale": "Rescale", + "cond_if": "CondIf", + "while_loop": "WhileLoop", + } + if tosaOpName not in map.keys(): + return "None" + else: + return map[tosaOpName] + + +def getSerializeArgsForOp(tosaOpName, allSerializeArgs, tosaArgs): + """ + Returns the arguments required by the Serialization library for the TOSA operator specified. + Generates code to initialize Serialization arguments. If a matching TOSA argument exists, + that value is used for initialization, otherwise a default value e.g. 0 is used. + """ + serOpType = getSerializeOpType(tosaOpName) + if serOpType not in allSerializeArgs.keys(): + return {} + else: + serOpArgs = copy.deepcopy(allSerializeArgs[serOpType]) + tosaArgsDict = {arg["name"]: arg for arg in tosaArgs} + serTosaTypeMap = {"ResizeMode": "tosa_mode"} + for arg in serOpArgs: + argName = arg["name"] + init = "" + # Translate TOSA data types to Serialization data types for initialization + if arg["dType"] in serTosaTypeMap.keys(): + init = f" = translate_client_{serTosaTypeMap[arg['dType']]}(client_{argName})" + # Initialize Serialization arguments to their matching function parameter + elif argName in tosaArgsDict: + if arg["SV"] == "V": + shape = tosaArgsDict[argName]["shape"] + if shape == "[]": + init = f"(&client_{argName}[0], &client_{argName}[0] + client_{argName}_len)" + else: + init = f"(&client_{argName}[0], &client_{argName}{shape})" + else: + init = f" = client_{argName}" + else: + # Initialize Serialization arguments with no matching fuction parameter + if arg["SV"] == "V": + init = "" + else: + if arg["dType"] == "DType": + arg["dType"] = "tosa::DType" + init = " = tosa::DType::DType_FP32" + else: + init = " = 0" + arg["init"] = init + return serOpArgs + + +def updateTosaArgs(tosaArgs, serializeArgs, tosaXml): + """ + Replace TOSA argument data types with their matching Serialization argument data types. + Delete TOSA arguments where the type couldn't be determined. + Add Serialization arguments that have no matching TOSA argument. + """ + tosaArgTypes = getTosaArgTypes(tosaXml) + serArgsDict = {arg["name"]: arg for arg in serializeArgs} + tosaArgsNames = [arg["name"] for arg in tosaArgs] + delTosaArgs = [] + # Replace TOSA argument data types with their matching Serialization argument data types. + for tosaArg in tosaArgs: + if tosaArg["type"] in tosaArgTypes: + if tosaArg["name"] in serArgsDict: + tosaArg["type"] = serArgsDict[tosaArg["name"]]["dType"] + else: + # Delete TOSA argument whose data type can't be determined + delTosaArgs.append(tosaArgsNames.index(tosaArg["name"])) + # Delete corresponding length argument if one exists + lenArgName = f"{tosaArg['name']}_len" + if lenArgName in tosaArgsNames: + delTosaArgs.append(tosaArgsNames.index(lenArgName)) + # Delete TOSA arguments where the type couldn't be determined + for index in sorted(delTosaArgs, key=int, reverse=True): + del tosaArgs[index] + # Add Serialization arguments that have no matching TOSA argument + tosaArgNames = [arg["name"] for arg in tosaArgs] + for serArg in serializeArgs: + if (serArg["name"] not in tosaArgNames) and ( + not serArg["dType"] == "tosa::DType" + ): + serArgName = serArg["name"] + if serArg["SV"] == "V": + # For vector data types, insert a matching length argument + tosaArgs.insert( + len(tosaArgs) - 1, + { + "name": f"{serArgName}_len", + "type": "int32_t", + "shape": "", + "category": "", + }, + ) + init = f"(&client_{serArgName}[0], &client_{serArgName}[0] + client_{serArgName}_len)" + shape = "[]" + else: + init = f" = client_{serArg['name']}" + shape = "" + serArg["init"] = init + # Insert new argument + tosaArgs.insert( + len(tosaArgs) - 1, + { + "name": serArgName, + "type": serArg["dType"], + "shape": shape, + "category": "", + }, + ) + + +def getOperators(tosaXml): + """ + Return a list of TOSA operators as defined by tosa.xml. + """ + operators = [] + ignoreOps = ["while_loop", "cond_if", "const", "custom", "fft2d", "rfft2d"] + opsXml = tosaXml.getElementsByTagName("operator") + allSerializeArgs = getSerializeArgs() + for opXml in opsXml: + opName = opXml.getElementsByTagName("name")[0].firstChild.data.lower() + if opName not in ignoreOps: + operator = {"name": opName} + operator["serializeAttType"] = getSerializeOpType(opName) + tosaArgs = getTosaArgs(opXml) + serializeArgs = getSerializeArgsForOp(opName, allSerializeArgs, tosaArgs) + updateTosaArgs(tosaArgs, serializeArgs, tosaXml) + operator["arguments"] = tosaArgs + operator["serializeArgs"] = serializeArgs + operator["inputs"] = [ + arg["name"] for arg in tosaArgs if arg["category"] == "input" + ] + operator["outputs"] = [ + arg["name"] for arg in tosaArgs if arg["category"] == "output" + ] + operators.append(operator) + return operators + + +def getTosaArgs(opXml): + """ + Return the arguments required for the TOSA operator specified. + """ + arguments = [] + argsXml = opXml.getElementsByTagName("argument") + tosaTensorTypes = getTosaArgTypes(tosaXml) + tosaTypeMap = {"bool_t": "bool", "uint6_t": "uint8_t", "mode_t": "tosa_mode_t"} + for xmlArg in argsXml: + argName = xmlArg.getAttribute("name").lower() + argType = xmlArg.getAttribute("type") + argShape = xmlArg.getAttribute("shape") + argCategory = xmlArg.getAttribute("category") + # Update argument type + if argType[-1:] == "*": + argType = argType[:-1] + if argCategory in ["input", "output"] and argType in tosaTensorTypes: + argType = "tosa_tensor_t" + argShape = "" + if argType in tosaTypeMap: + argType = tosaTypeMap[argType] + # Add a length argument for arrays with unknown compile-time size + if argShape != "" and argShape[0] == "[" and not argShape[1:-1].isnumeric(): + argShape = "[]" + arguments.append( + { + "name": f"{argName}_len", + "type": "int32_t", + "shape": "", + "category": "", + } + ) + elif argShape == "" or not argShape[0] == "[": + argShape = "" + # Append argument + arguments.append( + { + "name": argName, + "type": argType, + "shape": argShape, + "category": argCategory, + } + ) + return arguments + + +def clangFormat(filename): + cmd = ["clang-format", "-i", filename] + with open(os.devnull, "w") as devnull: + subprocess.check_call(cmd, stdout=devnull) + + +def getSerializeArgs(): + """ + Parse attribute.def file and return a dictionary where the keys are Serialization library operator names. + The values are the arguments required by each Serialization library operator. + """ + serializeArgs = {} + with open("../../thirdparty/serialization_lib/include/attribute.def") as file: + preamble = True + inAtt = False + opName = "" + args = [] + for line in file: + if preamble and not line[: len("DEF_ATTRIBUTE(")] == "DEF_ATTRIBUTE(": + continue + else: + preamble = False + line = line.lstrip().rstrip() + if not inAtt and "DEF_ATTRIBUTE(" in line: + opName = line[len("DEF_ATTRIBUTE(") : line.find(",")] + inAtt = True + elif inAtt: + vals = line.split(",") + argName = vals[2].lstrip().strip() + if ")" in argName: + argName = argName[:-1] + arg = { + "name": argName, + "dType": vals[0].lstrip().strip(), + "SV": vals[1].lstrip().strip(), + } + args.append(arg) + if ")" in line: + serializeArgs[opName] = args + opName = "" + args = [] + inAtt = False + return serializeArgs + + +def renderTemplate(environment, dataTypes, operators, template, outfile): + content = template.render(dataTypes=dataTypes, operators=operators) + with open(outfile, mode="w", encoding="utf-8") as output: + output.write(content) + print(f"Created {outfile}") + + clangFormat(outfile) + + +def generate(environment, dataTypes, operators): + # Generate include/operators.h + template = environment.get_template("operators_h.j2") + outfile = os.path.join("..", "..", "reference_model", "include", "operators.h") + renderTemplate(environment, dataTypes, operators, template, outfile) + + # Generate src/operators.cc + template = environment.get_template("operators_cc.j2") + outfile = os.path.join("..", "..", "reference_model", "src", "operators.cc") + renderTemplate(environment, dataTypes, operators, template, outfile) + + +def getSerializeOpTypeMap(): + """ + Utility function for generating the map used in getSerializeOpType() + """ + import re + + allSerializeArgs = getSerializeArgs() + serArgs = [ + re.sub(r"(?>(status); \ + return static_cast(ustatus); \ + } \ + } while (false) + +namespace { + +tosa::DType translate_client_datatype(tosa_datatype_t type) +{ + switch (type) + { + case tosa_datatype_fp16_t: + return tosa::DType::DType_FP16; + case tosa_datatype_fp32_t: + return tosa::DType::DType_FP32; + default: + return tosa::DType::DType_UNKNOWN; + } +}; + +tosa::TosaSerializationTensor* translate_client_tensor(tosa_tensor_t& tensor, const std::string& name) +{ + std::vector shape(tensor.shape, tensor.shape + tensor.num_dims); + return new tosa::TosaSerializationTensor(name, shape, translate_client_datatype(tensor.data_type), {}); +} + +tosa::ResizeMode translate_client_tosa_mode(tosa_mode_t mode) { + switch(mode) { + case tosa_mode_nearest: + return tosa::ResizeMode_NEAREST; + case tosa_mode_max: + case tosa_mode_bilinear: + return tosa::ResizeMode_BILINEAR; + default: + return tosa::ResizeMode_UNKNOWN; + } +} + +} // namespace + +extern "C" +{ + {% for operator in operators: %} + tosa_status_t tosa_run_{{ operator.name }} ( + {%- for arg in operator.arguments: -%} + {% if arg.type != "tosa_tensor_t" -%}const {% endif -%}{{arg.type}} client_{{arg.name}}{{arg.shape}} + {% if loop.index < operator.arguments|length %},{% endif %} + {%- endfor -%} + ) + { + // Create operator attributes + {% for arg in operator.serializeArgs: %} + {%- if arg.SV == "V": -%} + const std::vector<{{arg.dType}}> {{arg.name}}{{arg.init}}; + {%- else: -%} + const {{arg.dType}} {{arg.name}}{{arg.init}}; + {%- endif -%} + {%- endfor -%} + + Tosa{{operator.serializeAttType}}Attribute attr + {%- if operator.serializeArgs|length > 0 -%} + ( + {%- for arg in operator.serializeArgs: -%} + {{arg.name}}{% if loop.index < operator.serializeArgs|length %}, {% endif %} + {%- endfor -%} + ) + {%- endif -%}; + + // Create tensors + {% for input in operator.inputs: -%} + tosa::TosaSerializationTensor* {{input}} = translate_client_tensor(client_{{input}}, "{{input}}"); + {%- endfor -%} + {% for output in operator.outputs: %} + tosa::TosaSerializationTensor* {{output}} = translate_client_tensor(client_{{output}}, "{{output}}"); + {%- endfor %} + + // Create operator + auto op = new tosa::TosaSerializationOperator(tosa::Op::Op_{{operator.name|upper}}, + {%- if operator.serializeAttType != "None" -%} + tosa::Attribute::Attribute_{{operator.serializeAttType}}Attribute + {%- else -%} + tosa::Attribute::Attribute_NONE + {%- endif -%}, + &attr, { + {%- for input in operator.inputs: -%} + {{input}}->GetName() + {%- if loop.index < operator.inputs|length -%},{%- endif -%} + {%- endfor -%} + }, + { + {%- for output in operator.outputs: -%} + {{output}}->GetName() + {%- if loop.index < operator.outputs|length -%},{%- endif -%} + {%- endfor -%} + }); + + // Create a tosa single-op basic block + tosa::TosaSerializationBasicBlock block("{{operator.name}}", { op }, + { + {%- for input in operator.inputs: -%} + {{input}}, + {%- endfor -%} + {%- for output in operator.outputs: -%} + {{output}} + {%- if loop.index < operator.outputs|length -%},{%- endif -%} + {%- endfor -%} + }, + { + {%- for input in operator.inputs: -%} + {{input}}->GetName() + {%- if loop.index < operator.inputs|length -%},{%- endif -%} + {%- endfor -%} + }, + { + {%- for output in operator.outputs: -%} + {{output}}->GetName() + {%- if loop.index < operator.outputs|length -%},{%- endif -%} + {%- endfor -%} + }); + + // Setup model + TosaReference::ModelRunnerImpl runner; + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.initialize(block)); + {% for input in operator.inputs: -%} + TOSA_RETURN_ON_ERROR(runner.setInput({{input}}->GetName(), client_{{input}}.data, client_{{input}}.size)); + {%- endfor %} + + // Execute + TOSA_RETURN_ON_GRAPH_STATUS_ERROR(runner.run()); + + // Extract outputs + {% for output in operator.outputs: -%} + TOSA_RETURN_ON_ERROR(runner.getOutput({{output}}->GetName(), client_{{output}}.data, client_{{output}}.size)); + {%- endfor %} + + return tosa_status_valid; + } + {% endfor %} + +} // extern "C" \ No newline at end of file diff --git a/scripts/operator_api/templates/operators_h.j2 b/scripts/operator_api/templates/operators_h.j2 new file mode 100644 index 0000000..803b76a --- /dev/null +++ b/scripts/operator_api/templates/operators_h.j2 @@ -0,0 +1,74 @@ + +// Copyright (c) 2022, ARM Limited. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// THIS FILE IS GENERATED. DO NOT EDIT! +// See scripts/operator_api/generate_api.py + +#ifndef OPERATORS_H_ +#define OPERATORS_H_ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + + // Note status needs to be aligned with graph_status + enum tosa_status_t + { + tosa_status_valid = 0, + tosa_status_unpredictable = 1, + tosa_status_error = 2 + }; + + enum tosa_mode_t + { + tosa_mode_unknown = 0, + tosa_mode_nearest = 1, + tosa_mode_bilinear = 2, + tosa_mode_min = 3, + tosa_mode_max = 4 + }; + + enum tosa_datatype_t + { + {% for dataType in dataTypes: -%} + {{dataType}} = {{loop.index-1}}, + {% endfor -%} + }; + + struct tosa_tensor_t + { + int32_t* shape; + int32_t num_dims; + tosa_datatype_t data_type; + uint8_t* data; + size_t size; + }; + + {% for operator in operators: %} + tosa_status_t tosa_run_{{ operator.name }} ( + {%- for arg in operator.arguments: -%} + {% if arg.type != "tosa_tensor_t" -%}const {% endif -%}{{arg.type}} client_{{arg.name}}{{arg.shape}} + {% if loop.index < operator.arguments|length %},{% endif %} + {%- endfor -%}); + {% endfor %} + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif // OPERATORS_H_ \ No newline at end of file diff --git a/thirdparty/specification b/thirdparty/specification new file mode 160000 index 0000000..0205d99 --- /dev/null +++ b/thirdparty/specification @@ -0,0 +1 @@ +Subproject commit 0205d99cbff58797bf6602ee5718d50c00d8309b -- cgit v1.2.1