From 144b2d2d735d5b2e25894bdeacef7e03761025ad Mon Sep 17 00:00:00 2001 From: Davide Grohmann Date: Tue, 31 May 2022 15:24:02 +0200 Subject: Refactor message_handler app to be more test friendly Fix importing indexed models, the inclusion of header files was not working. Change-Id: Idab7e002abc31f57acc4eda6a2f1cf2defdfd5c4 --- applications/message_handler/CMakeLists.txt | 38 +- .../indexed_networks/indexed_networks.hpp | 105 ++++ .../indexed_networks/network_template.hpp | 23 + applications/message_handler/lib/CMakeLists.txt | 36 ++ .../message_handler/lib/core_driver_mutex.cpp | 83 +++ .../message_handler/lib/freertos_allocator.cpp | 45 ++ .../lib/include/message_handler.hpp | 187 +++++++ .../message_handler/lib/include/message_queue.hpp | 74 +++ .../message_handler/lib/include/networks.hpp | 64 +++ .../message_handler/lib/message_handler.cpp | 522 +++++++++++++++++ applications/message_handler/lib/message_queue.cpp | 151 +++++ applications/message_handler/main.cpp | 103 +--- applications/message_handler/message_handler.cpp | 616 --------------------- applications/message_handler/message_handler.hpp | 181 ------ applications/message_handler/message_queue.cpp | 151 ----- applications/message_handler/message_queue.hpp | 74 --- applications/message_handler/model_template.hpp | 23 - 17 files changed, 1319 insertions(+), 1157 deletions(-) create mode 100644 applications/message_handler/indexed_networks/indexed_networks.hpp create mode 100644 applications/message_handler/indexed_networks/network_template.hpp create mode 100644 applications/message_handler/lib/CMakeLists.txt create mode 100644 applications/message_handler/lib/core_driver_mutex.cpp create mode 100644 applications/message_handler/lib/freertos_allocator.cpp create mode 100644 applications/message_handler/lib/include/message_handler.hpp create mode 100644 applications/message_handler/lib/include/message_queue.hpp create mode 100644 applications/message_handler/lib/include/networks.hpp create mode 100644 applications/message_handler/lib/message_handler.cpp create mode 100644 applications/message_handler/lib/message_queue.cpp delete mode 100644 applications/message_handler/message_handler.cpp delete mode 100644 applications/message_handler/message_handler.hpp delete mode 100644 applications/message_handler/message_queue.cpp delete mode 100644 applications/message_handler/message_queue.hpp delete mode 100644 applications/message_handler/model_template.hpp diff --git a/applications/message_handler/CMakeLists.txt b/applications/message_handler/CMakeLists.txt index 040b27d..8d4ef54 100644 --- a/applications/message_handler/CMakeLists.txt +++ b/applications/message_handler/CMakeLists.txt @@ -21,36 +21,34 @@ if (NOT TARGET freertos_kernel) return() endif() +# Split total tensor arena equally for each NPU +if (TARGET ethosu_core_driver AND ETHOSU_TARGET_NPU_COUNT GREATER 0) + set(NUM_ARENAS ${ETHOSU_TARGET_NPU_COUNT}) +else() + set(NUM_ARENAS 1) +endif() + set(MESSAGE_HANDLER_ARENA_SIZE 2000000 CACHE STRING "Total size of all message handler tensor arenas") -set(MESSAGE_HANDLER_MODEL_0 FALSE CACHE STRING "Path to built in model 0") -set(MESSAGE_HANDLER_MODEL_1 FALSE CACHE STRING "Path to built in model 1") -set(MESSAGE_HANDLER_MODEL_2 FALSE CACHE STRING "Path to built in model 2") -set(MESSAGE_HANDLER_MODEL_3 FALSE CACHE STRING "Path to built in model 3") +math(EXPR TENSOR_ARENA_SIZE "${MESSAGE_HANDLER_ARENA_SIZE} / ${NUM_ARENAS}") + +add_subdirectory(lib) + +set(MESSAGE_HANDLER_MODEL_0 "" CACHE STRING "Path to built in model 0") +set(MESSAGE_HANDLER_MODEL_1 "" CACHE STRING "Path to built in model 1") +set(MESSAGE_HANDLER_MODEL_2 "" CACHE STRING "Path to built in model 2") +set(MESSAGE_HANDLER_MODEL_3 "" CACHE STRING "Path to built in model 3") ethosu_add_executable(message_handler SOURCES main.cpp - message_handler.cpp - message_queue.cpp LIBRARIES - cmsis_device - $<$:ethosu_core_driver> - ethosu_mailbox - freertos_kernel - inference_process) + message_handler_lib + freertos_kernel) target_include_directories(message_handler PRIVATE + indexed_networks ${LINUX_DRIVER_STACK_PATH}/kernel) -# Split total tensor arena equally for each NPU -if (TARGET ethosu_core_driver AND ETHOSU_TARGET_NPU_COUNT GREATER 0) - set(NUM_ARENAS ${ETHOSU_TARGET_NPU_COUNT}) -else() - set(NUM_ARENAS 1) -endif() - -math(EXPR TENSOR_ARENA_SIZE "${MESSAGE_HANDLER_ARENA_SIZE} / ${NUM_ARENAS}") - target_compile_definitions(message_handler PRIVATE TENSOR_ARENA_SIZE=${TENSOR_ARENA_SIZE} $<$:MODEL_0=${MESSAGE_HANDLER_MODEL_0}> diff --git a/applications/message_handler/indexed_networks/indexed_networks.hpp b/applications/message_handler/indexed_networks/indexed_networks.hpp new file mode 100644 index 0000000..0b6d62a --- /dev/null +++ b/applications/message_handler/indexed_networks/indexed_networks.hpp @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef INDEXED_NETWORKS_H +#define INDEXED_NETWORKS_H + +#include "networks.hpp" + +#include +#include + +#define XSTRINGIFY(src) #src +#define STRINGIFY(src) XSTRINGIFY(src) + +namespace { +#if defined(__has_include) + +#if defined(MODEL_0) +namespace Model0 { +#include STRINGIFY(MODEL_0) +} +#endif + +#if defined(MODEL_1) +namespace Model1 { +#include STRINGIFY(MODEL_1) +} +#endif + +#if defined(MODEL_2) +namespace Model2 { +#include STRINGIFY(MODEL_2) +} +#endif + +#if defined(MODEL_3) +namespace Model3 { +#include STRINGIFY(MODEL_3) +} +#endif + +#endif +} // namespace + +namespace MessageHandler { + +class WithIndexedNetworks : public BaseNetworks { +public: + static bool getIndexedNetwork(const uint32_t index, void *&data, size_t &size) { + switch (index) { +#if defined(MODEL_0) + case 0: + data = reinterpret_cast(Model0::networkModel); + size = sizeof(Model0::networkModel); + break; +#endif + +#if defined(MODEL_1) + case 1: + data = reinterpret_cast(Model1::networkModel); + size = sizeof(Model1::networkModel); + break; +#endif + +#if defined(MODEL_2) + case 2: + data = reinterpret_cast(Model2::networkModel); + size = sizeof(Model2::networkModel); + break; +#endif + +#if defined(MODEL_3) + case 3: + data = reinterpret_cast(Model3::networkModel); + size = sizeof(Model3::networkModel); + break; +#endif + + default: + printf("Error: Network model index out of range. index=%" PRIu32 "\n", index); + return true; + } + + return false; + } +}; + +} // namespace MessageHandler + +#endif diff --git a/applications/message_handler/indexed_networks/network_template.hpp b/applications/message_handler/indexed_networks/network_template.hpp new file mode 100644 index 0000000..353d7d3 --- /dev/null +++ b/applications/message_handler/indexed_networks/network_template.hpp @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +__attribute__((section(".sram.data"), aligned(16))) uint8_t networkModel[] = { + /* Add network model here */ +}; diff --git a/applications/message_handler/lib/CMakeLists.txt b/applications/message_handler/lib/CMakeLists.txt new file mode 100644 index 0000000..7e2826b --- /dev/null +++ b/applications/message_handler/lib/CMakeLists.txt @@ -0,0 +1,36 @@ +# +# Copyright (c) 2020-2022 Arm Limited. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the License); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an AS IS BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_library(message_handler_lib STATIC) + +target_include_directories(message_handler_lib PUBLIC include + PRIVATE ${LINUX_DRIVER_STACK_PATH}/kernel) + +target_link_libraries(message_handler_lib PUBLIC ethosu_mailbox + PRIVATE + cmsis_device + $<$:ethosu_core_driver> + freertos_kernel + inference_process + tflu) + +target_sources(message_handler_lib PRIVATE + message_handler.cpp + message_queue.cpp + core_driver_mutex.cpp + freertos_allocator.cpp) diff --git a/applications/message_handler/lib/core_driver_mutex.cpp b/applications/message_handler/lib/core_driver_mutex.cpp new file mode 100644 index 0000000..bc043fa --- /dev/null +++ b/applications/message_handler/lib/core_driver_mutex.cpp @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if defined(ETHOSU) + +#include "FreeRTOS.h" +#include "semphr.h" + +#include +#include + +extern "C" { + +void *ethosu_mutex_create(void) { + return xSemaphoreCreateMutex(); +} + +int ethosu_mutex_lock(void *mutex) { + SemaphoreHandle_t handle = reinterpret_cast(mutex); + if (xSemaphoreTake(handle, portMAX_DELAY) != pdTRUE) { + printf("Error: Failed to lock mutex.\n"); + return -1; + } + return 0; +} + +int ethosu_mutex_unlock(void *mutex) { + SemaphoreHandle_t handle = reinterpret_cast(mutex); + if (xSemaphoreGive(handle) != pdTRUE) { + printf("Error: Failed to unlock mutex.\n"); + return -1; + } + return 0; +} + +void *ethosu_semaphore_create(void) { + return xSemaphoreCreateBinary(); +} + +int ethosu_semaphore_take(void *sem) { + SemaphoreHandle_t handle = reinterpret_cast(sem); + if (xSemaphoreTake(handle, portMAX_DELAY) != pdTRUE) { + printf("Error: Failed to take semaphore.\n"); + return -1; + } + return 0; +} + +int ethosu_semaphore_give(void *sem) { + SemaphoreHandle_t handle = reinterpret_cast(sem); + if (xPortIsInsideInterrupt()) { + if (xSemaphoreGiveFromISR(handle, NULL) != pdTRUE) { + printf("Error: Failed to give semaphore from ISR.\n"); + return -1; + } + } else { + /* A FreeRTOS binary semaphore is fundamentally a queue that can only hold one item. If the queue is full, + * xSemaphoreGive will return a pdFALSE value. Ignoring the return value in here, as a semaphore give failure + * does not affect the application correctness. */ + if (xSemaphoreGive(handle) != pdTRUE) { + // do nothing + } + } + return 0; +} +} + +#endif diff --git a/applications/message_handler/lib/freertos_allocator.cpp b/applications/message_handler/lib/freertos_allocator.cpp new file mode 100644 index 0000000..c7fdba6 --- /dev/null +++ b/applications/message_handler/lib/freertos_allocator.cpp @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "FreeRTOS.h" +#include +#include + +void *operator new(size_t size) { + return pvPortMalloc(size); +} + +void *operator new[](size_t size) { + return pvPortMalloc(size); +} + +void operator delete(void *ptr) { + vPortFree(ptr); +} + +void operator delete(void *ptr, std::size_t) { + vPortFree(ptr); +} + +void operator delete[](void *ptr) { + vPortFree(ptr); +} + +void operator delete[](void *ptr, std::size_t) { + vPortFree(ptr); +} diff --git a/applications/message_handler/lib/include/message_handler.hpp b/applications/message_handler/lib/include/message_handler.hpp new file mode 100644 index 0000000..3c227be --- /dev/null +++ b/applications/message_handler/lib/include/message_handler.hpp @@ -0,0 +1,187 @@ +/* + * Copyright (c) 2020-2022 Arm Limited. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MESSAGE_HANDLER_H +#define MESSAGE_HANDLER_H + +#include "FreeRTOS.h" +#include "queue.h" +#include "semphr.h" + +#include "message_queue.hpp" +#include "networks.hpp" +#include +#if defined(ETHOSU) +#include +#endif +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace MessageHandler { + +template +class Queue { +public: + using Predicate = std::function; + + Queue() { + mutex = xSemaphoreCreateMutex(); + size = xSemaphoreCreateCounting(capacity, 0u); + + if (mutex == nullptr || size == nullptr) { + printf("Error: failed to allocate memory for inference queue\n"); + } + } + + ~Queue() { + vSemaphoreDelete(mutex); + vSemaphoreDelete(size); + } + + bool push(const T &data) { + xSemaphoreTake(mutex, portMAX_DELAY); + if (list.size() >= capacity) { + xSemaphoreGive(mutex); + return false; + } + + list.push_back(data); + xSemaphoreGive(mutex); + + // increase number of available inferences to pop + xSemaphoreGive(size); + return true; + } + + void pop(T &data) { + // decrease the number of available inferences to pop + xSemaphoreTake(size, portMAX_DELAY); + + xSemaphoreTake(mutex, portMAX_DELAY); + data = list.front(); + list.pop_front(); + xSemaphoreGive(mutex); + } + + bool erase(Predicate pred) { + // let's optimistically assume we are removing an inference, so decrease pop + if (pdFALSE == xSemaphoreTake(size, 0)) { + // if there are no inferences return immediately + return false; + } + + xSemaphoreTake(mutex, portMAX_DELAY); + auto found = std::find_if(list.begin(), list.end(), pred); + bool erased = found != list.end(); + if (erased) { + list.erase(found); + } + xSemaphoreGive(mutex); + + if (!erased) { + // no inference erased, so let's put the size count back + xSemaphoreGive(size); + } + + return erased; + } + +private: + std::list list; + + SemaphoreHandle_t mutex; + SemaphoreHandle_t size; +}; + +class IncomingMessageHandler { +public: + IncomingMessageHandler(EthosU::ethosu_core_queue &inputMessageQueue, + EthosU::ethosu_core_queue &outputMessageQueue, + Mailbox::Mailbox &mailbox, + std::shared_ptr> inferenceInputQueue, + QueueHandle_t inferenceOutputQueue, + SemaphoreHandle_t messageNotify, + std::shared_ptr networks); + + void run(); + +private: + bool handleMessage(); + bool handleInferenceOutput(); + static void handleIrq(void *userArg); + + void sendPong(); + void sendErrorAndResetQueue(EthosU::ethosu_core_msg_err_type type, const char *message); + void sendVersionRsp(); + void sendCapabilitiesRsp(uint64_t userArg); + void sendNetworkInfoRsp(uint64_t userArg, EthosU::ethosu_core_network_buffer &network); + void sendInferenceRsp(EthosU::ethosu_core_inference_rsp &inference); + void sendFailedInferenceRsp(uint64_t userArg, uint32_t status); + void sendCancelInferenceRsp(uint64_t userArg, uint32_t status); + void readCapabilties(EthosU::ethosu_core_msg_capabilities_rsp &rsp); + + MessageQueue::QueueImpl inputMessageQueue; + MessageQueue::QueueImpl outputMessageQueue; + Mailbox::Mailbox &mailbox; + InferenceProcess::InferenceParser parser; + std::shared_ptr> inferenceInputQueue; + QueueHandle_t inferenceOutputQueue; + SemaphoreHandle_t messageNotify; + EthosU::ethosu_core_msg_capabilities_rsp capabilities; + std::shared_ptr networks; +}; + +class InferenceHandler { +public: + InferenceHandler(uint8_t *tensorArena, + size_t arenaSize, + std::shared_ptr> inferenceInputQueue, + QueueHandle_t inferenceOutputQueue, + SemaphoreHandle_t messageNotify, + std::shared_ptr networks); + + void run(); + +private: + void runInference(EthosU::ethosu_core_inference_req &req, EthosU::ethosu_core_inference_rsp &rsp); + bool getInferenceJob(const EthosU::ethosu_core_inference_req &req, InferenceProcess::InferenceJob &job); + +#if defined(ETHOSU) + friend void ::ethosu_inference_begin(struct ethosu_driver *drv, void *userArg); + friend void ::ethosu_inference_end(struct ethosu_driver *drv, void *userArg); +#endif + std::shared_ptr> inferenceInputQueue; + QueueHandle_t inferenceOutputQueue; + SemaphoreHandle_t messageNotify; + InferenceProcess::InferenceProcess inference; + EthosU::ethosu_core_inference_req *currentReq; + EthosU::ethosu_core_inference_rsp *currentRsp; + std::shared_ptr networks; +}; + +} // namespace MessageHandler + +#endif diff --git a/applications/message_handler/lib/include/message_queue.hpp b/applications/message_handler/lib/include/message_queue.hpp new file mode 100644 index 0000000..4140c62 --- /dev/null +++ b/applications/message_handler/lib/include/message_queue.hpp @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2020-2022 Arm Limited. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MESSAGE_QUEUE_H +#define MESSAGE_QUEUE_H + +#include +#include + +namespace MessageQueue { + +template +struct Queue { + EthosU::ethosu_core_queue_header header; + uint8_t data[SIZE]; + + constexpr Queue() : header({SIZE, 0, 0}) {} + + constexpr EthosU::ethosu_core_queue *toQueue() { + return reinterpret_cast(&header); + } +}; + +class QueueImpl { +public: + struct Vec { + const void *base; + size_t length; + }; + + QueueImpl(EthosU::ethosu_core_queue &queue); + + bool empty() const; + size_t available() const; + size_t capacity() const; + void reset(); + bool read(uint8_t *dst, uint32_t length); + template + bool read(T &dst) { + return read(reinterpret_cast(&dst), sizeof(dst)); + } + bool write(const Vec *vec, size_t length); + bool write(const uint32_t type, const void *src = nullptr, uint32_t length = 0); + template + bool write(const uint32_t type, const T &src) { + return write(type, reinterpret_cast(&src), sizeof(src)); + } + +private: + void cleanHeader() const; + void cleanHeaderData() const; + void invalidateHeader() const; + void invalidateHeaderData() const; + + EthosU::ethosu_core_queue &queue; +}; +} // namespace MessageQueue + +#endif diff --git a/applications/message_handler/lib/include/networks.hpp b/applications/message_handler/lib/include/networks.hpp new file mode 100644 index 0000000..eb01d10 --- /dev/null +++ b/applications/message_handler/lib/include/networks.hpp @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef NETWORKS_H +#define NETWORKS_H + +#include + +#include +#include + +using namespace EthosU; + +namespace MessageHandler { + +class Networks { +public: + virtual ~Networks() {} + virtual bool getNetwork(const ethosu_core_network_buffer &buffer, void *&data, size_t &size) = 0; +}; + +template +class BaseNetworks : public Networks { +public: + bool getNetwork(const ethosu_core_network_buffer &buffer, void *&data, size_t &size) override { + switch (buffer.type) { + case ETHOSU_CORE_NETWORK_BUFFER: + data = reinterpret_cast(buffer.buffer.ptr); + size = buffer.buffer.size; + return false; + case ETHOSU_CORE_NETWORK_INDEX: + return T::getIndexedNetwork(buffer.index, data, size); + default: + printf("Error: Unsupported network model type. type=%" PRIu32 "\n", buffer.type); + return true; + } + } +}; + +class NoIndexedNetworks : public BaseNetworks { + static bool getIndexedNetwork(const uint32_t index, void *&data, size_t &size) { + printf("Error: Network model index out of range. index=%" PRIu32 "\n", index); + return true; + } +}; + +} // namespace MessageHandler + +#endif diff --git a/applications/message_handler/lib/message_handler.cpp b/applications/message_handler/lib/message_handler.cpp new file mode 100644 index 0000000..7b6377f --- /dev/null +++ b/applications/message_handler/lib/message_handler.cpp @@ -0,0 +1,522 @@ +/* + * Copyright (c) 2020-2022 Arm Limited. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "message_handler.hpp" + +#include "cmsis_compiler.h" + +#ifdef ETHOSU +#include +#include +#endif + +#include "FreeRTOS.h" +#include "queue.h" +#include "semphr.h" + +#include +#include + +using namespace EthosU; +using namespace MessageQueue; + +namespace MessageHandler { + +/**************************************************************************** + * IncomingMessageHandler + ****************************************************************************/ + +IncomingMessageHandler::IncomingMessageHandler( + EthosU::ethosu_core_queue &_inputMessageQueue, + EthosU::ethosu_core_queue &_outputMessageQueue, + Mailbox::Mailbox &_mailbox, + std::shared_ptr> _inferenceInputQueue, + QueueHandle_t _inferenceOutputQueue, + SemaphoreHandle_t _messageNotify, + std::shared_ptr _networks) : + inputMessageQueue(_inputMessageQueue), + outputMessageQueue(_outputMessageQueue), mailbox(_mailbox), inferenceInputQueue(_inferenceInputQueue), + inferenceOutputQueue(_inferenceOutputQueue), messageNotify(_messageNotify), networks(_networks) { + mailbox.registerCallback(handleIrq, reinterpret_cast(this)); + readCapabilties(capabilities); +} + +void IncomingMessageHandler::run() { + while (true) { + // Wait for event + xSemaphoreTake(messageNotify, portMAX_DELAY); + + // Handle all inference outputs and all messages in queue + while (handleInferenceOutput() || handleMessage()) {} + } +} + +void IncomingMessageHandler::handleIrq(void *userArg) { + if (userArg == nullptr) { + return; + } + IncomingMessageHandler *_this = reinterpret_cast(userArg); + xSemaphoreGiveFromISR(_this->messageNotify, nullptr); +} + +void IncomingMessageHandler::sendErrorAndResetQueue(ethosu_core_msg_err_type type, const char *message) { + ethosu_core_msg_err error; + error.type = type; + + for (size_t i = 0; i < sizeof(error.msg) && message[i]; i++) { + error.msg[i] = message[i]; + } + printf("ERROR: Msg: \"%s\"\n", error.msg); + + if (!outputMessageQueue.write(ETHOSU_CORE_MSG_ERR, error)) { + printf("ERROR: Msg: Failed to write error response. No mailbox message sent\n"); + } else { + mailbox.sendMessage(); + } + inputMessageQueue.reset(); +} + +bool IncomingMessageHandler::handleInferenceOutput() { + struct ethosu_core_inference_rsp rsp; + if (pdTRUE != xQueueReceive(inferenceOutputQueue, &rsp, 0)) { + return false; + } + + sendInferenceRsp(rsp); + return true; +} + +bool IncomingMessageHandler::handleMessage() { + struct ethosu_core_msg msg; + + if (inputMessageQueue.available() == 0) { + return false; + } + + // Read msg header + // Only process a complete message header, else send error message + // and reset queue + if (!inputMessageQueue.read(msg)) { + sendErrorAndResetQueue(ETHOSU_CORE_MSG_ERR_INVALID_SIZE, "Failed to read a complete header"); + return false; + } + + printf("Msg: header magic=%" PRIX32 ", type=%" PRIu32 ", length=%" PRIu32 "\n", msg.magic, msg.type, msg.length); + + if (msg.magic != ETHOSU_CORE_MSG_MAGIC) { + printf("Msg: Invalid Magic\n"); + sendErrorAndResetQueue(ETHOSU_CORE_MSG_ERR_INVALID_MAGIC, "Invalid magic"); + return false; + } + + switch (msg.type) { + case ETHOSU_CORE_MSG_PING: { + printf("Msg: Ping\n"); + sendPong(); + break; + } + case ETHOSU_CORE_MSG_ERR: { + ethosu_core_msg_err error; + if (!inputMessageQueue.read(error)) { + printf("ERROR: Msg: Failed to receive error message\n"); + } else { + printf("Msg: Received an error response, type=%" PRIu32 ", msg=\"%s\"\n", error.type, error.msg); + } + + inputMessageQueue.reset(); + return false; + } + case ETHOSU_CORE_MSG_VERSION_REQ: { + printf("Msg: Version request\n"); + sendVersionRsp(); + break; + } + case ETHOSU_CORE_MSG_CAPABILITIES_REQ: { + ethosu_core_capabilities_req req; + if (!inputMessageQueue.read(req)) { + sendErrorAndResetQueue(ETHOSU_CORE_MSG_ERR_INVALID_PAYLOAD, "CapabilitiesReq. Failed to read payload"); + break; + } + + printf("Msg: Capabilities request.user_arg=0x%" PRIx64 "\n", req.user_arg); + sendCapabilitiesRsp(req.user_arg); + break; + } + case ETHOSU_CORE_MSG_INFERENCE_REQ: { + ethosu_core_inference_req req; + if (!inputMessageQueue.read(req)) { + sendErrorAndResetQueue(ETHOSU_CORE_MSG_ERR_INVALID_PAYLOAD, "InferenceReq. Failed to read payload"); + break; + } + + printf("Msg: InferenceReq. user_arg=0x%" PRIx64 ", network_type=%" PRIu32 ", ", req.user_arg, req.network.type); + + if (req.network.type == ETHOSU_CORE_NETWORK_BUFFER) { + printf("network.buffer={0x%" PRIx32 ", %" PRIu32 "},\n", req.network.buffer.ptr, req.network.buffer.size); + } else { + printf("network.index=%" PRIu32 ",\n", req.network.index); + } + + printf("ifm_count=%" PRIu32 ", ifm=[", req.ifm_count); + for (uint32_t i = 0; i < req.ifm_count; ++i) { + if (i > 0) { + printf(", "); + } + + printf("{0x%" PRIx32 ", %" PRIu32 "}", req.ifm[i].ptr, req.ifm[i].size); + } + printf("]"); + + printf(", ofm_count=%" PRIu32 ", ofm=[", req.ofm_count); + for (uint32_t i = 0; i < req.ofm_count; ++i) { + if (i > 0) { + printf(", "); + } + + printf("{0x%" PRIx32 ", %" PRIu32 "}", req.ofm[i].ptr, req.ofm[i].size); + } + printf("]\n"); + + if (!inferenceInputQueue->push(req)) { + printf("Msg: Inference queue full. Rejecting inference user_arg=0x%" PRIx64 "\n", req.user_arg); + sendFailedInferenceRsp(req.user_arg, ETHOSU_CORE_STATUS_REJECTED); + } + break; + } + case ETHOSU_CORE_MSG_CANCEL_INFERENCE_REQ: { + ethosu_core_cancel_inference_req req; + if (!inputMessageQueue.read(req)) { + sendErrorAndResetQueue(ETHOSU_CORE_MSG_ERR_INVALID_PAYLOAD, "CancelInferenceReq. Failed to read payload"); + break; + } + printf("Msg: CancelInferenceReq. user_arg=0x%" PRIx64 ", inference_handle=0x%" PRIx64 "\n", + req.user_arg, + req.inference_handle); + + bool found = + inferenceInputQueue->erase([req](auto &inf_req) { return inf_req.user_arg == req.inference_handle; }); + + // NOTE: send an inference response with status ABORTED if the inference has been droped from the queue + if (found) { + sendFailedInferenceRsp(req.inference_handle, ETHOSU_CORE_STATUS_ABORTED); + } + + sendCancelInferenceRsp(req.user_arg, found ? ETHOSU_CORE_STATUS_OK : ETHOSU_CORE_STATUS_ERROR); + break; + } + case ETHOSU_CORE_MSG_NETWORK_INFO_REQ: { + ethosu_core_network_info_req req; + if (!inputMessageQueue.read(req)) { + sendErrorAndResetQueue(ETHOSU_CORE_MSG_ERR_INVALID_PAYLOAD, "NetworkInfoReq. Failed to read payload"); + break; + } + + printf("Msg: NetworkInfoReq. user_arg=0x%" PRIx64 "\n", req.user_arg); + sendNetworkInfoRsp(req.user_arg, req.network); + break; + } + default: { + char errMsg[128]; + snprintf(&errMsg[0], + sizeof(errMsg), + "Msg: Unknown type: %" PRIu32 " with payload length %" PRIu32 " bytes\n", + msg.type, + msg.length); + + sendErrorAndResetQueue(ETHOSU_CORE_MSG_ERR_UNSUPPORTED_TYPE, errMsg); + return false; + } + } + + return true; +} + +void IncomingMessageHandler::sendPong() { + if (!outputMessageQueue.write(ETHOSU_CORE_MSG_PONG)) { + printf("ERROR: Msg: Failed to write pong response. No mailbox message sent\n"); + } else { + mailbox.sendMessage(); + } +} + +void IncomingMessageHandler::sendVersionRsp() { + ethosu_core_msg_version version = { + ETHOSU_CORE_MSG_VERSION_MAJOR, + ETHOSU_CORE_MSG_VERSION_MINOR, + ETHOSU_CORE_MSG_VERSION_PATCH, + 0, + }; + + if (!outputMessageQueue.write(ETHOSU_CORE_MSG_VERSION_RSP, version)) { + printf("ERROR: Failed to write version response. No mailbox message sent\n"); + } else { + mailbox.sendMessage(); + } +} + +void IncomingMessageHandler::sendCapabilitiesRsp(uint64_t userArg) { + capabilities.user_arg = userArg; + + if (!outputMessageQueue.write(ETHOSU_CORE_MSG_CAPABILITIES_RSP, capabilities)) { + printf("ERROR: Failed to write capabilities response. No mailbox message sent\n"); + } else { + mailbox.sendMessage(); + } +} + +void IncomingMessageHandler::sendNetworkInfoRsp(uint64_t userArg, ethosu_core_network_buffer &network) { + ethosu_core_network_info_rsp rsp; + rsp.user_arg = userArg; + rsp.ifm_count = 0; + rsp.ofm_count = 0; + + void *buffer; + size_t size; + + bool failed = networks->getNetwork(network, buffer, size); + + if (!failed) { + failed = parser.parseModel(buffer, + rsp.desc, + InferenceProcess::makeArray(rsp.ifm_size, rsp.ifm_count, ETHOSU_CORE_BUFFER_MAX), + InferenceProcess::makeArray(rsp.ofm_size, rsp.ofm_count, ETHOSU_CORE_BUFFER_MAX)); + } + rsp.status = failed ? ETHOSU_CORE_STATUS_ERROR : ETHOSU_CORE_STATUS_OK; + + if (!outputMessageQueue.write(ETHOSU_CORE_MSG_NETWORK_INFO_RSP, rsp)) { + printf("ERROR: Msg: Failed to write network info response. No mailbox message sent\n"); + } else { + mailbox.sendMessage(); + } +} + +void IncomingMessageHandler::sendInferenceRsp(ethosu_core_inference_rsp &rsp) { + if (!outputMessageQueue.write(ETHOSU_CORE_MSG_INFERENCE_RSP, rsp)) { + printf("ERROR: Msg: Failed to write inference response. No mailbox message sent\n"); + } else { + mailbox.sendMessage(); + } +} + +void IncomingMessageHandler::sendFailedInferenceRsp(uint64_t userArg, uint32_t status) { + ethosu_core_inference_rsp rsp; + rsp.user_arg = userArg; + rsp.status = status; + if (!outputMessageQueue.write(ETHOSU_CORE_MSG_INFERENCE_RSP, rsp)) { + printf("ERROR: Msg: Failed to write inference response. No mailbox message sent\n"); + } else { + mailbox.sendMessage(); + } +} +void IncomingMessageHandler::sendCancelInferenceRsp(uint64_t userArg, uint32_t status) { + ethosu_core_cancel_inference_rsp cancellation; + cancellation.user_arg = userArg; + cancellation.status = status; + if (!outputMessageQueue.write(ETHOSU_CORE_MSG_CANCEL_INFERENCE_RSP, cancellation)) { + printf("ERROR: Msg: Failed to write cancel inference response. No mailbox message sent\n"); + } else { + mailbox.sendMessage(); + } +} + +void IncomingMessageHandler::readCapabilties(ethosu_core_msg_capabilities_rsp &rsp) { + rsp = {}; + +#ifdef ETHOSU + struct ethosu_driver_version version; + ethosu_get_driver_version(&version); + + struct ethosu_hw_info info; + struct ethosu_driver *drv = ethosu_reserve_driver(); + ethosu_get_hw_info(drv, &info); + ethosu_release_driver(drv); + + rsp.user_arg = 0; + rsp.version_status = info.version.version_status; + rsp.version_minor = info.version.version_minor; + rsp.version_major = info.version.version_major; + rsp.product_major = info.version.product_major; + rsp.arch_patch_rev = info.version.arch_patch_rev; + rsp.arch_minor_rev = info.version.arch_minor_rev; + rsp.arch_major_rev = info.version.arch_major_rev; + rsp.driver_patch_rev = version.patch; + rsp.driver_minor_rev = version.minor; + rsp.driver_major_rev = version.major; + rsp.macs_per_cc = info.cfg.macs_per_cc; + rsp.cmd_stream_version = info.cfg.cmd_stream_version; + rsp.custom_dma = info.cfg.custom_dma; +#endif +} + +/**************************************************************************** + * InferenceHandler + ****************************************************************************/ + +InferenceHandler::InferenceHandler(uint8_t *tensorArena, + size_t arenaSize, + std::shared_ptr> _inferenceInputQueue, + QueueHandle_t _inferenceOutputQueue, + SemaphoreHandle_t _messageNotify, + std::shared_ptr _networks) : + inferenceInputQueue(_inferenceInputQueue), + inferenceOutputQueue(_inferenceOutputQueue), messageNotify(_messageNotify), inference(tensorArena, arenaSize), + networks(_networks) {} + +void InferenceHandler::run() { + ethosu_core_inference_req req; + ethosu_core_inference_rsp rsp; + + while (true) { + inferenceInputQueue->pop(req); + + runInference(req, rsp); + + xQueueSend(inferenceOutputQueue, &rsp, portMAX_DELAY); + xSemaphoreGive(messageNotify); + } +} + +void InferenceHandler::runInference(ethosu_core_inference_req &req, ethosu_core_inference_rsp &rsp) { + currentReq = &req; + currentRsp = &rsp; + + /* + * Run inference + */ + + InferenceProcess::InferenceJob job; + bool failed = getInferenceJob(req, job); + + if (!failed) { + job.invalidate(); + failed = inference.runJob(job); + job.clean(); + } + +#if defined(ETHOSU) + /* + * Print PMU counters + */ + + if (!failed) { + const int numEvents = std::min(static_cast(ETHOSU_PMU_Get_NumEventCounters()), ETHOSU_CORE_PMU_MAX); + + for (int i = 0; i < numEvents; i++) { + printf("ethosu_pmu_cntr%d : %" PRIu32 "\n", i, rsp.pmu_event_count[i]); + } + + if (rsp.pmu_cycle_counter_enable) { + printf("ethosu_pmu_cycle_cntr : %" PRIu64 " cycles\n", rsp.pmu_cycle_counter_count); + } + } +#endif + + /* + * Send inference response + */ + + rsp.user_arg = req.user_arg; + rsp.ofm_count = job.output.size(); + rsp.status = failed ? ETHOSU_CORE_STATUS_ERROR : ETHOSU_CORE_STATUS_OK; + + for (size_t i = 0; i < job.output.size(); ++i) { + rsp.ofm_size[i] = job.output[i].size; + } + + currentReq = nullptr; + currentRsp = nullptr; +} + +bool InferenceHandler::getInferenceJob(const ethosu_core_inference_req &req, InferenceProcess::InferenceJob &job) { + bool failed = networks->getNetwork(req.network, job.networkModel.data, job.networkModel.size); + if (failed) { + return true; + } + + for (uint32_t i = 0; i < req.ifm_count; ++i) { + job.input.push_back(InferenceProcess::DataPtr(reinterpret_cast(req.ifm[i].ptr), req.ifm[i].size)); + } + + for (uint32_t i = 0; i < req.ofm_count; ++i) { + job.output.push_back(InferenceProcess::DataPtr(reinterpret_cast(req.ofm[i].ptr), req.ofm[i].size)); + } + + job.externalContext = this; + + return false; +} + +} // namespace MessageHandler + +#if defined(ETHOSU) +extern "C" void ethosu_inference_begin(struct ethosu_driver *drv, void *userArg) { + MessageHandler::InferenceHandler *self = static_cast(userArg); + + // Calculate maximum number of events + const int numEvents = std::min(static_cast(ETHOSU_PMU_Get_NumEventCounters()), ETHOSU_CORE_PMU_MAX); + + // Enable PMU + ETHOSU_PMU_Enable(drv); + + // Configure and enable events + for (int i = 0; i < numEvents; i++) { + ETHOSU_PMU_Set_EVTYPER(drv, i, static_cast(self->currentReq->pmu_event_config[i])); + ETHOSU_PMU_CNTR_Enable(drv, 1 << i); + } + + // Enable cycle counter + if (self->currentReq->pmu_cycle_counter_enable) { + ETHOSU_PMU_PMCCNTR_CFG_Set_Stop_Event(drv, ETHOSU_PMU_NPU_IDLE); + ETHOSU_PMU_PMCCNTR_CFG_Set_Start_Event(drv, ETHOSU_PMU_NPU_ACTIVE); + + ETHOSU_PMU_CNTR_Enable(drv, ETHOSU_PMU_CCNT_Msk); + ETHOSU_PMU_CYCCNT_Reset(drv); + } + + // Reset all counters + ETHOSU_PMU_EVCNTR_ALL_Reset(drv); +} + +extern "C" void ethosu_inference_end(struct ethosu_driver *drv, void *userArg) { + MessageHandler::InferenceHandler *self = static_cast(userArg); + + // Get cycle counter + self->currentRsp->pmu_cycle_counter_enable = self->currentReq->pmu_cycle_counter_enable; + if (self->currentReq->pmu_cycle_counter_enable) { + self->currentRsp->pmu_cycle_counter_count = ETHOSU_PMU_Get_CCNTR(drv); + } + + // Calculate maximum number of events + const int numEvents = std::min(static_cast(ETHOSU_PMU_Get_NumEventCounters()), ETHOSU_CORE_PMU_MAX); + + // Get event counters + int i; + for (i = 0; i < numEvents; i++) { + self->currentRsp->pmu_event_config[i] = self->currentReq->pmu_event_config[i]; + self->currentRsp->pmu_event_count[i] = ETHOSU_PMU_Get_EVCNTR(drv, i); + } + + for (; i < ETHOSU_CORE_PMU_MAX; i++) { + self->currentRsp->pmu_event_config[i] = 0; + self->currentRsp->pmu_event_count[i] = 0; + } + + // Disable PMU + ETHOSU_PMU_Disable(drv); +} +#endif diff --git a/applications/message_handler/lib/message_queue.cpp b/applications/message_handler/lib/message_queue.cpp new file mode 100644 index 0000000..c3890fe --- /dev/null +++ b/applications/message_handler/lib/message_queue.cpp @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2020-2022 Arm Limited. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "message_queue.hpp" + +#include +#include +#include +#include + +namespace MessageQueue { + +QueueImpl::QueueImpl(EthosU::ethosu_core_queue &_queue) : queue(_queue) { + cleanHeaderData(); +} + +bool QueueImpl::empty() const { + invalidateHeaderData(); + + return queue.header.read == queue.header.write; +} + +size_t QueueImpl::available() const { + invalidateHeaderData(); + + size_t avail = queue.header.write - queue.header.read; + + if (queue.header.read > queue.header.write) { + avail += queue.header.size; + } + + return avail; +} + +size_t QueueImpl::capacity() const { + return queue.header.size - available(); +} + +bool QueueImpl::read(uint8_t *dst, uint32_t length) { + const uint8_t *end = dst + length; + + // Available will invalidate the cache + if (length > available()) { + return false; + } + + uint32_t rpos = queue.header.read; + + while (dst < end) { + *dst++ = queue.data[rpos]; + rpos = (rpos + 1) % queue.header.size; + } + + queue.header.read = rpos; + + cleanHeader(); + + return true; +} + +bool QueueImpl::write(const Vec *vec, size_t length) { + size_t total = 0; + + for (size_t i = 0; i < length; i++) { + total += vec[i].length; + } + + invalidateHeader(); + + if (total > capacity()) { + return false; + } + + uint32_t wpos = queue.header.write; + + for (size_t i = 0; i < length; i++) { + const uint8_t *src = reinterpret_cast(vec[i].base); + const uint8_t *end = src + vec[i].length; + + while (src < end) { + queue.data[wpos] = *src++; + wpos = (wpos + 1) % queue.header.size; + } + } + + // Update the write position last + queue.header.write = wpos; + + cleanHeaderData(); + + return true; +} + +bool QueueImpl::write(const uint32_t type, const void *src, uint32_t length) { + EthosU::ethosu_core_msg msg = {ETHOSU_CORE_MSG_MAGIC, type, length}; + Vec vec[2] = {{&msg, sizeof(msg)}, {src, length}}; + + return write(vec, 2); +} + +// Skip to magic or end of queue +void QueueImpl::reset() { + invalidateHeader(); + queue.header.read = queue.header.write; + cleanHeader(); +} + +void QueueImpl::cleanHeader() const { +#if defined(__DCACHE_PRESENT) && (__DCACHE_PRESENT == 1U) + SCB_CleanDCache_by_Addr(reinterpret_cast(&queue.header), sizeof(queue.header)); +#endif +} + +void QueueImpl::cleanHeaderData() const { +#if defined(__DCACHE_PRESENT) && (__DCACHE_PRESENT == 1U) + SCB_CleanDCache_by_Addr(reinterpret_cast(&queue.header), sizeof(queue.header)); + uintptr_t queueDataPtr = reinterpret_cast(&queue.data[0]); + SCB_CleanDCache_by_Addr(reinterpret_cast(queueDataPtr & ~3), queue.header.size + (queueDataPtr & 3)); +#endif +} + +void QueueImpl::invalidateHeader() const { +#if defined(__DCACHE_PRESENT) && (__DCACHE_PRESENT == 1U) + SCB_InvalidateDCache_by_Addr(reinterpret_cast(&queue.header), sizeof(queue.header)); +#endif +} + +void QueueImpl::invalidateHeaderData() const { +#if defined(__DCACHE_PRESENT) && (__DCACHE_PRESENT == 1U) + SCB_InvalidateDCache_by_Addr(reinterpret_cast(&queue.header), sizeof(queue.header)); + uintptr_t queueDataPtr = reinterpret_cast(&queue.data[0]); + SCB_InvalidateDCache_by_Addr(reinterpret_cast(queueDataPtr & ~3), + queue.header.size + (queueDataPtr & 3)); +#endif +} +} // namespace MessageQueue diff --git a/applications/message_handler/main.cpp b/applications/message_handler/main.cpp index 0b4860a..caa778b 100644 --- a/applications/message_handler/main.cpp +++ b/applications/message_handler/main.cpp @@ -29,10 +29,12 @@ #include #include "ethosu_core_interface.h" +#include "indexed_networks.hpp" #include "message_handler.hpp" #include "message_queue.hpp" -#include +#include "networks.hpp" +#include #if defined(MHU_V2) #include #elif defined(MHU_JUNO) @@ -82,94 +84,6 @@ Mailbox::MHUDummy mailbox; } // namespace -/**************************************************************************** - * Override new operators to call in FreeRTOS allocator - ****************************************************************************/ - -void *operator new(size_t size) { - return pvPortMalloc(size); -} - -void *operator new[](size_t size) { - return pvPortMalloc(size); -} - -void operator delete(void *ptr) { - vPortFree(ptr); -} - -void operator delete(void *ptr, std::size_t) { - vPortFree(ptr); -} - -void operator delete[](void *ptr) { - vPortFree(ptr); -} - -void operator delete[](void *ptr, std::size_t) { - vPortFree(ptr); -} - -/**************************************************************************** - * Mutex & Semaphore - ****************************************************************************/ - -extern "C" { - -void *ethosu_mutex_create(void) { - return xSemaphoreCreateMutex(); -} - -int ethosu_mutex_lock(void *mutex) { - SemaphoreHandle_t handle = reinterpret_cast(mutex); - if (xSemaphoreTake(handle, portMAX_DELAY) != pdTRUE) { - printf("Error: Failed to lock mutex.\n"); - return -1; - } - return 0; -} - -int ethosu_mutex_unlock(void *mutex) { - SemaphoreHandle_t handle = reinterpret_cast(mutex); - if (xSemaphoreGive(handle) != pdTRUE) { - printf("Error: Failed to unlock mutex.\n"); - return -1; - } - return 0; -} - -void *ethosu_semaphore_create(void) { - return xSemaphoreCreateBinary(); -} - -int ethosu_semaphore_take(void *sem) { - SemaphoreHandle_t handle = reinterpret_cast(sem); - if (xSemaphoreTake(handle, portMAX_DELAY) != pdTRUE) { - printf("Error: Failed to take semaphore.\n"); - return -1; - } - return 0; -} - -int ethosu_semaphore_give(void *sem) { - SemaphoreHandle_t handle = reinterpret_cast(sem); - if (xPortIsInsideInterrupt()) { - if (xSemaphoreGiveFromISR(handle, NULL) != pdTRUE) { - printf("Error: Failed to give semaphore from ISR.\n"); - return -1; - } - } else { - /* A FreeRTOS binary semaphore is fundamentally a queue that can only hold one item. If the queue is full, - * xSemaphoreGive will return a pdFALSE value. Ignoring the return value in here, as a semaphore give failure - * does not affect the application correctness. */ - if (xSemaphoreGive(handle) != pdTRUE) { - // do nothing - } - } - return 0; -} -} - /**************************************************************************** * Application ****************************************************************************/ @@ -179,13 +93,16 @@ struct TaskParams { TaskParams() : messageNotify(xSemaphoreCreateBinary()), inferenceInputQueue(std::make_shared>()), - inferenceOutputQueue(xQueueCreate(10, sizeof(ethosu_core_inference_rsp))) {} + inferenceOutputQueue(xQueueCreate(10, sizeof(ethosu_core_inference_rsp))), + networks(std::make_shared()) {} SemaphoreHandle_t messageNotify; // Used to pass inference requests to the inference runner task std::shared_ptr> inferenceInputQueue; // Queue for message responses to the remote host QueueHandle_t inferenceOutputQueue; + // Networks provider + std::shared_ptr networks; }; struct InferenceTaskParams { @@ -207,7 +124,8 @@ void inferenceTask(void *pvParameters) { arenaSize, params->taskParams->inferenceInputQueue, params->taskParams->inferenceOutputQueue, - params->taskParams->messageNotify); + params->taskParams->messageNotify, + params->taskParams->networks); process.run(); } @@ -221,7 +139,8 @@ void messageTask(void *pvParameters) { mailbox, params->inferenceInputQueue, params->inferenceOutputQueue, - params->messageNotify); + params->messageNotify, + params->networks); #ifdef MHU_IRQ // Register mailbox interrupt handler diff --git a/applications/message_handler/message_handler.cpp b/applications/message_handler/message_handler.cpp deleted file mode 100644 index 4a3938e..0000000 --- a/applications/message_handler/message_handler.cpp +++ /dev/null @@ -1,616 +0,0 @@ -/* - * Copyright (c) 2020-2022 Arm Limited. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "message_handler.hpp" - -#include "cmsis_compiler.h" - -#ifdef ETHOSU -#include -#include -#endif - -#include "FreeRTOS.h" -#include "queue.h" -#include "semphr.h" - -#include -#include -#include - -#define XSTRINGIFY(src) #src -#define STRINGIFY(src) XSTRINGIFY(src) - -using namespace EthosU; -using namespace MessageQueue; - -/**************************************************************************** - * Models - ****************************************************************************/ - -namespace { -#if defined(__has_include) - -#if __has_include(STRINGIFY(MODEL_0)) -namespace Model0 { -#include STRINGIFY(MODEL_0) -} -#endif - -#if __has_include(STRINGIFY(MODEL_1)) -namespace Model1 { -#include STRINGIFY(MODEL_1) -} -#endif - -#if __has_include(STRINGIFY(MODEL_2)) -namespace Model2 { -#include STRINGIFY(MODEL_2) -} -#endif - -#if __has_include(STRINGIFY(MODEL_3)) -namespace Model3 { -#include STRINGIFY(MODEL_3) -} -#endif - -#endif -} // namespace - -namespace MessageHandler { - -/**************************************************************************** - * IncomingMessageHandler - ****************************************************************************/ - -namespace { -bool getNetwork(const ethosu_core_buffer &buffer, void *&data, size_t &size) { - data = reinterpret_cast(buffer.ptr); - size = buffer.size; - return false; -} - -bool getNetwork(const uint32_t index, void *&data, size_t &size) { - switch (index) { -#if __has_include(STRINGIFY(MODEL_0)) - case 0: - data = reinterpret_cast(Model0::networkModel); - size = sizeof(Model0::networkModel); - break; -#endif - -#if __has_include(STRINGIFY(MODEL_1)) - case 1: - data = reinterpret_cast(Model1::networkModel); - size = sizeof(Model1::networkModel); - break; -#endif - -#if __has_include(STRINGIFY(MODEL_2)) - case 2: - data = reinterpret_cast(Model2::networkModel); - size = sizeof(Model2::networkModel); - break; -#endif - -#if __has_include(STRINGIFY(MODEL_3)) - case 3: - data = reinterpret_cast(Model3::networkModel); - size = sizeof(Model3::networkModel); - break; -#endif - - default: - printf("Error: Network model index out of range. index=%" PRIu32 "\n", index); - return true; - } - - return false; -} - -bool getNetwork(const ethosu_core_network_buffer &buffer, void *&data, size_t &size) { - switch (buffer.type) { - case ETHOSU_CORE_NETWORK_BUFFER: - return getNetwork(buffer.buffer, data, size); - case ETHOSU_CORE_NETWORK_INDEX: - return getNetwork(buffer.index, data, size); - default: - printf("Error: Unsupported network model type. type=%" PRIu32 "\n", buffer.type); - return true; - } -} - -}; // namespace - -IncomingMessageHandler::IncomingMessageHandler( - EthosU::ethosu_core_queue &_inputMessageQueue, - EthosU::ethosu_core_queue &_outputMessageQueue, - Mailbox::Mailbox &_mailbox, - std::shared_ptr> _inferenceInputQueue, - QueueHandle_t _inferenceOutputQueue, - SemaphoreHandle_t _messageNotify) : - inputMessageQueue(_inputMessageQueue), - outputMessageQueue(_outputMessageQueue), mailbox(_mailbox), inferenceInputQueue(_inferenceInputQueue), - inferenceOutputQueue(_inferenceOutputQueue), messageNotify(_messageNotify) { - mailbox.registerCallback(handleIrq, reinterpret_cast(this)); - readCapabilties(capabilities); -} - -void IncomingMessageHandler::run() { - while (true) { - // Wait for event - xSemaphoreTake(messageNotify, portMAX_DELAY); - - // Handle all inference outputs and all messages in queue - while (handleInferenceOutput() || handleMessage()) {} - } -} - -void IncomingMessageHandler::handleIrq(void *userArg) { - if (userArg == nullptr) { - return; - } - IncomingMessageHandler *_this = reinterpret_cast(userArg); - xSemaphoreGiveFromISR(_this->messageNotify, nullptr); -} - -void IncomingMessageHandler::sendErrorAndResetQueue(ethosu_core_msg_err_type type, const char *message) { - ethosu_core_msg_err error; - error.type = type; - - for (size_t i = 0; i < sizeof(error.msg) && message[i]; i++) { - error.msg[i] = message[i]; - } - printf("ERROR: Msg: \"%s\"\n", error.msg); - - if (!outputMessageQueue.write(ETHOSU_CORE_MSG_ERR, error)) { - printf("ERROR: Msg: Failed to write error response. No mailbox message sent\n"); - } else { - mailbox.sendMessage(); - } - inputMessageQueue.reset(); -} - -bool IncomingMessageHandler::handleInferenceOutput() { - struct ethosu_core_inference_rsp rsp; - if (pdTRUE != xQueueReceive(inferenceOutputQueue, &rsp, 0)) { - return false; - } - - sendInferenceRsp(rsp); - return true; -} - -bool IncomingMessageHandler::handleMessage() { - struct ethosu_core_msg msg; - - if (inputMessageQueue.available() == 0) { - return false; - } - - // Read msg header - // Only process a complete message header, else send error message - // and reset queue - if (!inputMessageQueue.read(msg)) { - sendErrorAndResetQueue(ETHOSU_CORE_MSG_ERR_INVALID_SIZE, "Failed to read a complete header"); - return false; - } - - printf("Msg: header magic=%" PRIX32 ", type=%" PRIu32 ", length=%" PRIu32 "\n", msg.magic, msg.type, msg.length); - - if (msg.magic != ETHOSU_CORE_MSG_MAGIC) { - printf("Msg: Invalid Magic\n"); - sendErrorAndResetQueue(ETHOSU_CORE_MSG_ERR_INVALID_MAGIC, "Invalid magic"); - return false; - } - - switch (msg.type) { - case ETHOSU_CORE_MSG_PING: { - printf("Msg: Ping\n"); - sendPong(); - break; - } - case ETHOSU_CORE_MSG_ERR: { - ethosu_core_msg_err error; - if (!inputMessageQueue.read(error)) { - printf("ERROR: Msg: Failed to receive error message\n"); - } else { - printf("Msg: Received an error response, type=%" PRIu32 ", msg=\"%s\"\n", error.type, error.msg); - } - - inputMessageQueue.reset(); - return false; - } - case ETHOSU_CORE_MSG_VERSION_REQ: { - printf("Msg: Version request\n"); - sendVersionRsp(); - break; - } - case ETHOSU_CORE_MSG_CAPABILITIES_REQ: { - ethosu_core_capabilities_req req; - if (!inputMessageQueue.read(req)) { - sendErrorAndResetQueue(ETHOSU_CORE_MSG_ERR_INVALID_PAYLOAD, "CapabilitiesReq. Failed to read payload"); - break; - } - - printf("Msg: Capabilities request.user_arg=0x%" PRIx64 "\n", req.user_arg); - sendCapabilitiesRsp(req.user_arg); - break; - } - case ETHOSU_CORE_MSG_INFERENCE_REQ: { - ethosu_core_inference_req req; - if (!inputMessageQueue.read(req)) { - sendErrorAndResetQueue(ETHOSU_CORE_MSG_ERR_INVALID_PAYLOAD, "InferenceReq. Failed to read payload"); - break; - } - - printf("Msg: InferenceReq. user_arg=0x%" PRIx64 ", network_type=%" PRIu32 ", ", req.user_arg, req.network.type); - - if (req.network.type == ETHOSU_CORE_NETWORK_BUFFER) { - printf("network.buffer={0x%" PRIx32 ", %" PRIu32 "},\n", req.network.buffer.ptr, req.network.buffer.size); - } else { - printf("network.index=%" PRIu32 ",\n", req.network.index); - } - - printf("ifm_count=%" PRIu32 ", ifm=[", req.ifm_count); - for (uint32_t i = 0; i < req.ifm_count; ++i) { - if (i > 0) { - printf(", "); - } - - printf("{0x%" PRIx32 ", %" PRIu32 "}", req.ifm[i].ptr, req.ifm[i].size); - } - printf("]"); - - printf(", ofm_count=%" PRIu32 ", ofm=[", req.ofm_count); - for (uint32_t i = 0; i < req.ofm_count; ++i) { - if (i > 0) { - printf(", "); - } - - printf("{0x%" PRIx32 ", %" PRIu32 "}", req.ofm[i].ptr, req.ofm[i].size); - } - printf("]\n"); - - if (!inferenceInputQueue->push(req)) { - printf("Msg: Inference queue full. Rejecting inference user_arg=0x%" PRIx64 "\n", req.user_arg); - sendFailedInferenceRsp(req.user_arg, ETHOSU_CORE_STATUS_REJECTED); - } - break; - } - case ETHOSU_CORE_MSG_CANCEL_INFERENCE_REQ: { - ethosu_core_cancel_inference_req req; - if (!inputMessageQueue.read(req)) { - sendErrorAndResetQueue(ETHOSU_CORE_MSG_ERR_INVALID_PAYLOAD, "CancelInferenceReq. Failed to read payload"); - break; - } - printf("Msg: CancelInferenceReq. user_arg=0x%" PRIx64 ", inference_handle=0x%" PRIx64 "\n", - req.user_arg, - req.inference_handle); - - bool found = - inferenceInputQueue->erase([req](auto &inf_req) { return inf_req.user_arg == req.inference_handle; }); - - // NOTE: send an inference response with status ABORTED if the inference has been droped from the queue - if (found) { - sendFailedInferenceRsp(req.inference_handle, ETHOSU_CORE_STATUS_ABORTED); - } - - sendCancelInferenceRsp(req.user_arg, found ? ETHOSU_CORE_STATUS_OK : ETHOSU_CORE_STATUS_ERROR); - break; - } - case ETHOSU_CORE_MSG_NETWORK_INFO_REQ: { - ethosu_core_network_info_req req; - if (!inputMessageQueue.read(req)) { - sendErrorAndResetQueue(ETHOSU_CORE_MSG_ERR_INVALID_PAYLOAD, "NetworkInfoReq. Failed to read payload"); - break; - } - - printf("Msg: NetworkInfoReq. user_arg=0x%" PRIx64 "\n", req.user_arg); - sendNetworkInfoRsp(req.user_arg, req.network); - break; - } - default: { - char errMsg[128]; - snprintf(&errMsg[0], - sizeof(errMsg), - "Msg: Unknown type: %" PRIu32 " with payload length %" PRIu32 " bytes\n", - msg.type, - msg.length); - - sendErrorAndResetQueue(ETHOSU_CORE_MSG_ERR_UNSUPPORTED_TYPE, errMsg); - return false; - } - } - - return true; -} - -void IncomingMessageHandler::sendPong() { - if (!outputMessageQueue.write(ETHOSU_CORE_MSG_PONG)) { - printf("ERROR: Msg: Failed to write pong response. No mailbox message sent\n"); - } else { - mailbox.sendMessage(); - } -} - -void IncomingMessageHandler::sendVersionRsp() { - ethosu_core_msg_version version = { - ETHOSU_CORE_MSG_VERSION_MAJOR, - ETHOSU_CORE_MSG_VERSION_MINOR, - ETHOSU_CORE_MSG_VERSION_PATCH, - 0, - }; - - if (!outputMessageQueue.write(ETHOSU_CORE_MSG_VERSION_RSP, version)) { - printf("ERROR: Failed to write version response. No mailbox message sent\n"); - } else { - mailbox.sendMessage(); - } -} - -void IncomingMessageHandler::sendCapabilitiesRsp(uint64_t userArg) { - capabilities.user_arg = userArg; - - if (!outputMessageQueue.write(ETHOSU_CORE_MSG_CAPABILITIES_RSP, capabilities)) { - printf("ERROR: Failed to write capabilities response. No mailbox message sent\n"); - } else { - mailbox.sendMessage(); - } -} - -void IncomingMessageHandler::sendNetworkInfoRsp(uint64_t userArg, ethosu_core_network_buffer &network) { - ethosu_core_network_info_rsp rsp; - rsp.user_arg = userArg; - rsp.ifm_count = 0; - rsp.ofm_count = 0; - - void *buffer; - size_t size; - - bool failed = getNetwork(network, buffer, size); - - if (!failed) { - failed = parser.parseModel(buffer, - rsp.desc, - InferenceProcess::makeArray(rsp.ifm_size, rsp.ifm_count, ETHOSU_CORE_BUFFER_MAX), - InferenceProcess::makeArray(rsp.ofm_size, rsp.ofm_count, ETHOSU_CORE_BUFFER_MAX)); - } - rsp.status = failed ? ETHOSU_CORE_STATUS_ERROR : ETHOSU_CORE_STATUS_OK; - - if (!outputMessageQueue.write(ETHOSU_CORE_MSG_NETWORK_INFO_RSP, rsp)) { - printf("ERROR: Msg: Failed to write network info response. No mailbox message sent\n"); - } else { - mailbox.sendMessage(); - } -} - -void IncomingMessageHandler::sendInferenceRsp(ethosu_core_inference_rsp &rsp) { - if (!outputMessageQueue.write(ETHOSU_CORE_MSG_INFERENCE_RSP, rsp)) { - printf("ERROR: Msg: Failed to write inference response. No mailbox message sent\n"); - } else { - mailbox.sendMessage(); - } -} - -void IncomingMessageHandler::sendFailedInferenceRsp(uint64_t userArg, uint32_t status) { - ethosu_core_inference_rsp rsp; - rsp.user_arg = userArg; - rsp.status = status; - if (!outputMessageQueue.write(ETHOSU_CORE_MSG_INFERENCE_RSP, rsp)) { - printf("ERROR: Msg: Failed to write inference response. No mailbox message sent\n"); - } else { - mailbox.sendMessage(); - } -} -void IncomingMessageHandler::sendCancelInferenceRsp(uint64_t userArg, uint32_t status) { - ethosu_core_cancel_inference_rsp cancellation; - cancellation.user_arg = userArg; - cancellation.status = status; - if (!outputMessageQueue.write(ETHOSU_CORE_MSG_CANCEL_INFERENCE_RSP, cancellation)) { - printf("ERROR: Msg: Failed to write cancel inference response. No mailbox message sent\n"); - } else { - mailbox.sendMessage(); - } -} - -void IncomingMessageHandler::readCapabilties(ethosu_core_msg_capabilities_rsp &rsp) { - rsp = {}; - -#ifdef ETHOSU - struct ethosu_driver_version version; - ethosu_get_driver_version(&version); - - struct ethosu_hw_info info; - struct ethosu_driver *drv = ethosu_reserve_driver(); - ethosu_get_hw_info(drv, &info); - ethosu_release_driver(drv); - - rsp.user_arg = 0; - rsp.version_status = info.version.version_status; - rsp.version_minor = info.version.version_minor; - rsp.version_major = info.version.version_major; - rsp.product_major = info.version.product_major; - rsp.arch_patch_rev = info.version.arch_patch_rev; - rsp.arch_minor_rev = info.version.arch_minor_rev; - rsp.arch_major_rev = info.version.arch_major_rev; - rsp.driver_patch_rev = version.patch; - rsp.driver_minor_rev = version.minor; - rsp.driver_major_rev = version.major; - rsp.macs_per_cc = info.cfg.macs_per_cc; - rsp.cmd_stream_version = info.cfg.cmd_stream_version; - rsp.custom_dma = info.cfg.custom_dma; -#endif -} - -/**************************************************************************** - * InferenceHandler - ****************************************************************************/ - -InferenceHandler::InferenceHandler(uint8_t *tensorArena, - size_t arenaSize, - std::shared_ptr> _inferenceInputQueue, - QueueHandle_t _inferenceOutputQueue, - SemaphoreHandle_t _messageNotify) : - inferenceInputQueue(_inferenceInputQueue), - inferenceOutputQueue(_inferenceOutputQueue), messageNotify(_messageNotify), inference(tensorArena, arenaSize) {} - -void InferenceHandler::run() { - ethosu_core_inference_req req; - ethosu_core_inference_rsp rsp; - - while (true) { - inferenceInputQueue->pop(req); - - runInference(req, rsp); - - xQueueSend(inferenceOutputQueue, &rsp, portMAX_DELAY); - xSemaphoreGive(messageNotify); - } -} - -void InferenceHandler::runInference(ethosu_core_inference_req &req, ethosu_core_inference_rsp &rsp) { - currentReq = &req; - currentRsp = &rsp; - - /* - * Run inference - */ - - InferenceProcess::InferenceJob job; - bool failed = getInferenceJob(req, job); - - if (!failed) { - job.invalidate(); - failed = inference.runJob(job); - job.clean(); - } - -#if defined(ETHOSU) - /* - * Print PMU counters - */ - - if (!failed) { - const int numEvents = std::min(static_cast(ETHOSU_PMU_Get_NumEventCounters()), ETHOSU_CORE_PMU_MAX); - - for (int i = 0; i < numEvents; i++) { - printf("ethosu_pmu_cntr%d : %" PRIu32 "\n", i, rsp.pmu_event_count[i]); - } - - if (rsp.pmu_cycle_counter_enable) { - printf("ethosu_pmu_cycle_cntr : %" PRIu64 " cycles\n", rsp.pmu_cycle_counter_count); - } - } -#endif - - /* - * Send inference response - */ - - rsp.user_arg = req.user_arg; - rsp.ofm_count = job.output.size(); - rsp.status = failed ? ETHOSU_CORE_STATUS_ERROR : ETHOSU_CORE_STATUS_OK; - - for (size_t i = 0; i < job.output.size(); ++i) { - rsp.ofm_size[i] = job.output[i].size; - } - - currentReq = nullptr; - currentRsp = nullptr; -} - -bool InferenceHandler::getInferenceJob(const ethosu_core_inference_req &req, InferenceProcess::InferenceJob &job) { - bool failed = getNetwork(req.network, job.networkModel.data, job.networkModel.size); - if (failed) { - return true; - } - - for (uint32_t i = 0; i < req.ifm_count; ++i) { - job.input.push_back(InferenceProcess::DataPtr(reinterpret_cast(req.ifm[i].ptr), req.ifm[i].size)); - } - - for (uint32_t i = 0; i < req.ofm_count; ++i) { - job.output.push_back(InferenceProcess::DataPtr(reinterpret_cast(req.ofm[i].ptr), req.ofm[i].size)); - } - - job.externalContext = this; - - return false; -} - -} // namespace MessageHandler - -#if defined(ETHOSU) -extern "C" void ethosu_inference_begin(struct ethosu_driver *drv, void *userArg) { - MessageHandler::InferenceHandler *self = static_cast(userArg); - - // Calculate maximum number of events - const int numEvents = std::min(static_cast(ETHOSU_PMU_Get_NumEventCounters()), ETHOSU_CORE_PMU_MAX); - - // Enable PMU - ETHOSU_PMU_Enable(drv); - - // Configure and enable events - for (int i = 0; i < numEvents; i++) { - ETHOSU_PMU_Set_EVTYPER(drv, i, static_cast(self->currentReq->pmu_event_config[i])); - ETHOSU_PMU_CNTR_Enable(drv, 1 << i); - } - - // Enable cycle counter - if (self->currentReq->pmu_cycle_counter_enable) { - ETHOSU_PMU_PMCCNTR_CFG_Set_Stop_Event(drv, ETHOSU_PMU_NPU_IDLE); - ETHOSU_PMU_PMCCNTR_CFG_Set_Start_Event(drv, ETHOSU_PMU_NPU_ACTIVE); - - ETHOSU_PMU_CNTR_Enable(drv, ETHOSU_PMU_CCNT_Msk); - ETHOSU_PMU_CYCCNT_Reset(drv); - } - - // Reset all counters - ETHOSU_PMU_EVCNTR_ALL_Reset(drv); -} - -extern "C" void ethosu_inference_end(struct ethosu_driver *drv, void *userArg) { - MessageHandler::InferenceHandler *self = static_cast(userArg); - - // Get cycle counter - self->currentRsp->pmu_cycle_counter_enable = self->currentReq->pmu_cycle_counter_enable; - if (self->currentReq->pmu_cycle_counter_enable) { - self->currentRsp->pmu_cycle_counter_count = ETHOSU_PMU_Get_CCNTR(drv); - } - - // Calculate maximum number of events - const int numEvents = std::min(static_cast(ETHOSU_PMU_Get_NumEventCounters()), ETHOSU_CORE_PMU_MAX); - - // Get event counters - int i; - for (i = 0; i < numEvents; i++) { - self->currentRsp->pmu_event_config[i] = self->currentReq->pmu_event_config[i]; - self->currentRsp->pmu_event_count[i] = ETHOSU_PMU_Get_EVCNTR(drv, i); - } - - for (; i < ETHOSU_CORE_PMU_MAX; i++) { - self->currentRsp->pmu_event_config[i] = 0; - self->currentRsp->pmu_event_count[i] = 0; - } - - // Disable PMU - ETHOSU_PMU_Disable(drv); -} -#endif diff --git a/applications/message_handler/message_handler.hpp b/applications/message_handler/message_handler.hpp deleted file mode 100644 index dd05059..0000000 --- a/applications/message_handler/message_handler.hpp +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Copyright (c) 2020-2022 Arm Limited. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef MESSAGE_HANDLER_H -#define MESSAGE_HANDLER_H - -#include "FreeRTOS.h" -#include "queue.h" -#include "semphr.h" - -#include "message_queue.hpp" -#include -#if defined(ETHOSU) -#include -#endif -#include -#include -#include - -#include -#include -#include -#include -#include - -namespace MessageHandler { - -template -class Queue { -public: - using Predicate = std::function; - - Queue() { - mutex = xSemaphoreCreateMutex(); - size = xSemaphoreCreateCounting(capacity, 0u); - - if (mutex == nullptr || size == nullptr) { - printf("Error: failed to allocate memory for inference queue\n"); - } - } - - ~Queue() { - vSemaphoreDelete(mutex); - vSemaphoreDelete(size); - } - - bool push(const T &data) { - xSemaphoreTake(mutex, portMAX_DELAY); - if (list.size() >= capacity) { - xSemaphoreGive(mutex); - return false; - } - - list.push_back(data); - xSemaphoreGive(mutex); - - // increase number of available inferences to pop - xSemaphoreGive(size); - return true; - } - - void pop(T &data) { - // decrease the number of available inferences to pop - xSemaphoreTake(size, portMAX_DELAY); - - xSemaphoreTake(mutex, portMAX_DELAY); - data = list.front(); - list.pop_front(); - xSemaphoreGive(mutex); - } - - bool erase(Predicate pred) { - // let's optimistically assume we are removing an inference, so decrease pop - if (pdFALSE == xSemaphoreTake(size, 0)) { - // if there are no inferences return immediately - return false; - } - - xSemaphoreTake(mutex, portMAX_DELAY); - auto found = std::find_if(list.begin(), list.end(), pred); - bool erased = found != list.end(); - if (erased) { - list.erase(found); - } - xSemaphoreGive(mutex); - - if (!erased) { - // no inference erased, so let's put the size count back - xSemaphoreGive(size); - } - - return erased; - } - -private: - std::list list; - - SemaphoreHandle_t mutex; - SemaphoreHandle_t size; -}; - -class IncomingMessageHandler { -public: - IncomingMessageHandler(EthosU::ethosu_core_queue &inputMessageQueue, - EthosU::ethosu_core_queue &outputMessageQueue, - Mailbox::Mailbox &mailbox, - std::shared_ptr> inferenceInputQueue, - QueueHandle_t inferenceOutputQueue, - SemaphoreHandle_t messageNotify); - - void run(); - -private: - bool handleMessage(); - bool handleInferenceOutput(); - static void handleIrq(void *userArg); - - void sendPong(); - void sendErrorAndResetQueue(EthosU::ethosu_core_msg_err_type type, const char *message); - void sendVersionRsp(); - void sendCapabilitiesRsp(uint64_t userArg); - void sendNetworkInfoRsp(uint64_t userArg, EthosU::ethosu_core_network_buffer &network); - void sendInferenceRsp(EthosU::ethosu_core_inference_rsp &inference); - void sendFailedInferenceRsp(uint64_t userArg, uint32_t status); - void sendCancelInferenceRsp(uint64_t userArg, uint32_t status); - void readCapabilties(EthosU::ethosu_core_msg_capabilities_rsp &rsp); - - MessageQueue::QueueImpl inputMessageQueue; - MessageQueue::QueueImpl outputMessageQueue; - Mailbox::Mailbox &mailbox; - InferenceProcess::InferenceParser parser; - std::shared_ptr> inferenceInputQueue; - QueueHandle_t inferenceOutputQueue; - SemaphoreHandle_t messageNotify; - EthosU::ethosu_core_msg_capabilities_rsp capabilities; -}; - -class InferenceHandler { -public: - InferenceHandler(uint8_t *tensorArena, - size_t arenaSize, - std::shared_ptr> inferenceInputQueue, - QueueHandle_t inferenceOutputQueue, - SemaphoreHandle_t messageNotify); - - void run(); - -private: - void runInference(EthosU::ethosu_core_inference_req &req, EthosU::ethosu_core_inference_rsp &rsp); - bool getInferenceJob(const EthosU::ethosu_core_inference_req &req, InferenceProcess::InferenceJob &job); - -#if defined(ETHOSU) - friend void ::ethosu_inference_begin(struct ethosu_driver *drv, void *userArg); - friend void ::ethosu_inference_end(struct ethosu_driver *drv, void *userArg); -#endif - std::shared_ptr> inferenceInputQueue; - QueueHandle_t inferenceOutputQueue; - SemaphoreHandle_t messageNotify; - InferenceProcess::InferenceProcess inference; - EthosU::ethosu_core_inference_req *currentReq; - EthosU::ethosu_core_inference_rsp *currentRsp; -}; - -} // namespace MessageHandler - -#endif diff --git a/applications/message_handler/message_queue.cpp b/applications/message_handler/message_queue.cpp deleted file mode 100644 index c3890fe..0000000 --- a/applications/message_handler/message_queue.cpp +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright (c) 2020-2022 Arm Limited. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "message_queue.hpp" - -#include -#include -#include -#include - -namespace MessageQueue { - -QueueImpl::QueueImpl(EthosU::ethosu_core_queue &_queue) : queue(_queue) { - cleanHeaderData(); -} - -bool QueueImpl::empty() const { - invalidateHeaderData(); - - return queue.header.read == queue.header.write; -} - -size_t QueueImpl::available() const { - invalidateHeaderData(); - - size_t avail = queue.header.write - queue.header.read; - - if (queue.header.read > queue.header.write) { - avail += queue.header.size; - } - - return avail; -} - -size_t QueueImpl::capacity() const { - return queue.header.size - available(); -} - -bool QueueImpl::read(uint8_t *dst, uint32_t length) { - const uint8_t *end = dst + length; - - // Available will invalidate the cache - if (length > available()) { - return false; - } - - uint32_t rpos = queue.header.read; - - while (dst < end) { - *dst++ = queue.data[rpos]; - rpos = (rpos + 1) % queue.header.size; - } - - queue.header.read = rpos; - - cleanHeader(); - - return true; -} - -bool QueueImpl::write(const Vec *vec, size_t length) { - size_t total = 0; - - for (size_t i = 0; i < length; i++) { - total += vec[i].length; - } - - invalidateHeader(); - - if (total > capacity()) { - return false; - } - - uint32_t wpos = queue.header.write; - - for (size_t i = 0; i < length; i++) { - const uint8_t *src = reinterpret_cast(vec[i].base); - const uint8_t *end = src + vec[i].length; - - while (src < end) { - queue.data[wpos] = *src++; - wpos = (wpos + 1) % queue.header.size; - } - } - - // Update the write position last - queue.header.write = wpos; - - cleanHeaderData(); - - return true; -} - -bool QueueImpl::write(const uint32_t type, const void *src, uint32_t length) { - EthosU::ethosu_core_msg msg = {ETHOSU_CORE_MSG_MAGIC, type, length}; - Vec vec[2] = {{&msg, sizeof(msg)}, {src, length}}; - - return write(vec, 2); -} - -// Skip to magic or end of queue -void QueueImpl::reset() { - invalidateHeader(); - queue.header.read = queue.header.write; - cleanHeader(); -} - -void QueueImpl::cleanHeader() const { -#if defined(__DCACHE_PRESENT) && (__DCACHE_PRESENT == 1U) - SCB_CleanDCache_by_Addr(reinterpret_cast(&queue.header), sizeof(queue.header)); -#endif -} - -void QueueImpl::cleanHeaderData() const { -#if defined(__DCACHE_PRESENT) && (__DCACHE_PRESENT == 1U) - SCB_CleanDCache_by_Addr(reinterpret_cast(&queue.header), sizeof(queue.header)); - uintptr_t queueDataPtr = reinterpret_cast(&queue.data[0]); - SCB_CleanDCache_by_Addr(reinterpret_cast(queueDataPtr & ~3), queue.header.size + (queueDataPtr & 3)); -#endif -} - -void QueueImpl::invalidateHeader() const { -#if defined(__DCACHE_PRESENT) && (__DCACHE_PRESENT == 1U) - SCB_InvalidateDCache_by_Addr(reinterpret_cast(&queue.header), sizeof(queue.header)); -#endif -} - -void QueueImpl::invalidateHeaderData() const { -#if defined(__DCACHE_PRESENT) && (__DCACHE_PRESENT == 1U) - SCB_InvalidateDCache_by_Addr(reinterpret_cast(&queue.header), sizeof(queue.header)); - uintptr_t queueDataPtr = reinterpret_cast(&queue.data[0]); - SCB_InvalidateDCache_by_Addr(reinterpret_cast(queueDataPtr & ~3), - queue.header.size + (queueDataPtr & 3)); -#endif -} -} // namespace MessageQueue diff --git a/applications/message_handler/message_queue.hpp b/applications/message_handler/message_queue.hpp deleted file mode 100644 index 4140c62..0000000 --- a/applications/message_handler/message_queue.hpp +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2020-2022 Arm Limited. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef MESSAGE_QUEUE_H -#define MESSAGE_QUEUE_H - -#include -#include - -namespace MessageQueue { - -template -struct Queue { - EthosU::ethosu_core_queue_header header; - uint8_t data[SIZE]; - - constexpr Queue() : header({SIZE, 0, 0}) {} - - constexpr EthosU::ethosu_core_queue *toQueue() { - return reinterpret_cast(&header); - } -}; - -class QueueImpl { -public: - struct Vec { - const void *base; - size_t length; - }; - - QueueImpl(EthosU::ethosu_core_queue &queue); - - bool empty() const; - size_t available() const; - size_t capacity() const; - void reset(); - bool read(uint8_t *dst, uint32_t length); - template - bool read(T &dst) { - return read(reinterpret_cast(&dst), sizeof(dst)); - } - bool write(const Vec *vec, size_t length); - bool write(const uint32_t type, const void *src = nullptr, uint32_t length = 0); - template - bool write(const uint32_t type, const T &src) { - return write(type, reinterpret_cast(&src), sizeof(src)); - } - -private: - void cleanHeader() const; - void cleanHeaderData() const; - void invalidateHeader() const; - void invalidateHeaderData() const; - - EthosU::ethosu_core_queue &queue; -}; -} // namespace MessageQueue - -#endif diff --git a/applications/message_handler/model_template.hpp b/applications/message_handler/model_template.hpp deleted file mode 100644 index 353d7d3..0000000 --- a/applications/message_handler/model_template.hpp +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2022 Arm Limited. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -__attribute__((section(".sram.data"), aligned(16))) uint8_t networkModel[] = { - /* Add network model here */ -}; -- cgit v1.2.1