From 6d2e5b7b85573a2b22a4a165400574cff99d4065 Mon Sep 17 00:00:00 2001 From: Davide Grohmann Date: Wed, 24 Aug 2022 17:01:40 +0200 Subject: Add cancel inference tests These tests are run with a special custom firmware Change-Id: I0dfcae1fbaa1287d4b52fd8f6c802bebb5248d3b --- tests/CMakeLists.txt | 27 +++--- tests/cancel_inference_test.cpp | 158 +++++++++++++++++++++++++++++++++ tests/main.cpp | 192 ---------------------------------------- tests/run_inference_test.cpp | 164 ++++++++++++++++++++++++++++++++++ tests/test_assertions.hpp | 62 +++++++++++++ 5 files changed, 400 insertions(+), 203 deletions(-) create mode 100644 tests/cancel_inference_test.cpp delete mode 100644 tests/main.cpp create mode 100644 tests/run_inference_test.cpp create mode 100644 tests/test_assertions.hpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 37b7d8b..4b18862 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -20,18 +20,23 @@ set(CORE_PLATFORM_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../../core_platform) file(GLOB models LIST_DIRECTORIES true "${CORE_PLATFORM_PATH}/applications/baremetal/models/${ETHOSU_TARGET_NPU_CONFIG}/*") -# Build executable -foreach(model ${models}) - get_filename_component(modelname ${model} NAME) +function(add_test_executable testname) + # Build executable + foreach(model ${models}) + get_filename_component(modelname ${model} NAME) - add_executable(lds_${modelname}_tests "main.cpp") + add_executable(lds_${testname}_${modelname} ${testname}.cpp) - target_include_directories(lds_${modelname}_tests PRIVATE - ${model}) + target_include_directories(lds_${testname}_${modelname} PRIVATE + ${model}) - # Link agains ethosu library - target_link_libraries(lds_${modelname}_tests PRIVATE ethosu) + # Link agains ethosu library + target_link_libraries(lds_${testname}_${modelname} PRIVATE ethosu) - # Install target - install(TARGETS lds_${modelname}_tests DESTINATION "bin") -endforeach() + # Install target + install(TARGETS lds_${testname}_${modelname} DESTINATION "bin") + endforeach() +endfunction() + +add_test_executable(run_inference_test) +add_test_executable(cancel_inference_test) diff --git a/tests/cancel_inference_test.cpp b/tests/cancel_inference_test.cpp new file mode 100644 index 0000000..36514f4 --- /dev/null +++ b/tests/cancel_inference_test.cpp @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "input.h" +#include "model.h" +#include "output.h" +#include "test_assertions.hpp" + +using namespace EthosU; + +namespace { + +int64_t defaultTimeout = 60000000000; + +void testCancelInference(const Device &device) { + try { + auto networkBuffer = std::make_shared(device, sizeof(networkModelData)); + networkBuffer->resize(sizeof(networkModelData)); + std::memcpy(networkBuffer->data(), networkModelData, sizeof(networkModelData)); + auto network = std::make_shared(device, networkBuffer); + + std::vector> inputBuffers; + std::vector> outputBuffers; + + auto inputBuffer = std::make_shared(device, sizeof(inputData)); + inputBuffer->resize(sizeof(inputData)); + std::memcpy(inputBuffer->data(), inputData, sizeof(inputData)); + + inputBuffers.push_back(inputBuffer); + outputBuffers.push_back(std::make_shared(device, sizeof(expectedOutputData))); + std::vector enabledCounters(Inference::getMaxPmuEventCounters()); + + auto inference = std::make_shared(network, + inputBuffers.begin(), + inputBuffers.end(), + outputBuffers.begin(), + outputBuffers.end(), + enabledCounters, + false); + + InferenceStatus status = inference->status(); + TEST_ASSERT(status == InferenceStatus::RUNNING); + + bool success = inference->cancel(); + TEST_ASSERT(success); + + status = inference->status(); + TEST_ASSERT(status == InferenceStatus::ABORTED); + + bool timedout = inference->wait(defaultTimeout); + TEST_ASSERT(!timedout); + + } catch (std::exception &e) { throw TestFailureException("Inference run test: ", e.what()); } +} + +void testRejectInference(const Device &device) { + try { + auto networkBuffer = std::make_shared(device, sizeof(networkModelData)); + networkBuffer->resize(sizeof(networkModelData)); + std::memcpy(networkBuffer->data(), networkModelData, sizeof(networkModelData)); + auto network = std::make_shared(device, networkBuffer); + + std::vector> inputBuffers; + std::vector> outputBuffers; + + auto inputBuffer = std::make_shared(device, sizeof(inputData)); + inputBuffer->resize(sizeof(inputData)); + std::memcpy(inputBuffer->data(), inputData, sizeof(inputData)); + + inputBuffers.push_back(inputBuffer); + outputBuffers.push_back(std::make_shared(device, sizeof(expectedOutputData))); + std::vector enabledCounters(Inference::getMaxPmuEventCounters()); + + std::shared_ptr inferences[5]; + + for (int i = 0; i < 5; i++) { + inferences[i] = std::make_shared(network, + inputBuffers.begin(), + inputBuffers.end(), + outputBuffers.begin(), + outputBuffers.end(), + enabledCounters, + false); + + InferenceStatus status = inferences[i]->status(); + TEST_ASSERT(status == InferenceStatus::RUNNING); + } + + auto inference = std::make_shared(network, + inputBuffers.begin(), + inputBuffers.end(), + outputBuffers.begin(), + outputBuffers.end(), + enabledCounters, + false); + + bool timedout = inference->wait(defaultTimeout); + TEST_ASSERT(!timedout); + + InferenceStatus status = inference->status(); + TEST_ASSERT(status == InferenceStatus::REJECTED); + + for (int i = 0; i < 5; i++) { + + bool success = inferences[i]->cancel(); + TEST_ASSERT(success); + + InferenceStatus status = inferences[i]->status(); + TEST_ASSERT(status == InferenceStatus::ABORTED); + + bool timedout = inference->wait(defaultTimeout); + TEST_ASSERT(!timedout); + } + } catch (std::exception &e) { throw TestFailureException("Inference run test: ", e.what()); } +} + +} // namespace + +int main() { + Device device; + + try { + testCancelInference(device); + testRejectInference(device); + } catch (TestFailureException &e) { + std::cerr << "Test failure: " << e.what() << std::endl; + return 1; + } + + return 0; +} diff --git a/tests/main.cpp b/tests/main.cpp deleted file mode 100644 index 43300d3..0000000 --- a/tests/main.cpp +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Copyright (c) 2022 Arm Limited. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "input.h" -#include "model.h" -#include "output.h" - -#define TEST_ASSERT(v) \ - do { \ - if (!(v)) { \ - throw TestFailureException(__FILE__, ":", __LINE__, " ERROR test failed: '", #v, "'"); \ - } \ - } while (0) - -#define FAIL() TEST_ASSERT(false) - -using namespace EthosU; - -namespace { - -int64_t defaultTimeout = 60000000000; - -template -std::string string_format(std::ostringstream &stringStream) { - return stringStream.str(); -} - -template -std::string string_format(std::ostringstream &stringStream, T t, Args... args) { - stringStream << t; - return string_format(stringStream, args...); -} - -class TestFailureException : public std::exception { -public: - template - TestFailureException(const char *msg, Args... args) { - std::ostringstream stringStream; - this->msg = string_format(stringStream, msg, args...); - } - const char *what() const throw() { - return msg.c_str(); - } - -private: - std::string msg; -}; - -void testPing(const Device &device) { - int r; - try { - r = device.ioctl(ETHOSU_IOCTL_PING); - } catch (std::exception &e) { throw TestFailureException("Ping test: ", e.what()); } - - TEST_ASSERT(r == 0); -} - -void testVersion(const Device &device) { - int r; - try { - r = device.ioctl(ETHOSU_IOCTL_VERSION_REQ); - } catch (std::exception &e) { throw TestFailureException("Version test: ", e.what()); } - - TEST_ASSERT(r == 0); -} - -void testCapabilties(const Device &device) { - Capabilities capabilities; - try { - capabilities = device.capabilities(); - } catch (std::exception &e) { throw TestFailureException("Capabilities test: ", e.what()); } - - TEST_ASSERT(capabilities.hwId.architecture > SemanticVersion()); -} - -void testNetworkInfoNotExistentIndex(const Device &device) { - try { - Network(device, 0); - FAIL(); - } catch (Exception &e) { - // good it should have thrown - } catch (std::exception &e) { throw TestFailureException("NetworkInfo no index test: ", e.what()); } -} - -void testNetworkInfoBuffer(const Device &device) { - try { - std::shared_ptr buffer = std::make_shared(device, sizeof(networkModelData)); - buffer->resize(sizeof(networkModelData)); - std::memcpy(buffer->data(), networkModelData, sizeof(networkModelData)); - Network network(device, buffer); - - TEST_ASSERT(network.getIfmDims().size() == 1); - TEST_ASSERT(network.getOfmDims().size() == 1); - } catch (std::exception &e) { throw TestFailureException("NetworkInfo buffer test: ", e.what()); } -} - -void testNetworkInfoUnparsableBuffer(const Device &device) { - try { - auto buffer = std::make_shared(device, sizeof(networkModelData) / 4); - buffer->resize(sizeof(networkModelData) / 4); - std::memcpy(buffer->data(), networkModelData + sizeof(networkModelData) / 4, sizeof(networkModelData) / 4); - - try { - Network network(device, buffer); - FAIL(); - } catch (Exception) { - // good, it should have thrown! - } - } catch (std::exception &e) { throw TestFailureException("NetworkInfo unparsable buffer test: ", e.what()); } -} - -void testRunInferenceBuffer(const Device &device) { - try { - auto networkBuffer = std::make_shared(device, sizeof(networkModelData)); - networkBuffer->resize(sizeof(networkModelData)); - std::memcpy(networkBuffer->data(), networkModelData, sizeof(networkModelData)); - auto network = std::make_shared(device, networkBuffer); - - std::vector> inputBuffers; - std::vector> outputBuffers; - - auto inputBuffer = std::make_shared(device, sizeof(inputData)); - inputBuffer->resize(sizeof(inputData)); - std::memcpy(inputBuffer->data(), inputData, sizeof(inputData)); - - inputBuffers.push_back(inputBuffer); - outputBuffers.push_back(std::make_shared(device, sizeof(expectedOutputData))); - std::vector enabledCounters(Inference::getMaxPmuEventCounters()); - - auto inference = std::make_shared(network, - inputBuffers.begin(), - inputBuffers.end(), - outputBuffers.begin(), - outputBuffers.end(), - enabledCounters, - false); - - bool timedout = inference->wait(defaultTimeout); - TEST_ASSERT(!timedout); - - TEST_ASSERT(std::memcmp(expectedOutputData, outputBuffers[0]->data(), sizeof(expectedOutputData)) == 0); - - } catch (std::exception &e) { throw TestFailureException("Inference run test: ", e.what()); } -} - -} // namespace - -int main() { - Device device; - - try { - testPing(device); - testVersion(device); - testCapabilties(device); - testNetworkInfoNotExistentIndex(device); - testNetworkInfoBuffer(device); - testNetworkInfoUnparsableBuffer(device); - testRunInferenceBuffer(device); - } catch (TestFailureException &e) { - std::cerr << "Test failure: " << e.what() << std::endl; - return 1; - } - - return 0; -} diff --git a/tests/run_inference_test.cpp b/tests/run_inference_test.cpp new file mode 100644 index 0000000..94bb499 --- /dev/null +++ b/tests/run_inference_test.cpp @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "input.h" +#include "model.h" +#include "output.h" +#include "test_assertions.hpp" + +using namespace EthosU; + +namespace { + +int64_t defaultTimeout = 60000000000; + +void testPing(const Device &device) { + int r; + try { + r = device.ioctl(ETHOSU_IOCTL_PING); + } catch (std::exception &e) { throw TestFailureException("Ping test: ", e.what()); } + + TEST_ASSERT(r == 0); +} + +void testVersion(const Device &device) { + int r; + try { + r = device.ioctl(ETHOSU_IOCTL_VERSION_REQ); + } catch (std::exception &e) { throw TestFailureException("Version test: ", e.what()); } + + TEST_ASSERT(r == 0); +} + +void testCapabilties(const Device &device) { + Capabilities capabilities; + try { + capabilities = device.capabilities(); + } catch (std::exception &e) { throw TestFailureException("Capabilities test: ", e.what()); } + + TEST_ASSERT(capabilities.hwId.architecture > SemanticVersion()); +} + +void testNetworkInfoNotExistentIndex(const Device &device) { + try { + Network(device, 0); + FAIL(); + } catch (Exception &e) { + // good it should have thrown + } catch (std::exception &e) { throw TestFailureException("NetworkInfo no index test: ", e.what()); } +} + +void testNetworkInfoBuffer(const Device &device) { + try { + std::shared_ptr buffer = std::make_shared(device, sizeof(networkModelData)); + buffer->resize(sizeof(networkModelData)); + std::memcpy(buffer->data(), networkModelData, sizeof(networkModelData)); + Network network(device, buffer); + + TEST_ASSERT(network.getIfmDims().size() == 1); + TEST_ASSERT(network.getOfmDims().size() == 1); + } catch (std::exception &e) { throw TestFailureException("NetworkInfo buffer test: ", e.what()); } +} + +void testNetworkInfoUnparsableBuffer(const Device &device) { + try { + auto buffer = std::make_shared(device, sizeof(networkModelData) / 4); + buffer->resize(sizeof(networkModelData) / 4); + std::memcpy(buffer->data(), networkModelData + sizeof(networkModelData) / 4, sizeof(networkModelData) / 4); + + try { + Network network(device, buffer); + FAIL(); + } catch (Exception) { + // good, it should have thrown! + } + } catch (std::exception &e) { throw TestFailureException("NetworkInfo unparsable buffer test: ", e.what()); } +} + +void testRunInferenceBuffer(const Device &device) { + try { + auto networkBuffer = std::make_shared(device, sizeof(networkModelData)); + networkBuffer->resize(sizeof(networkModelData)); + std::memcpy(networkBuffer->data(), networkModelData, sizeof(networkModelData)); + auto network = std::make_shared(device, networkBuffer); + + std::vector> inputBuffers; + std::vector> outputBuffers; + + auto inputBuffer = std::make_shared(device, sizeof(inputData)); + inputBuffer->resize(sizeof(inputData)); + std::memcpy(inputBuffer->data(), inputData, sizeof(inputData)); + + inputBuffers.push_back(inputBuffer); + outputBuffers.push_back(std::make_shared(device, sizeof(expectedOutputData))); + std::vector enabledCounters(Inference::getMaxPmuEventCounters()); + + auto inference = std::make_shared(network, + inputBuffers.begin(), + inputBuffers.end(), + outputBuffers.begin(), + outputBuffers.end(), + enabledCounters, + false); + + bool timedout = inference->wait(defaultTimeout); + TEST_ASSERT(!timedout); + + InferenceStatus status = inference->status(); + TEST_ASSERT(status == InferenceStatus::OK); + + bool success = inference->cancel(); + TEST_ASSERT(!success); + + TEST_ASSERT(std::memcmp(expectedOutputData, outputBuffers[0]->data(), sizeof(expectedOutputData)) == 0); + + } catch (std::exception &e) { throw TestFailureException("Inference run test: ", e.what()); } +} + +} // namespace + +int main() { + Device device; + + try { + testPing(device); + testVersion(device); + testCapabilties(device); + testNetworkInfoNotExistentIndex(device); + testNetworkInfoBuffer(device); + testNetworkInfoUnparsableBuffer(device); + testRunInferenceBuffer(device); + } catch (TestFailureException &e) { + std::cerr << "Test failure: " << e.what() << std::endl; + return 1; + } + + return 0; +} diff --git a/tests/test_assertions.hpp b/tests/test_assertions.hpp new file mode 100644 index 0000000..f90325f --- /dev/null +++ b/tests/test_assertions.hpp @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TEST_ASSERTIONS_H +#define TEST_ASSERTIONS_H + +#include +#include + +namespace { +template +std::string string_format(std::ostringstream &stringStream) { + return stringStream.str(); +} + +template +std::string string_format(std::ostringstream &stringStream, T t, Args... args) { + stringStream << t; + return string_format(stringStream, args...); +} + +class TestFailureException : public std::exception { +public: + template + TestFailureException(const char *msg, Args... args) { + std::ostringstream stringStream; + this->msg = string_format(stringStream, msg, args...); + } + const char *what() const throw() { + return msg.c_str(); + } + +private: + std::string msg; +}; +} // namespace + +#define TEST_ASSERT(v) \ + do { \ + if (!(v)) { \ + throw TestFailureException(__FILE__, ":", __LINE__, " ERROR test failed: '", #v, "'"); \ + } \ + } while (0) + +#define FAIL() TEST_ASSERT(false) + +#endif -- cgit v1.2.1