diff options
Diffstat (limited to 'tests/InferenceTest.inl')
| -rw-r--r-- | tests/InferenceTest.inl | 407 |
1 files changed, 0 insertions, 407 deletions
diff --git a/tests/InferenceTest.inl b/tests/InferenceTest.inl deleted file mode 100644 index c05e70d9f7..0000000000 --- a/tests/InferenceTest.inl +++ /dev/null @@ -1,407 +0,0 @@ -// -// Copyright © 2017 Arm Ltd. All rights reserved. -// SPDX-License-Identifier: MIT -// -#include "InferenceTest.hpp" - -#include <boost/algorithm/string.hpp> -#include <boost/numeric/conversion/cast.hpp> -#include <boost/filesystem/path.hpp> -#include <boost/assert.hpp> -#include <boost/format.hpp> -#include <boost/program_options.hpp> -#include <boost/filesystem/operations.hpp> - -#include <fstream> -#include <iostream> -#include <iomanip> -#include <array> -#include <chrono> - -using namespace std; -using namespace std::chrono; -using namespace armnn::test; - -namespace armnn -{ -namespace test -{ - -using TContainer = boost::variant<std::vector<float>, std::vector<int>, std::vector<unsigned char>>; - -template <typename TTestCaseDatabase, typename TModel> -ClassifierTestCase<TTestCaseDatabase, TModel>::ClassifierTestCase( - int& numInferencesRef, - int& numCorrectInferencesRef, - const std::vector<unsigned int>& validationPredictions, - std::vector<unsigned int>* validationPredictionsOut, - TModel& model, - unsigned int testCaseId, - unsigned int label, - std::vector<typename TModel::DataType> modelInput) - : InferenceModelTestCase<TModel>( - model, testCaseId, std::vector<TContainer>{ modelInput }, { model.GetOutputSize() }) - , m_Label(label) - , m_QuantizationParams(model.GetQuantizationParams()) - , m_NumInferencesRef(numInferencesRef) - , m_NumCorrectInferencesRef(numCorrectInferencesRef) - , m_ValidationPredictions(validationPredictions) - , m_ValidationPredictionsOut(validationPredictionsOut) -{ -} - -struct ClassifierResultProcessor : public boost::static_visitor<> -{ - using ResultMap = std::map<float,int>; - - ClassifierResultProcessor(float scale, int offset) - : m_Scale(scale) - , m_Offset(offset) - {} - - void operator()(const std::vector<float>& values) - { - SortPredictions(values, [](float value) - { - return value; - }); - } - - void operator()(const std::vector<uint8_t>& values) - { - auto& scale = m_Scale; - auto& offset = m_Offset; - SortPredictions(values, [&scale, &offset](uint8_t value) - { - return armnn::Dequantize(value, scale, offset); - }); - } - - void operator()(const std::vector<int>& values) - { - boost::ignore_unused(values); - BOOST_ASSERT_MSG(false, "Non-float predictions output not supported."); - } - - ResultMap& GetResultMap() { return m_ResultMap; } - -private: - template<typename Container, typename Delegate> - void SortPredictions(const Container& c, Delegate delegate) - { - int index = 0; - for (const auto& value : c) - { - int classification = index++; - // Take the first class with each probability - // This avoids strange results when looping over batched results produced - // with identical test data. - ResultMap::iterator lb = m_ResultMap.lower_bound(value); - - if (lb == m_ResultMap.end() || !m_ResultMap.key_comp()(value, lb->first)) - { - // If the key is not already in the map, insert it. - m_ResultMap.insert(lb, ResultMap::value_type(delegate(value), classification)); - } - } - } - - ResultMap m_ResultMap; - - float m_Scale=0.0f; - int m_Offset=0; -}; - -template <typename TTestCaseDatabase, typename TModel> -TestCaseResult ClassifierTestCase<TTestCaseDatabase, TModel>::ProcessResult(const InferenceTestOptions& params) -{ - auto& output = this->GetOutputs()[0]; - const auto testCaseId = this->GetTestCaseId(); - - ClassifierResultProcessor resultProcessor(m_QuantizationParams.first, m_QuantizationParams.second); - boost::apply_visitor(resultProcessor, output); - - ARMNN_LOG(info) << "= Prediction values for test #" << testCaseId; - auto it = resultProcessor.GetResultMap().rbegin(); - for (int i=0; i<5 && it != resultProcessor.GetResultMap().rend(); ++i) - { - ARMNN_LOG(info) << "Top(" << (i+1) << ") prediction is " << it->second << - " with value: " << (it->first); - ++it; - } - - unsigned int prediction = 0; - boost::apply_visitor([&](auto&& value) - { - prediction = boost::numeric_cast<unsigned int>( - std::distance(value.begin(), std::max_element(value.begin(), value.end()))); - }, - output); - - // If we're just running the defaultTestCaseIds, each one must be classified correctly. - if (params.m_IterationCount == 0 && prediction != m_Label) - { - ARMNN_LOG(error) << "Prediction for test case " << testCaseId << " (" << prediction << ")" << - " is incorrect (should be " << m_Label << ")"; - return TestCaseResult::Failed; - } - - // If a validation file was provided as input, it checks that the prediction matches. - if (!m_ValidationPredictions.empty() && prediction != m_ValidationPredictions[testCaseId]) - { - ARMNN_LOG(error) << "Prediction for test case " << testCaseId << " (" << prediction << ")" << - " doesn't match the prediction in the validation file (" << m_ValidationPredictions[testCaseId] << ")"; - return TestCaseResult::Failed; - } - - // If a validation file was requested as output, it stores the predictions. - if (m_ValidationPredictionsOut) - { - m_ValidationPredictionsOut->push_back(prediction); - } - - // Updates accuracy stats. - m_NumInferencesRef++; - if (prediction == m_Label) - { - m_NumCorrectInferencesRef++; - } - - return TestCaseResult::Ok; -} - -template <typename TDatabase, typename InferenceModel> -template <typename TConstructDatabaseCallable, typename TConstructModelCallable> -ClassifierTestCaseProvider<TDatabase, InferenceModel>::ClassifierTestCaseProvider( - TConstructDatabaseCallable constructDatabase, TConstructModelCallable constructModel) - : m_ConstructModel(constructModel) - , m_ConstructDatabase(constructDatabase) - , m_NumInferences(0) - , m_NumCorrectInferences(0) -{ -} - -template <typename TDatabase, typename InferenceModel> -void ClassifierTestCaseProvider<TDatabase, InferenceModel>::AddCommandLineOptions( - boost::program_options::options_description& options) -{ - namespace po = boost::program_options; - - options.add_options() - ("validation-file-in", po::value<std::string>(&m_ValidationFileIn)->default_value(""), - "Reads expected predictions from the given file and confirms they match the actual predictions.") - ("validation-file-out", po::value<std::string>(&m_ValidationFileOut)->default_value(""), - "Predictions are saved to the given file for later use via --validation-file-in.") - ("data-dir,d", po::value<std::string>(&m_DataDir)->required(), - "Path to directory containing test data"); - - InferenceModel::AddCommandLineOptions(options, m_ModelCommandLineOptions); -} - -template <typename TDatabase, typename InferenceModel> -bool ClassifierTestCaseProvider<TDatabase, InferenceModel>::ProcessCommandLineOptions( - const InferenceTestOptions& commonOptions) -{ - if (!ValidateDirectory(m_DataDir)) - { - return false; - } - - ReadPredictions(); - - m_Model = m_ConstructModel(commonOptions, m_ModelCommandLineOptions); - if (!m_Model) - { - return false; - } - - m_Database = std::make_unique<TDatabase>(m_ConstructDatabase(m_DataDir.c_str(), *m_Model)); - if (!m_Database) - { - return false; - } - - return true; -} - -template <typename TDatabase, typename InferenceModel> -std::unique_ptr<IInferenceTestCase> -ClassifierTestCaseProvider<TDatabase, InferenceModel>::GetTestCase(unsigned int testCaseId) -{ - std::unique_ptr<typename TDatabase::TTestCaseData> testCaseData = m_Database->GetTestCaseData(testCaseId); - if (testCaseData == nullptr) - { - return nullptr; - } - - return std::make_unique<ClassifierTestCase<TDatabase, InferenceModel>>( - m_NumInferences, - m_NumCorrectInferences, - m_ValidationPredictions, - m_ValidationFileOut.empty() ? nullptr : &m_ValidationPredictionsOut, - *m_Model, - testCaseId, - testCaseData->m_Label, - std::move(testCaseData->m_InputImage)); -} - -template <typename TDatabase, typename InferenceModel> -bool ClassifierTestCaseProvider<TDatabase, InferenceModel>::OnInferenceTestFinished() -{ - const double accuracy = boost::numeric_cast<double>(m_NumCorrectInferences) / - boost::numeric_cast<double>(m_NumInferences); - ARMNN_LOG(info) << std::fixed << std::setprecision(3) << "Overall accuracy: " << accuracy; - - // If a validation file was requested as output, the predictions are saved to it. - if (!m_ValidationFileOut.empty()) - { - std::ofstream validationFileOut(m_ValidationFileOut.c_str(), std::ios_base::trunc | std::ios_base::out); - if (validationFileOut.good()) - { - for (const unsigned int prediction : m_ValidationPredictionsOut) - { - validationFileOut << prediction << std::endl; - } - } - else - { - ARMNN_LOG(error) << "Failed to open output validation file: " << m_ValidationFileOut; - return false; - } - } - - return true; -} - -template <typename TDatabase, typename InferenceModel> -void ClassifierTestCaseProvider<TDatabase, InferenceModel>::ReadPredictions() -{ - // Reads the expected predictions from the input validation file (if provided). - if (!m_ValidationFileIn.empty()) - { - std::ifstream validationFileIn(m_ValidationFileIn.c_str(), std::ios_base::in); - if (validationFileIn.good()) - { - while (!validationFileIn.eof()) - { - unsigned int i; - validationFileIn >> i; - m_ValidationPredictions.emplace_back(i); - } - } - else - { - throw armnn::Exception(boost::str(boost::format("Failed to open input validation file: %1%") - % m_ValidationFileIn)); - } - } -} - -template<typename TConstructTestCaseProvider> -int InferenceTestMain(int argc, - char* argv[], - const std::vector<unsigned int>& defaultTestCaseIds, - TConstructTestCaseProvider constructTestCaseProvider) -{ - // Configures logging for both the ARMNN library and this test program. -#ifdef NDEBUG - armnn::LogSeverity level = armnn::LogSeverity::Info; -#else - armnn::LogSeverity level = armnn::LogSeverity::Debug; -#endif - armnn::ConfigureLogging(true, true, level); - - try - { - std::unique_ptr<IInferenceTestCaseProvider> testCaseProvider = constructTestCaseProvider(); - if (!testCaseProvider) - { - return 1; - } - - InferenceTestOptions inferenceTestOptions; - if (!ParseCommandLine(argc, argv, *testCaseProvider, inferenceTestOptions)) - { - return 1; - } - - const bool success = InferenceTest(inferenceTestOptions, defaultTestCaseIds, *testCaseProvider); - return success ? 0 : 1; - } - catch (armnn::Exception const& e) - { - ARMNN_LOG(fatal) << "Armnn Error: " << e.what(); - return 1; - } -} - -// -// This function allows us to create a classifier inference test based on: -// - a model file name -// - which can be a binary or a text file for protobuf formats -// - an input tensor name -// - an output tensor name -// - a set of test case ids -// - a callback method which creates an object that can return images -// called 'Database' in these tests -// - and an input tensor shape -// -template<typename TDatabase, - typename TParser, - typename TConstructDatabaseCallable> -int ClassifierInferenceTestMain(int argc, - char* argv[], - const char* modelFilename, - bool isModelBinary, - const char* inputBindingName, - const char* outputBindingName, - const std::vector<unsigned int>& defaultTestCaseIds, - TConstructDatabaseCallable constructDatabase, - const armnn::TensorShape* inputTensorShape) - -{ - BOOST_ASSERT(modelFilename); - BOOST_ASSERT(inputBindingName); - BOOST_ASSERT(outputBindingName); - - return InferenceTestMain(argc, argv, defaultTestCaseIds, - [=] - () - { - using InferenceModel = InferenceModel<TParser, typename TDatabase::DataType>; - using TestCaseProvider = ClassifierTestCaseProvider<TDatabase, InferenceModel>; - - return make_unique<TestCaseProvider>(constructDatabase, - [&] - (const InferenceTestOptions &commonOptions, - typename InferenceModel::CommandLineOptions modelOptions) - { - if (!ValidateDirectory(modelOptions.m_ModelDir)) - { - return std::unique_ptr<InferenceModel>(); - } - - typename InferenceModel::Params modelParams; - modelParams.m_ModelPath = modelOptions.m_ModelDir + modelFilename; - modelParams.m_InputBindings = { inputBindingName }; - modelParams.m_OutputBindings = { outputBindingName }; - - if (inputTensorShape) - { - modelParams.m_InputShapes.push_back(*inputTensorShape); - } - - modelParams.m_IsModelBinary = isModelBinary; - modelParams.m_ComputeDevices = modelOptions.GetComputeDevicesAsBackendIds(); - modelParams.m_VisualizePostOptimizationModel = modelOptions.m_VisualizePostOptimizationModel; - modelParams.m_EnableFp16TurboMode = modelOptions.m_EnableFp16TurboMode; - - return std::make_unique<InferenceModel>(modelParams, - commonOptions.m_EnableProfiling, - commonOptions.m_DynamicBackendsPath); - }); - }); -} - -} // namespace test -} // namespace armnn |