From 5ea9f31a18056cd79c6a57746f1627acaa1a1597 Mon Sep 17 00:00:00 2001 From: James Ward Date: Thu, 29 Oct 2020 16:19:02 +0000 Subject: IVGCVSW-5468 Rewrite QuantizationDataSet.cpp to avoid use of CsvReader * Remove armnnUtils/CsvReader and usage * Remove armnn/CsvReaderTest and usage * Replace functionality in QuantizationDataSet.cpp Signed-off-by: James Ward Change-Id: I7213904482afa93ae6d607aa5e69117c8c34ea81 --- Android.mk | 5 +- CMakeLists.txt | 4 +- src/armnn/test/CsvReaderTest.cpp | 130 --------------------- src/armnnQuantizer/QuantizationDataSet.cpp | 63 +++++----- src/armnnUtils/CsvReader.cpp | 63 ---------- src/armnnUtils/CsvReader.hpp | 25 ---- .../NetworkExecutionUtils.hpp | 1 - 7 files changed, 38 insertions(+), 253 deletions(-) delete mode 100644 src/armnn/test/CsvReaderTest.cpp delete mode 100644 src/armnnUtils/CsvReader.cpp delete mode 100644 src/armnnUtils/CsvReader.hpp diff --git a/Android.mk b/Android.mk index 0f23cf9c1e..2af7be5095 100644 --- a/Android.mk +++ b/Android.mk @@ -128,7 +128,6 @@ LOCAL_SRC_FILES := \ src/armnn/TypesUtils.cpp \ src/armnn/Utils.cpp \ src/armnn/WallClockTimer.cpp \ - src/armnnUtils/CsvReader.cpp \ src/armnnUtils/DataLayoutIndexed.cpp \ src/armnnUtils/DotSerializer.cpp \ src/armnnUtils/FloatingPointConverter.cpp \ @@ -238,8 +237,7 @@ LOCAL_SRC_FILES := \ src/profiling/backends/BackendProfiling.cpp LOCAL_STATIC_LIBRARIES := \ - arm_compute_library \ - libboost_system + arm_compute_library LOCAL_SHARED_LIBRARIES := \ liblog @@ -354,7 +352,6 @@ endif # PLATFORM_VERSION == Q or later LOCAL_SRC_FILES := \ $(ARMNN_BACKEND_TEST_SOURCES) \ src/armnn/test/ConstTensorLayerVisitor.cpp \ - src/armnn/test/CsvReaderTest.cpp \ src/armnn/test/EndToEndTest.cpp \ src/armnn/ExecutionFrame.cpp \ src/armnn/test/ExecutionFrameTest.cpp \ diff --git a/CMakeLists.txt b/CMakeLists.txt index abe9bf1e12..a6d88e096c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,8 +59,6 @@ list(APPEND armnnUtils_sources src/armnnUtils/LeakChecking.hpp src/armnnUtils/ModelAccuracyChecker.cpp src/armnnUtils/ModelAccuracyChecker.hpp - src/armnnUtils/CsvReader.cpp - src/armnnUtils/CsvReader.hpp src/armnnUtils/FloatingPointConverter.cpp src/armnnUtils/VerificationHelpers.hpp src/armnnUtils/VerificationHelpers.cpp @@ -177,6 +175,7 @@ if(BUILD_ARMNN_QUANTIZER AND ARMNNREF) target_include_directories(armnnQuantizer PRIVATE include/armnnDeserializer) target_include_directories(armnnQuantizer PRIVATE src/armnnUtils) target_include_directories(armnnQuantizer PRIVATE src/armnn) + target_include_directories(armnnQuantizer PRIVATE third-party) include_directories(SYSTEM "${FLATBUFFERS_INCLUDE_PATH}") set_target_properties(armnnQuantizer PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}) @@ -626,7 +625,6 @@ if(BUILD_UNIT_TESTS) src/armnn/test/ConstTensorLayerVisitor.hpp src/armnn/test/ConstTensorLayerVisitor.cpp src/armnn/test/CreateWorkload.hpp - src/armnn/test/CsvReaderTest.cpp src/armnn/test/EndToEndTest.cpp src/armnn/test/ExecutionFrameTest.cpp src/armnn/test/FloatingPointConverterTest.cpp diff --git a/src/armnn/test/CsvReaderTest.cpp b/src/armnn/test/CsvReaderTest.cpp deleted file mode 100644 index 96185c8408..0000000000 --- a/src/armnn/test/CsvReaderTest.cpp +++ /dev/null @@ -1,130 +0,0 @@ -// -// Copyright © 2017 Arm Ltd. All rights reserved. -// SPDX-License-Identifier: MIT -// -#include "CsvReader.hpp" -#include "armnn/Optional.hpp" -#include - -#include - -#include -#include - -using namespace armnnUtils; - -namespace { -struct TestHelper { - - TestHelper() - { - BOOST_TEST_MESSAGE("setup fixture"); - } - - ~TestHelper() - { - BOOST_TEST_MESSAGE("teardown fixture"); - TearDown(); - } - - std::string CreateTempCsvFile() - { - fs::path p = armnnUtils::Filesystem::NamedTempFile("Armnn-CreateTempCsvFileTest-TempFile.csv"); - try - { - std::ofstream ofs{p}; - ofs << "airplane, bicycle , bird , \"m,o,n,k,e,y\"\n"; - ofs << "banana, shoe, \"ice\""; - ofs.close(); - } catch (std::exception &e) - { - std::cerr << "Unable to write to file at location [" << p.c_str() << "] : " << e.what() << std::endl; - BOOST_TEST(false); - } - - m_CsvFile = p; - return p.string(); - } - - int CheckStringsMatch(CsvRow &row, unsigned int index, std::string expectedValue) - { - return row.values.at(index).compare(expectedValue); - } - - void TearDown() - { - RemoveCsvFile(); - } - - void RemoveCsvFile() - { - if (m_CsvFile) - { - try - { - fs::remove(m_CsvFile.value()); - } - catch (std::exception &e) - { - std::cerr << "Unable to delete file [" << m_CsvFile.value() << "] : " << e.what() << std::endl; - BOOST_TEST(false); - } - } - } - - armnn::Optional m_CsvFile; -}; -} - -BOOST_AUTO_TEST_SUITE(CsvReaderTest) - -BOOST_FIXTURE_TEST_CASE(TestParseVector, TestHelper) -{ - CsvReader reader; - std::vector csvStrings; - csvStrings.reserve(2); - csvStrings.push_back("airplane, automobile , bird , \"c,a,t\""); - csvStrings.push_back("banana, shoe, \"ice\""); - - std::vector row = reader.ParseVector(csvStrings); - CsvRow row1 = row[0]; - CsvRow row2 = row[1]; - - BOOST_CHECK(row.size() == 2); - - BOOST_CHECK(row1.values.size() == 4); - BOOST_CHECK(CheckStringsMatch(row1, 0, "airplane") == 0); - BOOST_CHECK(CheckStringsMatch(row1, 1, "automobile") == 0); - BOOST_CHECK(CheckStringsMatch(row1, 2, "bird") == 0); - BOOST_CHECK(CheckStringsMatch(row1, 3, "c,a,t") == 0); - - BOOST_CHECK(row2.values.size() == 3); - BOOST_CHECK(CheckStringsMatch(row2, 0, "banana") == 0); - BOOST_CHECK(CheckStringsMatch(row2, 1, "shoe") == 0); - BOOST_CHECK(CheckStringsMatch(row2, 2, "ice") == 0); -} - -BOOST_FIXTURE_TEST_CASE(TestLoadingFileFromDisk, TestHelper) -{ - CsvReader reader; - std::string theFilePath = TestHelper::CreateTempCsvFile(); - - std::vector row = reader.ParseFile(theFilePath); - CsvRow row1 = row[0]; - CsvRow row2 = row[1]; - - BOOST_CHECK(row.size() == 2); - - BOOST_CHECK(row1.values.size() == 4); - BOOST_CHECK(CheckStringsMatch(row1, 0, "airplane") == 0); - BOOST_CHECK(CheckStringsMatch(row1, 1, "bicycle") == 0); - BOOST_CHECK(CheckStringsMatch(row1, 2, "bird") == 0); - BOOST_CHECK(CheckStringsMatch(row1, 3, "m,o,n,k,e,y") == 0); - - BOOST_CHECK(row2.values.size() == 3); - BOOST_CHECK(CheckStringsMatch(row2, 0, "banana") == 0); - BOOST_CHECK(CheckStringsMatch(row2, 1, "shoe") == 0); - BOOST_CHECK(CheckStringsMatch(row2, 2, "ice") == 0); -} - -BOOST_AUTO_TEST_SUITE_END() diff --git a/src/armnnQuantizer/QuantizationDataSet.cpp b/src/armnnQuantizer/QuantizationDataSet.cpp index 819b437a55..acd301a470 100644 --- a/src/armnnQuantizer/QuantizationDataSet.cpp +++ b/src/armnnQuantizer/QuantizationDataSet.cpp @@ -4,8 +4,10 @@ // #include "QuantizationDataSet.hpp" -#include "CsvReader.hpp" +#include + +#include #include #include @@ -67,49 +69,49 @@ armnn::TensorInfo InputLayerVisitor::GetTensorInfo(armnn::LayerBindingId layerBi } -unsigned int GetPassIdFromCsvRow(std::vector csvRows, unsigned int rowIndex) +unsigned int GetPassIdFromCsvRow(std::vector tokens, unsigned int lineIndex) { unsigned int passId; try { - passId = static_cast(std::stoi(csvRows[rowIndex].values[0])); + passId = static_cast(std::stoi(tokens[0])); } catch (const std::invalid_argument&) { - throw armnn::ParseException("Pass ID [" + csvRows[rowIndex].values[0] + "]" + - " is not correct format on CSV row " + std::to_string(rowIndex)); + throw armnn::ParseException(fmt::format("Pass ID [{}] is not correct format on CSV row {}", + tokens[0], lineIndex)); } return passId; } -armnn::LayerBindingId GetBindingIdFromCsvRow(std::vector csvRows, unsigned int rowIndex) +armnn::LayerBindingId GetBindingIdFromCsvRow(std::vector tokens, unsigned int lineIndex) { armnn::LayerBindingId bindingId; try { - bindingId = std::stoi(csvRows[rowIndex].values[1]); + bindingId = std::stoi(tokens[1]); } catch (const std::invalid_argument&) { - throw armnn::ParseException("Binding ID [" + csvRows[rowIndex].values[0] + "]" + - " is not correct format on CSV row " + std::to_string(rowIndex)); + throw armnn::ParseException(fmt::format("Binding ID [{}] is not correct format on CSV row {}", + tokens[1], lineIndex)); } return bindingId; } -std::string GetFileNameFromCsvRow(std::vector csvRows, unsigned int rowIndex) +std::string GetFileNameFromCsvRow(std::vector tokens, unsigned int lineIndex) { - std::string fileName = csvRows[rowIndex].values[2]; + std::string fileName = armnn::stringUtils::StringTrim(tokens[2]); if (!fs::exists(fileName)) { - throw armnn::ParseException("File [ " + fileName + "] provided on CSV row " + std::to_string(rowIndex) + - " does not exist."); + throw armnn::ParseException(fmt::format("File [{}] provided on CSV row {} does not exist.", + fileName, lineIndex)); } if (fileName.empty()) { - throw armnn::ParseException("Filename cannot be empty on CSV row " + std::to_string(rowIndex)); + throw armnn::ParseException(fmt::format("Filename cannot be empty on CSV row {} ", lineIndex)); } return fileName; } @@ -118,34 +120,41 @@ std::string GetFileNameFromCsvRow(std::vector csvRows, unsig void QuantizationDataSet::ParseCsvFile() { std::map passIdToQuantizationInput; - armnnUtils::CsvReader reader; if (m_CsvFilePath == "") { throw armnn::Exception("CSV file not specified."); } - // Parse CSV file and extract data - std::vector csvRows = reader.ParseFile(m_CsvFilePath); - if (csvRows.empty()) + std::ifstream inf (m_CsvFilePath.c_str()); + std::string line; + std::vector tokens; + unsigned int lineIndex = 0; + + if (!inf) { - throw armnn::Exception("CSV file [" + m_CsvFilePath + "] is empty."); + throw armnn::Exception(fmt::format("CSV file {} not found.", m_CsvFilePath)); } - for (unsigned int i = 0; i < csvRows.size(); ++i) + while (getline(inf, line)) { - if (csvRows[i].values.size() != 3) + tokens = armnn::stringUtils::StringTokenizer(line, ","); + + if (tokens.size() != 3) { - throw armnn::Exception("CSV file [" + m_CsvFilePath + "] does not have correct number of entries " + - "on line " + std::to_string(i) + ". Expected 3 entries " + - "but was " + std::to_string(csvRows[i].values.size())); + throw armnn::Exception(fmt::format("CSV file [{}] does not have correct number of entries" \ + "on line {}. Expected 3 entries but was {}.", + m_CsvFilePath, lineIndex, tokens.size())); + } - unsigned int passId = GetPassIdFromCsvRow(csvRows, i); - armnn::LayerBindingId bindingId = GetBindingIdFromCsvRow(csvRows, i); - std::string rawFileName = GetFileNameFromCsvRow(csvRows, i); + unsigned int passId = GetPassIdFromCsvRow(tokens, lineIndex); + armnn::LayerBindingId bindingId = GetBindingIdFromCsvRow(tokens, lineIndex); + std::string rawFileName = GetFileNameFromCsvRow(tokens, lineIndex); AddInputData(passId, bindingId, rawFileName, passIdToQuantizationInput); + + ++lineIndex; } if (passIdToQuantizationInput.empty()) diff --git a/src/armnnUtils/CsvReader.cpp b/src/armnnUtils/CsvReader.cpp deleted file mode 100644 index feee4d18c6..0000000000 --- a/src/armnnUtils/CsvReader.cpp +++ /dev/null @@ -1,63 +0,0 @@ -// -// Copyright © 2017 Arm Ltd. All rights reserved. -// SPDX-License-Identifier: MIT -// - -#include "CsvReader.hpp" -#include "armnn/utility/StringUtils.hpp" - -#include - -#include -#include -#include - -using Tokenizer = boost::tokenizer>; - -namespace armnnUtils -{ - -CsvRow ParseLine(const std::string& csvLine) -{ - Tokenizer tokenizer(csvLine); - CsvRow entry; - - for (const auto &token : tokenizer) - { - entry.values.push_back(armnn::stringUtils::StringTrimCopy(token)); - } - return entry; -} - -std::vector CsvReader::ParseFile(const std::string& csvFile) -{ - std::vector result; - - std::ifstream in(csvFile.c_str()); - if (!in.is_open()) - return result; - - std::string line; - while (getline(in, line)) - { - if(!line.empty()) - { - CsvRow entry = ParseLine(line); - result.push_back(entry); - } - } - return result; -} - -std::vector CsvReader::ParseVector(const std::vector& csvVector) -{ - std::vector result; - - for (auto const& line: csvVector) - { - CsvRow entry = ParseLine(line); - result.push_back(entry); - } - return result; -} -} // namespace armnnUtils \ No newline at end of file diff --git a/src/armnnUtils/CsvReader.hpp b/src/armnnUtils/CsvReader.hpp deleted file mode 100644 index d95f4f7055..0000000000 --- a/src/armnnUtils/CsvReader.hpp +++ /dev/null @@ -1,25 +0,0 @@ -// -// Copyright © 2017 Arm Ltd. All rights reserved. -// SPDX-License-Identifier: MIT -// -#pragma once - -#include -#include - -namespace armnnUtils -{ - -struct CsvRow -{ - std::vector values; -}; - -class CsvReader -{ -public: - static std::vector ParseFile(const std::string& csvFile); - - static std::vector ParseVector(const std::vector& csvVector); -}; -} // namespace armnnUtils diff --git a/tests/NetworkExecutionUtils/NetworkExecutionUtils.hpp b/tests/NetworkExecutionUtils/NetworkExecutionUtils.hpp index d101d4a23c..9d9e616e98 100644 --- a/tests/NetworkExecutionUtils/NetworkExecutionUtils.hpp +++ b/tests/NetworkExecutionUtils/NetworkExecutionUtils.hpp @@ -5,7 +5,6 @@ #pragma once -#include "CsvReader.hpp" #include #include -- cgit v1.2.1