diff options
author | James Ward <james.ward@arm.com> | 2020-10-29 16:19:02 +0000 |
---|---|---|
committer | James Ward <james.ward@arm.com> | 2020-10-29 21:30:43 +0000 |
commit | 5ea9f31a18056cd79c6a57746f1627acaa1a1597 (patch) | |
tree | d4e1ba148445b0cb3cea4741259f3efe50a6ac7b /src | |
parent | 67e95f2aebc827f2e3c571385b9e623f09a65141 (diff) | |
download | armnn-5ea9f31a18056cd79c6a57746f1627acaa1a1597.tar.gz |
IVGCVSW-5468 Rewrite QuantizationDataSet.cpp to avoid use of CsvReader
* Remove armnnUtils/CsvReader and usage
* Remove armnn/CsvReaderTest and usage
* Replace functionality in QuantizationDataSet.cpp
Signed-off-by: James Ward <james.ward@arm.com>
Change-Id: I7213904482afa93ae6d607aa5e69117c8c34ea81
Diffstat (limited to 'src')
-rw-r--r-- | src/armnn/test/CsvReaderTest.cpp | 130 | ||||
-rw-r--r-- | src/armnnQuantizer/QuantizationDataSet.cpp | 63 | ||||
-rw-r--r-- | src/armnnUtils/CsvReader.cpp | 63 | ||||
-rw-r--r-- | src/armnnUtils/CsvReader.hpp | 25 |
4 files changed, 36 insertions, 245 deletions
diff --git a/src/armnn/test/CsvReaderTest.cpp b/src/armnn/test/CsvReaderTest.cpp deleted file mode 100644 index 96185c8408..0000000000 --- a/src/armnn/test/CsvReaderTest.cpp +++ /dev/null @@ -1,130 +0,0 @@ -// -// Copyright © 2017 Arm Ltd. All rights reserved. -// SPDX-License-Identifier: MIT -// -#include "CsvReader.hpp" -#include "armnn/Optional.hpp" -#include <Filesystem.hpp> - -#include <boost/test/unit_test.hpp> - -#include <iostream> -#include <string> - -using namespace armnnUtils; - -namespace { -struct TestHelper { - - TestHelper() - { - BOOST_TEST_MESSAGE("setup fixture"); - } - - ~TestHelper() - { - BOOST_TEST_MESSAGE("teardown fixture"); - TearDown(); - } - - std::string CreateTempCsvFile() - { - fs::path p = armnnUtils::Filesystem::NamedTempFile("Armnn-CreateTempCsvFileTest-TempFile.csv"); - try - { - std::ofstream ofs{p}; - ofs << "airplane, bicycle , bird , \"m,o,n,k,e,y\"\n"; - ofs << "banana, shoe, \"ice\""; - ofs.close(); - } catch (std::exception &e) - { - std::cerr << "Unable to write to file at location [" << p.c_str() << "] : " << e.what() << std::endl; - BOOST_TEST(false); - } - - m_CsvFile = p; - return p.string(); - } - - int CheckStringsMatch(CsvRow &row, unsigned int index, std::string expectedValue) - { - return row.values.at(index).compare(expectedValue); - } - - void TearDown() - { - RemoveCsvFile(); - } - - void RemoveCsvFile() - { - if (m_CsvFile) - { - try - { - fs::remove(m_CsvFile.value()); - } - catch (std::exception &e) - { - std::cerr << "Unable to delete file [" << m_CsvFile.value() << "] : " << e.what() << std::endl; - BOOST_TEST(false); - } - } - } - - armnn::Optional<fs::path> m_CsvFile; -}; -} - -BOOST_AUTO_TEST_SUITE(CsvReaderTest) - -BOOST_FIXTURE_TEST_CASE(TestParseVector, TestHelper) -{ - CsvReader reader; - std::vector<std::string> csvStrings; - csvStrings.reserve(2); - csvStrings.push_back("airplane, automobile , bird , \"c,a,t\""); - csvStrings.push_back("banana, shoe, \"ice\""); - - std::vector<CsvRow> row = reader.ParseVector(csvStrings); - CsvRow row1 = row[0]; - CsvRow row2 = row[1]; - - BOOST_CHECK(row.size() == 2); - - BOOST_CHECK(row1.values.size() == 4); - BOOST_CHECK(CheckStringsMatch(row1, 0, "airplane") == 0); - BOOST_CHECK(CheckStringsMatch(row1, 1, "automobile") == 0); - BOOST_CHECK(CheckStringsMatch(row1, 2, "bird") == 0); - BOOST_CHECK(CheckStringsMatch(row1, 3, "c,a,t") == 0); - - BOOST_CHECK(row2.values.size() == 3); - BOOST_CHECK(CheckStringsMatch(row2, 0, "banana") == 0); - BOOST_CHECK(CheckStringsMatch(row2, 1, "shoe") == 0); - BOOST_CHECK(CheckStringsMatch(row2, 2, "ice") == 0); -} - -BOOST_FIXTURE_TEST_CASE(TestLoadingFileFromDisk, TestHelper) -{ - CsvReader reader; - std::string theFilePath = TestHelper::CreateTempCsvFile(); - - std::vector<CsvRow> row = reader.ParseFile(theFilePath); - CsvRow row1 = row[0]; - CsvRow row2 = row[1]; - - BOOST_CHECK(row.size() == 2); - - BOOST_CHECK(row1.values.size() == 4); - BOOST_CHECK(CheckStringsMatch(row1, 0, "airplane") == 0); - BOOST_CHECK(CheckStringsMatch(row1, 1, "bicycle") == 0); - BOOST_CHECK(CheckStringsMatch(row1, 2, "bird") == 0); - BOOST_CHECK(CheckStringsMatch(row1, 3, "m,o,n,k,e,y") == 0); - - BOOST_CHECK(row2.values.size() == 3); - BOOST_CHECK(CheckStringsMatch(row2, 0, "banana") == 0); - BOOST_CHECK(CheckStringsMatch(row2, 1, "shoe") == 0); - BOOST_CHECK(CheckStringsMatch(row2, 2, "ice") == 0); -} - -BOOST_AUTO_TEST_SUITE_END() diff --git a/src/armnnQuantizer/QuantizationDataSet.cpp b/src/armnnQuantizer/QuantizationDataSet.cpp index 819b437a55..acd301a470 100644 --- a/src/armnnQuantizer/QuantizationDataSet.cpp +++ b/src/armnnQuantizer/QuantizationDataSet.cpp @@ -4,8 +4,10 @@ // #include "QuantizationDataSet.hpp" -#include "CsvReader.hpp" +#include <fmt/format.h> + +#include <armnn/utility/StringUtils.hpp> #include <armnn/utility/IgnoreUnused.hpp> #include <Filesystem.hpp> @@ -67,49 +69,49 @@ armnn::TensorInfo InputLayerVisitor::GetTensorInfo(armnn::LayerBindingId layerBi } -unsigned int GetPassIdFromCsvRow(std::vector<armnnUtils::CsvRow> csvRows, unsigned int rowIndex) +unsigned int GetPassIdFromCsvRow(std::vector<std::string> tokens, unsigned int lineIndex) { unsigned int passId; try { - passId = static_cast<unsigned int>(std::stoi(csvRows[rowIndex].values[0])); + passId = static_cast<unsigned int>(std::stoi(tokens[0])); } catch (const std::invalid_argument&) { - throw armnn::ParseException("Pass ID [" + csvRows[rowIndex].values[0] + "]" + - " is not correct format on CSV row " + std::to_string(rowIndex)); + throw armnn::ParseException(fmt::format("Pass ID [{}] is not correct format on CSV row {}", + tokens[0], lineIndex)); } return passId; } -armnn::LayerBindingId GetBindingIdFromCsvRow(std::vector<armnnUtils::CsvRow> csvRows, unsigned int rowIndex) +armnn::LayerBindingId GetBindingIdFromCsvRow(std::vector<std::string> tokens, unsigned int lineIndex) { armnn::LayerBindingId bindingId; try { - bindingId = std::stoi(csvRows[rowIndex].values[1]); + bindingId = std::stoi(tokens[1]); } catch (const std::invalid_argument&) { - throw armnn::ParseException("Binding ID [" + csvRows[rowIndex].values[0] + "]" + - " is not correct format on CSV row " + std::to_string(rowIndex)); + throw armnn::ParseException(fmt::format("Binding ID [{}] is not correct format on CSV row {}", + tokens[1], lineIndex)); } return bindingId; } -std::string GetFileNameFromCsvRow(std::vector<armnnUtils::CsvRow> csvRows, unsigned int rowIndex) +std::string GetFileNameFromCsvRow(std::vector<std::string> tokens, unsigned int lineIndex) { - std::string fileName = csvRows[rowIndex].values[2]; + std::string fileName = armnn::stringUtils::StringTrim(tokens[2]); if (!fs::exists(fileName)) { - throw armnn::ParseException("File [ " + fileName + "] provided on CSV row " + std::to_string(rowIndex) + - " does not exist."); + throw armnn::ParseException(fmt::format("File [{}] provided on CSV row {} does not exist.", + fileName, lineIndex)); } if (fileName.empty()) { - throw armnn::ParseException("Filename cannot be empty on CSV row " + std::to_string(rowIndex)); + throw armnn::ParseException(fmt::format("Filename cannot be empty on CSV row {} ", lineIndex)); } return fileName; } @@ -118,34 +120,41 @@ std::string GetFileNameFromCsvRow(std::vector<armnnUtils::CsvRow> csvRows, unsig void QuantizationDataSet::ParseCsvFile() { std::map<unsigned int, QuantizationInput> passIdToQuantizationInput; - armnnUtils::CsvReader reader; if (m_CsvFilePath == "") { throw armnn::Exception("CSV file not specified."); } - // Parse CSV file and extract data - std::vector<armnnUtils::CsvRow> csvRows = reader.ParseFile(m_CsvFilePath); - if (csvRows.empty()) + std::ifstream inf (m_CsvFilePath.c_str()); + std::string line; + std::vector<std::string> tokens; + unsigned int lineIndex = 0; + + if (!inf) { - throw armnn::Exception("CSV file [" + m_CsvFilePath + "] is empty."); + throw armnn::Exception(fmt::format("CSV file {} not found.", m_CsvFilePath)); } - for (unsigned int i = 0; i < csvRows.size(); ++i) + while (getline(inf, line)) { - if (csvRows[i].values.size() != 3) + tokens = armnn::stringUtils::StringTokenizer(line, ","); + + if (tokens.size() != 3) { - throw armnn::Exception("CSV file [" + m_CsvFilePath + "] does not have correct number of entries " + - "on line " + std::to_string(i) + ". Expected 3 entries " + - "but was " + std::to_string(csvRows[i].values.size())); + throw armnn::Exception(fmt::format("CSV file [{}] does not have correct number of entries" \ + "on line {}. Expected 3 entries but was {}.", + m_CsvFilePath, lineIndex, tokens.size())); + } - unsigned int passId = GetPassIdFromCsvRow(csvRows, i); - armnn::LayerBindingId bindingId = GetBindingIdFromCsvRow(csvRows, i); - std::string rawFileName = GetFileNameFromCsvRow(csvRows, i); + unsigned int passId = GetPassIdFromCsvRow(tokens, lineIndex); + armnn::LayerBindingId bindingId = GetBindingIdFromCsvRow(tokens, lineIndex); + std::string rawFileName = GetFileNameFromCsvRow(tokens, lineIndex); AddInputData(passId, bindingId, rawFileName, passIdToQuantizationInput); + + ++lineIndex; } if (passIdToQuantizationInput.empty()) diff --git a/src/armnnUtils/CsvReader.cpp b/src/armnnUtils/CsvReader.cpp deleted file mode 100644 index feee4d18c6..0000000000 --- a/src/armnnUtils/CsvReader.cpp +++ /dev/null @@ -1,63 +0,0 @@ -// -// Copyright © 2017 Arm Ltd. All rights reserved. -// SPDX-License-Identifier: MIT -// - -#include "CsvReader.hpp" -#include "armnn/utility/StringUtils.hpp" - -#include <boost/tokenizer.hpp> - -#include <fstream> -#include <string> -#include <vector> - -using Tokenizer = boost::tokenizer<boost::escaped_list_separator<char>>; - -namespace armnnUtils -{ - -CsvRow ParseLine(const std::string& csvLine) -{ - Tokenizer tokenizer(csvLine); - CsvRow entry; - - for (const auto &token : tokenizer) - { - entry.values.push_back(armnn::stringUtils::StringTrimCopy(token)); - } - return entry; -} - -std::vector<CsvRow> CsvReader::ParseFile(const std::string& csvFile) -{ - std::vector<CsvRow> result; - - std::ifstream in(csvFile.c_str()); - if (!in.is_open()) - return result; - - std::string line; - while (getline(in, line)) - { - if(!line.empty()) - { - CsvRow entry = ParseLine(line); - result.push_back(entry); - } - } - return result; -} - -std::vector<CsvRow> CsvReader::ParseVector(const std::vector<std::string>& csvVector) -{ - std::vector<CsvRow> result; - - for (auto const& line: csvVector) - { - CsvRow entry = ParseLine(line); - result.push_back(entry); - } - return result; -} -} // namespace armnnUtils
\ No newline at end of file diff --git a/src/armnnUtils/CsvReader.hpp b/src/armnnUtils/CsvReader.hpp deleted file mode 100644 index d95f4f7055..0000000000 --- a/src/armnnUtils/CsvReader.hpp +++ /dev/null @@ -1,25 +0,0 @@ -// -// Copyright © 2017 Arm Ltd. All rights reserved. -// SPDX-License-Identifier: MIT -// -#pragma once - -#include <vector> -#include <string> - -namespace armnnUtils -{ - -struct CsvRow -{ - std::vector<std::string> values; -}; - -class CsvReader -{ -public: - static std::vector<CsvRow> ParseFile(const std::string& csvFile); - - static std::vector<CsvRow> ParseVector(const std::vector<std::string>& csvVector); -}; -} // namespace armnnUtils |