From 8271f8144db825960699fffd190ab3e546ee65fc Mon Sep 17 00:00:00 2001 From: Sadik Armagan Date: Fri, 19 Apr 2019 09:55:06 +0100 Subject: IVGCVSW-2899 Create a tool to preprocess the images, generating the RAW tensor data from the image files * ImageTensorGenerator tool generates .raw file contains tensor of the image * ImageCSVFileGenerator tool generates .csv file contains list of .raw files Change-Id: Ic7e148857b9f885044bd69da1077b60104cd6509 Signed-off-by: Sadik Armagan --- tests/CMakeLists.txt | 32 +++ .../ImageCSVFileGenerator.cpp | 205 +++++++++++++++++++ .../ImageTensorGenerator/ImageTensorGenerator.cpp | 224 +++++++++++++++++++++ 3 files changed, 461 insertions(+) create mode 100644 tests/ImageCSVFileGenerator/ImageCSVFileGenerator.cpp create mode 100644 tests/ImageTensorGenerator/ImageTensorGenerator.cpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index cc103c98b6..b7a6d747e0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -284,3 +284,35 @@ if (BUILD_ARMNN_SERIALIZER OR BUILD_CAFFE_PARSER OR BUILD_TF_PARSER OR BUILD_TF_ ${Boost_PROGRAM_OPTIONS_LIBRARY}) addDllCopyCommands(ExecuteNetwork) endif() + +if(BUILD_ARMNN_QUANTIZER) + macro(ImageTensorExecutor executorName) + target_link_libraries(${executorName} ${CMAKE_THREAD_LIBS_INIT}) + if(OPENCL_LIBRARIES) + target_link_libraries(${executorName} ${OPENCL_LIBRARIES}) + endif() + target_link_libraries(${executorName} + ${Boost_SYSTEM_LIBRARY} + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_PROGRAM_OPTIONS_LIBRARY}) + addDllCopyCommands(${executorName}) + endmacro() + + set(ImageTensorGenerator_sources + InferenceTestImage.hpp + InferenceTestImage.cpp + ImageTensorGenerator/ImageTensorGenerator.cpp) + + add_executable_ex(ImageTensorGenerator ${ImageTensorGenerator_sources}) + target_include_directories(ImageTensorGenerator PRIVATE ../src/armnn) + target_include_directories(ImageTensorGenerator PRIVATE ../src/armnnUtils) + + target_link_libraries(ImageTensorGenerator armnn) + ImageTensorExecutor(ImageTensorGenerator) + + set(ImageCSVFileGenerator_sources + ImageCSVFileGenerator/ImageCSVFileGenerator.cpp) + + add_executable_ex(ImageCSVFileGenerator ${ImageCSVFileGenerator_sources}) + ImageTensorExecutor(ImageCSVFileGenerator) +endif() diff --git a/tests/ImageCSVFileGenerator/ImageCSVFileGenerator.cpp b/tests/ImageCSVFileGenerator/ImageCSVFileGenerator.cpp new file mode 100644 index 0000000000..1a14ebab07 --- /dev/null +++ b/tests/ImageCSVFileGenerator/ImageCSVFileGenerator.cpp @@ -0,0 +1,205 @@ +// +// Copyright © 2017 Arm Ltd. All rights reserved. +// SPDX-License-Identifier: MIT +// + +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace +{ + +// parses the command line to extract +// * the directory -i to look through .raw files from (must exist) +// * the name of the file -o the output CSV file path (must not already exist) +class CommandLineProcessor +{ +public: + bool ValidateDirectory(std::string& dir) + { + if (dir.empty()) + { + std::cerr << "No directory specified" << std::endl; + return false; + } + + if (dir[dir.length() - 1] != '/') + { + dir += "/"; + } + + if (!boost::filesystem::exists(dir)) + { + std::cerr << "Directory [" << dir << "] does not exist" << std::endl; + return false; + } + + if (!boost::filesystem::is_directory(dir)) + { + std::cerr << "Given directory [" << dir << "] is not a directory" << std::endl; + return false; + } + + return true; + } + + bool ValidateOutputFile(std::string& outputFileName) + { + if (outputFileName.empty()) + { + std::cerr << "No output file name specified" << std::endl; + return false; + } + + if (boost::filesystem::exists(outputFileName)) + { + std::cerr << "Output file [" << outputFileName << "] already exists" << std::endl; + return false; + } + + if (boost::filesystem::is_directory(outputFileName)) + { + std::cerr << "Output file [" << outputFileName << "] is a directory" << std::endl; + return false; + } + + boost::filesystem::path outputPath(outputFileName); + if (!boost::filesystem::exists(outputPath.parent_path())) + { + std::cerr << "Directory [" << outputPath.parent_path().c_str() << "] does not exist" << std::endl; + return false; + } + + return true; + } + + bool ProcessCommandLine(int argc, char* argv[]) + { + namespace po = boost::program_options; + + po::options_description desc("Options"); + try + { + desc.add_options() + ("help,h", "Display help messages") + ("indir,i", po::value(&m_InputDirectory)->required(), + "Directory that .raw files are stored in") + ("outfile,o", po::value(&m_OutputFileName)->required(), + "Output CSV file path"); + } + catch (const std::exception& e) + { + std::cerr << "Fatal internal error: [" << e.what() << "]" << std::endl; + return false; + } + + po::variables_map vm; + + try + { + po::store(po::parse_command_line(argc, argv, desc), vm); + + if (vm.count("help")) + { + std::cout << desc << std::endl; + return false; + } + + po::notify(vm); + } + catch (const po::error& e) + { + std::cerr << e.what() << std::endl << std::endl; + std::cerr << desc << std::endl; + return false; + } + + if (!ValidateDirectory(m_InputDirectory)) + { + return false; + } + + if (!ValidateOutputFile(m_OutputFileName)) + { + return false; + } + + return true; + } + + std::string GetInputDirectory() {return m_InputDirectory;} + std::string GetOutputFileName() {return m_OutputFileName;} + +private: + std::string m_InputDirectory; + std::string m_OutputFileName; +}; + +} // namespace anonymous + +int main(int argc, char* argv[]) +{ + CommandLineProcessor cmdline; + if (!cmdline.ProcessCommandLine(argc, argv)) + { + return -1; + } + + namespace fs = boost::filesystem; + + const std::string fileFormat(".raw"); + const std::string bindingId("0"); + + const std::string rawDirectory(cmdline.GetInputDirectory()); + const std::string outputPath(cmdline.GetOutputFileName()); + + std::vector rawFiles; + for (auto& entry : boost::make_iterator_range(fs::directory_iterator(rawDirectory), {})) + { + if (entry.path().extension().c_str() == fileFormat) + { + rawFiles.push_back(entry.path()); + } + } + + if (!rawFiles.empty()) + { + unsigned int pass = 0; + std::ofstream refinementData; + refinementData.open(outputPath, std::ofstream::out); + if (refinementData.is_open()) + { + for (auto const& raw : rawFiles) + { + refinementData << pass << ", " << bindingId << ", " << raw.c_str() << "\n"; + if (!refinementData) + { + std::cerr << "Failed to write to output file: " << outputPath << std::endl; + continue; + } + ++pass; + } + refinementData.close(); + } + else + { + std::cerr << "Failed to open output file: " << outputPath << std::endl; + return -1; + } + } + else + { + std::cerr << "No matching files with the \".raw\" extension found in the directory: " + << rawDirectory << std::endl; + return -1; + } + + return 0; +} \ No newline at end of file diff --git a/tests/ImageTensorGenerator/ImageTensorGenerator.cpp b/tests/ImageTensorGenerator/ImageTensorGenerator.cpp new file mode 100644 index 0000000000..1f537745b4 --- /dev/null +++ b/tests/ImageTensorGenerator/ImageTensorGenerator.cpp @@ -0,0 +1,224 @@ +// +// Copyright © 2017 Arm Ltd. All rights reserved. +// SPDX-License-Identifier: MIT +// + +#include "../InferenceTestImage.hpp" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace +{ + +// parses the command line to extract +// * the input image file -i the input image file path (must exist) +// * the layout -l the data layout output generated with (optional - default value is NHWC) +// * the output file -o the output raw tensor file path (must not already exist) +class CommandLineProcessor +{ +public: + bool ValidateInputFile(const std::string& inputFileName) + { + if (inputFileName.empty()) + { + std::cerr << "No input file name specified" << std::endl; + return false; + } + + if (!boost::filesystem::exists(inputFileName)) + { + std::cerr << "Input file [" << inputFileName << "] does not exist" << std::endl; + return false; + } + + if (boost::filesystem::is_directory(inputFileName)) + { + std::cerr << "Input file [" << inputFileName << "] is a directory" << std::endl; + return false; + } + + return true; + } + + bool ValidateLayout(const std::string& layout) + { + if (layout.empty()) + { + std::cerr << "No layout specified" << std::endl; + return false; + } + + std::vector supportedLayouts = { + "NHWC", + "NCHW" + }; + + auto iterator = std::find(supportedLayouts.begin(), supportedLayouts.end(), layout); + if (iterator == supportedLayouts.end()) + { + std::cerr << "Layout [" << layout << "] is not supported" << std::endl; + return false; + } + + return true; + } + + bool ValidateOutputFile(std::string& outputFileName) + { + if (outputFileName.empty()) + { + std::cerr << "No output file name specified" << std::endl; + return false; + } + + if (boost::filesystem::exists(outputFileName)) + { + std::cerr << "Output file [" << outputFileName << "] already exists" << std::endl; + return false; + } + + if (boost::filesystem::is_directory(outputFileName)) + { + std::cerr << "Output file [" << outputFileName << "] is a directory" << std::endl; + return false; + } + + boost::filesystem::path outputPath(outputFileName); + if (!boost::filesystem::exists(outputPath.parent_path())) + { + std::cerr << "Output directory [" << outputPath.parent_path().c_str() << "] does not exist" << std::endl; + return false; + } + + return true; + } + + bool ProcessCommandLine(int argc, char* argv[]) + { + namespace po = boost::program_options; + + po::options_description desc("Options"); + try + { + desc.add_options() + ("help,h", "Display help messages") + ("infile,i", po::value(&m_InputFileName)->required(), + "Input image file to generate tensor from") + ("layout,l", po::value(&m_Layout)->default_value("NHWC"), + "Output data layout, \"NHWC\" or \"NCHW\", default value NHWC") + ("outfile,o", po::value(&m_OutputFileName)->required(), + "Output raw tensor file path"); + } + catch (const std::exception& e) + { + std::cerr << "Fatal internal error: [" << e.what() << "]" << std::endl; + return false; + } + + po::variables_map vm; + + try + { + po::store(po::parse_command_line(argc, argv, desc), vm); + + if (vm.count("help")) + { + std::cout << desc << std::endl; + return false; + } + + po::notify(vm); + } + catch (const po::error& e) + { + std::cerr << e.what() << std::endl << std::endl; + std::cerr << desc << std::endl; + return false; + } + + if (!ValidateInputFile(m_InputFileName)) + { + return false; + } + + if (!ValidateLayout(m_Layout)) + { + return false; + } + + if (!ValidateOutputFile(m_OutputFileName)) + { + return false; + } + + return true; + } + + std::string GetInputFileName() {return m_InputFileName;} + std::string GetLayout() {return m_Layout;} + std::string GetOutputFileName() {return m_OutputFileName;} + +private: + std::string m_InputFileName; + std::string m_Layout; + std::string m_OutputFileName; +}; + +} // namespace anonymous + +int main(int argc, char* argv[]) +{ + CommandLineProcessor cmdline; + if (!cmdline.ProcessCommandLine(argc, argv)) + { + return -1; + } + + const std::string imagePath(cmdline.GetInputFileName()); + const std::string outputPath(cmdline.GetOutputFileName()); + + // generate image tensor + std::vector imageData; + try + { + InferenceTestImage testImage(imagePath.c_str()); + imageData = cmdline.GetLayout() == "NHWC" + ? GetImageDataAsNormalizedFloats(ImageChannelLayout::Rgb, testImage) + : GetImageDataInArmNnLayoutAsNormalizedFloats(ImageChannelLayout::Rgb, testImage); + } + catch (const InferenceTestImageException& e) + { + BOOST_LOG_TRIVIAL(fatal) << "Failed to load image file " << imagePath << " with error: " << e.what(); + return -1; + } + + std::ofstream imageTensorFile; + imageTensorFile.open(outputPath, std::ofstream::out); + if (imageTensorFile.is_open()) + { + std::copy(imageData.begin(), imageData.end(), std::ostream_iterator(imageTensorFile, " ")); + if (!imageTensorFile) + { + BOOST_LOG_TRIVIAL(fatal) << "Failed to write to output file" << outputPath; + imageTensorFile.close(); + return -1; + } + imageTensorFile.close(); + } + else + { + BOOST_LOG_TRIVIAL(fatal) << "Failed to open output file" << outputPath; + return -1; + } + + return 0; +} \ No newline at end of file -- cgit v1.2.1