diff options
author | Georgios Pinitas <georgios.pinitas@arm.com> | 2019-05-16 14:13:03 +0100 |
---|---|---|
committer | Georgios Pinitas <georgios.pinitas@arm.com> | 2019-05-22 12:38:27 +0000 |
commit | b8d5b958d489c21214cc23755d442e4e78f03878 (patch) | |
tree | 24e14a2258f5b8ff884d768f09c00e32923ee3b2 | |
parent | ac33d7eb7bf0db65c3663d9707e509ca91337349 (diff) | |
download | ComputeLibrary-b8d5b958d489c21214cc23755d442e4e78f03878.tar.gz |
COMPMID-2166: Add tests for importing memory mapped files.
Change-Id: I011773bbe0bf6774a9718d414b4b297b4d8996c0
Signed-off-by: Georgios Pinitas <georgios.pinitas@arm.com>
Reviewed-on: https://review.mlplatform.org/c/1179
Reviewed-by: Michalis Spyrou <michalis.spyrou@arm.com>
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
Tested-by: Arm Jenkins <bsgcomp@arm.com>
-rw-r--r-- | SConscript | 1 | ||||
-rw-r--r-- | arm_compute/core/utils/misc/MMappedFile.h | 110 | ||||
-rwxr-xr-x | scripts/clang_tidy_rules.py | 1 | ||||
-rw-r--r-- | src/core/utils/misc/MMappedFile.cpp | 192 | ||||
-rw-r--r-- | tests/validation/CL/UNIT/TensorAllocator.cpp | 69 | ||||
-rw-r--r-- | tests/validation/NEON/UNIT/TensorAllocator.cpp | 60 |
6 files changed, 433 insertions, 0 deletions
diff --git a/SConscript b/SConscript index 62b6073bf3..45c4ccc414 100644 --- a/SConscript +++ b/SConscript @@ -164,6 +164,7 @@ core_files += Glob('src/core/CPP/kernels/*.cpp') core_files += Glob('src/core/utils/helpers/*.cpp') core_files += Glob('src/core/utils/io/*.cpp') core_files += Glob('src/core/utils/quantization/*.cpp') +core_files += Glob('src/core/utils/misc/*.cpp') if env["logging"]: core_files += Glob('src/core/utils/logging/*.cpp') diff --git a/arm_compute/core/utils/misc/MMappedFile.h b/arm_compute/core/utils/misc/MMappedFile.h new file mode 100644 index 0000000000..4b13adb5e9 --- /dev/null +++ b/arm_compute/core/utils/misc/MMappedFile.h @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2019 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef __ARM_COMPUTE_MISC_MMAPPED_FILE_H__ +#define __ARM_COMPUTE_MISC_MMAPPED_FILE_H__ + +#if !defined(BARE_METAL) + +#include <string> +#include <utility> + +namespace arm_compute +{ +namespace utils +{ +namespace mmap_io +{ +/** Memory mapped file class */ +class MMappedFile +{ +public: + /** Constructor */ + MMappedFile(); + /** Constructor + * + * @note file will be created if it doesn't exist. + * + * @param[in] filename File to be mapped, if doesn't exist will be created. + * @param[in] size Size of file to map + * @param[in] offset Offset to mapping point, should be multiple of page size + */ + MMappedFile(std::string filename, size_t size, size_t offset); + /** Prevent instances of this class from being copied (As this class contains pointers) */ + MMappedFile(const MMappedFile &) = delete; + /** Default move constructor */ + MMappedFile(MMappedFile &&) = default; + /** Prevent instances of this class from being copied (As this class contains pointers) */ + MMappedFile &operator=(const MMappedFile &) = delete; + /** Default move assignment operator */ + MMappedFile &operator=(MMappedFile &&) = default; + /** Destructor */ + ~MMappedFile(); + /** Opens and maps a file + * + * @note file will be created if it doesn't exist. + * + * @param[in] filename File to be mapped, if doesn't exist will be created. + * @param[in] size Size of file to map. If 0 all the file will be mapped. + * @param[in] offset Offset to mapping point, should be multiple of page size. + * + * @return True if operation was successful else false + */ + bool map(const std::string &filename, size_t size, size_t offset); + /** Unmaps and closes file */ + void release(); + /** Mapped data accessor + * + * @return Pointer to the mapped data, nullptr if not mapped + */ + unsigned char *data(); + /** File size accessor + * + * @return Size of file + */ + size_t file_size() const; + /** Map size accessor + * + * @return Mapping size + */ + size_t map_size() const; + /** Checks if file mapped + * + * @return True if file is mapped else false + */ + bool is_mapped() const; + +private: + std::string _filename; + size_t _file_size; + size_t _map_size; + size_t _map_offset; + FILE *_fp; + void *_data; +}; +} // namespace mmap_io +} // namespace utils +} // namespace arm_compute +#endif // !defined(BARE_METAL) + +#endif /* __ARM_COMPUTE_MISC_MMAPPED_FILE_H__ */ diff --git a/scripts/clang_tidy_rules.py b/scripts/clang_tidy_rules.py index 4e6ede3efb..9b5fafb979 100755 --- a/scripts/clang_tidy_rules.py +++ b/scripts/clang_tidy_rules.py @@ -66,6 +66,7 @@ def filter_clang_tidy_lines( lines ): if ("uninitialized record type: '__ret'" in line or "local variable '__bound_functor' is still referred to by the global variable '__once_callable'" in line or "assigning newly created 'gsl::owner<>'" in line or + "calling legacy resource function without passing a 'gsl::owner<>'" in line or "deleting a pointer through a type that is not marked 'gsl::owner<>'" in line or (any(f in line for f in ["Error.cpp","Error.h"]) and "thrown exception type is not nothrow copy constructible" in line) or (any(f in line for f in ["Error.cpp","Error.h"]) and "uninitialized record type: 'args'" in line) or diff --git a/src/core/utils/misc/MMappedFile.cpp b/src/core/utils/misc/MMappedFile.cpp new file mode 100644 index 0000000000..6d0b0bed6a --- /dev/null +++ b/src/core/utils/misc/MMappedFile.cpp @@ -0,0 +1,192 @@ +/* + * Copyright (c) 2019 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#if !defined(BARE_METAL) + +#include "arm_compute/core/utils/misc/MMappedFile.h" + +#include <cstdio> +#include <cstring> +#include <tuple> + +#include <fcntl.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +namespace arm_compute +{ +namespace utils +{ +namespace mmap_io +{ +namespace +{ +/** File size accessor + * + * @param[in] filename File to extract its size + * + * @return A pair of size and status. + */ +std::pair<size_t, bool> get_file_size(const std::string &filename) +{ + struct stat st; // NOLINT + memset(&st, 0, sizeof(struct stat)); + if(stat(filename.c_str(), &st) == 0) + { + return std::make_pair(st.st_size, true); + } + else + { + return std::make_pair(0, false); + } +} + +/** Get OS page size + * + * @return Page size + */ +size_t get_page_size() +{ + return sysconf(_SC_PAGESIZE); +} +} // namespace + +MMappedFile::MMappedFile() + : _filename(), _file_size(0), _map_size(0), _map_offset(0), _fp(nullptr), _data(nullptr) +{ +} + +MMappedFile::MMappedFile(std::string filename, size_t size, size_t offset) + : _filename(std::move(filename)), _file_size(0), _map_size(size), _map_offset(offset), _fp(nullptr), _data(nullptr) +{ + map(_filename, _map_size, _map_offset); +} + +MMappedFile::~MMappedFile() +{ + release(); +} + +bool MMappedFile::map(const std::string &filename, size_t size, size_t offset) +{ + // Check if file is mapped + if(is_mapped()) + { + return false; + } + + // Open file + _fp = fopen(filename.c_str(), "a+be"); + if(_fp == nullptr) + { + return false; + } + + // Extract file descriptor + int fd = fileno(_fp); + bool status = fd >= 0; + if(status) + { + // Get file size + std::tie(_file_size, status) = get_file_size(_filename); + + if(status) + { + // Map all file from offset if map size is 0 + _map_size = (size == 0) ? _file_size : size; + _map_offset = offset; + + // Check offset mapping + if((_map_offset > _file_size) || (_map_offset % get_page_size() != 0)) + { + status = false; + } + else + { + // Truncate to file size + if(_map_offset + _map_size > _file_size) + { + _map_size = _file_size - _map_offset; + } + + // Perform mapping + _data = ::mmap(nullptr, _map_size, PROT_WRITE, MAP_SHARED, fd, _map_offset); + } + } + } + + if(!status) + { + fclose(_fp); + } + + return status; +} + +void MMappedFile::release() +{ + // Unmap file + if(_data != nullptr) + { + ::munmap(_data, _file_size); + _data = nullptr; + } + + // Close file + if(_fp != nullptr) + { + fclose(_fp); + _fp = nullptr; + } + + // Clear variables + _file_size = 0; + _map_size = 0; + _map_offset = 0; +} + +unsigned char *MMappedFile::data() +{ + return static_cast<unsigned char *>(_data); +} + +size_t MMappedFile::file_size() const +{ + return _file_size; +} + +size_t MMappedFile::map_size() const +{ + return _map_size; +} + +bool MMappedFile::is_mapped() const +{ + return _data != nullptr; +} +} // namespace mmap_io +} // namespace utils +} // namespace arm_compute +#endif // !defined(BARE_METAL) diff --git a/tests/validation/CL/UNIT/TensorAllocator.cpp b/tests/validation/CL/UNIT/TensorAllocator.cpp index 7e47e3d983..e5b37d8387 100644 --- a/tests/validation/CL/UNIT/TensorAllocator.cpp +++ b/tests/validation/CL/UNIT/TensorAllocator.cpp @@ -23,6 +23,7 @@ */ #include "arm_compute/runtime/CL/CLTensorAllocator.h" +#include "arm_compute/core/utils/misc/MMappedFile.h" #include "arm_compute/runtime/CL/CLMemoryGroup.h" #include "arm_compute/runtime/CL/CLScheduler.h" #include "arm_compute/runtime/CL/functions/CLActivationLayer.h" @@ -166,6 +167,74 @@ TEST_CASE(ImportMemoryMalloc, framework::DatasetMode::ALL) } } +#if !defined(BARE_METAL) +TEST_CASE(ImportMemoryMappedFile, framework::DatasetMode::ALL) +{ + // Check if import extension is supported + if(!device_supports_extension(CLKernelLibrary::get().get_device(), "cl_arm_import_memory_host")) + { + return; + } + else + { + const ActivationLayerInfo act_info(ActivationLayerInfo::ActivationFunction::RELU); + const TensorShape shape = TensorShape(24U, 16U, 3U); + const DataType data_type = DataType::F32; + + // Create tensor + const TensorInfo info(shape, 1, data_type); + CLTensor tensor; + tensor.allocator()->init(info); + + // Create and configure activation function + CLActivationLayer act_func; + act_func.configure(&tensor, nullptr, act_info); + + // Get number of elements + const size_t total_size_in_elems = tensor.info()->tensor_shape().total_size(); + const size_t total_size_in_bytes = tensor.info()->total_size(); + + // Create file + std::ofstream output_file("test_mmap_import.bin", std::ios::binary | std::ios::out); + output_file.seekp(total_size_in_bytes - 1); + output_file.write("", 1); + output_file.close(); + + // Map file + utils::mmap_io::MMappedFile mmapped_file("test_mmap_import.bin", 0 /** Whole file */, 0); + ARM_COMPUTE_EXPECT(mmapped_file.is_mapped(), framework::LogLevel::ERRORS); + unsigned char *data = mmapped_file.data(); + + cl::Buffer wrapped_buffer(import_malloc_memory_helper(data, total_size_in_bytes)); + ARM_COMPUTE_EXPECT(bool(tensor.allocator()->import_memory(wrapped_buffer)), framework::LogLevel::ERRORS); + ARM_COMPUTE_EXPECT(!tensor.info()->is_resizable(), framework::LogLevel::ERRORS); + + // Fill tensor + std::uniform_real_distribution<float> distribution(-5.f, 5.f); + std::mt19937 gen(library->seed()); + auto *typed_ptr = reinterpret_cast<float *>(data); + for(unsigned int i = 0; i < total_size_in_elems; ++i) + { + typed_ptr[i] = distribution(gen); + } + + // Execute function and sync + act_func.run(); + CLScheduler::get().sync(); + + // Validate result by checking that the input has no negative values + for(unsigned int i = 0; i < total_size_in_elems; ++i) + { + ARM_COMPUTE_EXPECT(typed_ptr[i] >= 0, framework::LogLevel::ERRORS); + } + + // Release resources + tensor.allocator()->free(); + ARM_COMPUTE_EXPECT(tensor.info()->is_resizable(), framework::LogLevel::ERRORS); + } +} +#endif // !defined(BARE_METAL) + TEST_SUITE_END() // TensorAllocator TEST_SUITE_END() // UNIT TEST_SUITE_END() // CL diff --git a/tests/validation/NEON/UNIT/TensorAllocator.cpp b/tests/validation/NEON/UNIT/TensorAllocator.cpp index 7ba83c11b3..217933da48 100644 --- a/tests/validation/NEON/UNIT/TensorAllocator.cpp +++ b/tests/validation/NEON/UNIT/TensorAllocator.cpp @@ -23,6 +23,7 @@ */ #include "arm_compute/runtime/TensorAllocator.h" +#include "arm_compute/core/utils/misc/MMappedFile.h" #include "arm_compute/core/utils/misc/Utility.h" #include "arm_compute/runtime/MemoryGroup.h" #include "arm_compute/runtime/MemoryRegion.h" @@ -141,6 +142,65 @@ TEST_CASE(ImportMemoryMalloc, framework::DatasetMode::ALL) ARM_COMPUTE_EXPECT(tensor.info()->is_resizable(), framework::LogLevel::ERRORS); } +#if !defined(BARE_METAL) +TEST_CASE(ImportMemoryMappedFile, framework::DatasetMode::ALL) +{ + const ActivationLayerInfo act_info(ActivationLayerInfo::ActivationFunction::RELU); + const TensorShape shape = TensorShape(24U, 16U, 3U); + const DataType data_type = DataType::F32; + + // Create tensor + const TensorInfo info(shape, 1, data_type); + Tensor tensor; + tensor.allocator()->init(info); + + // Create and configure activation function + NEActivationLayer act_func; + act_func.configure(&tensor, nullptr, act_info); + + // Get number of elements + const size_t total_size_in_elems = tensor.info()->tensor_shape().total_size(); + const size_t total_size_in_bytes = tensor.info()->total_size(); + + // Create file + std::ofstream output_file("test_mmap_import.bin", std::ios::binary | std::ios::out); + output_file.seekp(total_size_in_bytes - 1); + output_file.write("", 1); + output_file.close(); + + // Map file + utils::mmap_io::MMappedFile mmapped_file("test_mmap_import.bin", 0 /** Whole file */, 0); + ARM_COMPUTE_EXPECT(mmapped_file.is_mapped(), framework::LogLevel::ERRORS); + unsigned char *data = mmapped_file.data(); + + // Import memory mapped memory + ARM_COMPUTE_EXPECT(bool(tensor.allocator()->import_memory(data)), framework::LogLevel::ERRORS); + ARM_COMPUTE_EXPECT(!tensor.info()->is_resizable(), framework::LogLevel::ERRORS); + + // Fill tensor + std::uniform_real_distribution<float> distribution(-5.f, 5.f); + std::mt19937 gen(library->seed()); + auto *typed_ptr = reinterpret_cast<float *>(data); + for(unsigned int i = 0; i < total_size_in_elems; ++i) + { + typed_ptr[i] = distribution(gen); + } + + // Execute function and sync + act_func.run(); + + // Validate result by checking that the input has no negative values + for(unsigned int i = 0; i < total_size_in_elems; ++i) + { + ARM_COMPUTE_EXPECT(typed_ptr[i] >= 0, framework::LogLevel::ERRORS); + } + + // Release resources + tensor.allocator()->free(); + ARM_COMPUTE_EXPECT(tensor.info()->is_resizable(), framework::LogLevel::ERRORS); +} +#endif // !defined(BARE_METAL) + TEST_CASE(AlignedAlloc, framework::DatasetMode::ALL) { // Init tensor info |