diff options
Diffstat (limited to 'scripts/py/gen_test_data_cpp.py')
| -rw-r--r-- | scripts/py/gen_test_data_cpp.py | 317 |
1 files changed, 232 insertions, 85 deletions
diff --git a/scripts/py/gen_test_data_cpp.py b/scripts/py/gen_test_data_cpp.py index a9e2b75..1ee55ff 100644 --- a/scripts/py/gen_test_data_cpp.py +++ b/scripts/py/gen_test_data_cpp.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2021 - 2022 Arm Limited and/or its affiliates <open-source-office@arm.com> +# SPDX-FileCopyrightText: Copyright 2021-2023 Arm Limited and/or its affiliates <open-source-office@arm.com> # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,81 +18,170 @@ Utility script to convert a set of pairs of npy files in a given location into corresponding cpp files and a single hpp file referencing the vectors from the cpp files. """ -import datetime import math -import os -import numpy as np +import typing +from argparse import ArgumentParser +from dataclasses import dataclass from pathlib import Path -from argparse import ArgumentParser +import numpy as np from jinja2 import Environment, FileSystemLoader +from gen_utils import GenUtils + +# pylint: disable=duplicate-code parser = ArgumentParser() -parser.add_argument("--data_folder_path", type=str, help="path to ifm-ofm npy folder to convert.") -parser.add_argument("--source_folder_path", type=str, help="path to source folder to be generated.") -parser.add_argument("--header_folder_path", type=str, help="path to header folder to be generated.") -parser.add_argument("--usecase", type=str, default="", help="Test data file suffix.") -parser.add_argument("--namespaces", action='append', default=[]) -parser.add_argument("--license_template", type=str, help="Header template file", - default="header_template.txt") -parser.add_argument("-v", "--verbosity", action="store_true") -args = parser.parse_args() +parser.add_argument( + "--data_folder_path", + type=str, + help="path to ifm-ofm npy folder to convert." +) + +parser.add_argument( + "--source_folder_path", + type=str, + help="path to source folder to be generated." +) + +parser.add_argument( + "--header_folder_path", + type=str, + help="path to header folder to be generated." +) + +parser.add_argument( + "--usecase", + type=str, + default="", + help="Test data file suffix." +) + +parser.add_argument( + "--namespaces", + action='append', + default=[] +) + +parser.add_argument( + "--license_template", + type=str, + help="Header template file", + default="header_template.txt" +) + +parser.add_argument( + "-v", + "--verbosity", + action="store_true" +) + +parsed_args = parser.parse_args() env = Environment(loader=FileSystemLoader(Path(__file__).parent / 'templates'), trim_blocks=True, lstrip_blocks=True) -def write_hpp_file(header_filename, cc_file_path, header_template_file, num_ifms, num_ofms, - ifm_array_names, ifm_sizes, ofm_array_names, ofm_sizes, iofm_data_type): - header_file_path = Path(args.header_folder_path) / header_filename +# pylint: enable=duplicate-code +@dataclass +class TestDataParams: + """ + Template params for TestData.hpp + TestData.ccc + """ + ifm_count: int + ofm_count: int + ifm_var_names: typing.List[str] + ifm_var_sizes: typing.List[int] + ofm_var_names: typing.List[str] + ofm_var_sizes: typing.List[int] + data_type: str + + +@dataclass +class IofmParams: + """ + Template params for iofmdata.cc + """ + var_name: str + data_type: str + + +def write_hpp_file( + template_params: TestDataParams, + header_filename: str, + cc_file_path: str, + header_template_file: str +): + """ + Write TestData.hpp and TestData.cc + + @param template_params: Template parameters + @param header_filename: TestData.hpp path + @param cc_file_path: TestData.cc path + @param header_template_file: Header template file name + """ + header_file_path = Path(parsed_args.header_folder_path) / header_filename print(f"++ Generating {header_file_path}") - header_template = env.get_template(header_template_file) - hdr = header_template.render(script_name=Path(__file__).name, - gen_time=datetime.datetime.now(), - year=datetime.datetime.now().year) - env.get_template('TestData.hpp.template').stream(common_template_header=hdr, - ifm_count=num_ifms, - ofm_count=num_ofms, - ifm_var_names=ifm_array_names, - ifm_var_sizes=ifm_sizes, - ofm_var_names=ofm_array_names, - ofm_var_sizes=ofm_sizes, - data_type=iofm_data_type, - namespaces=args.namespaces) \ + hdr = GenUtils.gen_header(env, header_template_file) + env \ + .get_template('TestData.hpp.template') \ + .stream(common_template_header=hdr, + ifm_count=template_params.ifm_count, + ofm_count=template_params.ofm_count, + ifm_var_names=template_params.ifm_var_names, + ifm_var_sizes=template_params.ifm_var_sizes, + ofm_var_names=template_params.ofm_var_names, + ofm_var_sizes=template_params.ofm_var_sizes, + data_type=template_params.data_type, + namespaces=parsed_args.namespaces) \ .dump(str(header_file_path)) - env.get_template('TestData.cc.template').stream(common_template_header=hdr, - include_h=header_filename, - ifm_var_names=ifm_array_names, - ofm_var_names=ofm_array_names, - data_type=iofm_data_type, - namespaces=args.namespaces) \ + env \ + .get_template('TestData.cc.template') \ + .stream(common_template_header=hdr, + include_h=header_filename, + ifm_var_names=template_params.ifm_var_names, + ofm_var_names=template_params.ofm_var_names, + data_type=template_params.data_type, + namespaces=parsed_args.namespaces) \ .dump(str(cc_file_path)) -def write_individual_cc_file(filename, cc_filename, header_filename, header_template_file, array_name, iofm_data_type): +def write_individual_cc_file( + template_params: IofmParams, + header_filename: str, + filename: str, + cc_filename: Path, + header_template_file: str +): + """ + Write iofmdata.cc + + @param template_params: Template parameters + @param header_filename: Header file name + @param filename: Input file name + @param cc_filename: iofmdata.cc file name + @param header_template_file: Header template file name + """ print(f"++ Converting {filename} to {cc_filename.name}") - header_template = env.get_template(header_template_file) - hdr = header_template.render(script_name=Path(__file__).name, - gen_time=datetime.datetime.now(), - file_name=filename, - year=datetime.datetime.now().year) + hdr = GenUtils.gen_header(env, header_template_file, filename) # Convert the image and write it to the cc file - fm_data = (np.load(Path(args.data_folder_path) / filename)).flatten() + fm_data = (np.load(Path(parsed_args.data_folder_path) / filename)).flatten() type(fm_data.dtype) hex_line_generator = (', '.join(map(hex, sub_arr)) for sub_arr in np.array_split(fm_data, math.ceil(len(fm_data) / 20))) - env.get_template('iofmdata.cc.template').stream(common_template_header=hdr, - include_h=header_filename, - var_name=array_name, - fm_data=hex_line_generator, - data_type=iofm_data_type, - namespaces=args.namespaces) \ + env \ + .get_template('iofmdata.cc.template') \ + .stream(common_template_header=hdr, + include_h=header_filename, + var_name=template_params.var_name, + fm_data=hex_line_generator, + data_type=template_params.data_type, + namespaces=parsed_args.namespaces) \ .dump(str(cc_filename)) @@ -104,59 +193,117 @@ def get_npy_vec_size(filename: str) -> int: Return: size in bytes """ - data = np.load(Path(args.data_folder_path) / filename) + data = np.load(Path(parsed_args.data_folder_path) / filename) return data.size * data.dtype.itemsize -def main(args): - # Keep the count of the images converted - ifm_array_names = [] - ofm_array_names = [] +def write_cc_files(args, count, iofm_data_type, add_usecase_fname, prefix): + """ + Write all cc files + + @param args: User-provided args + @param count: File count + @param iofm_data_type: Data type + @param add_usecase_fname: Use case suffix + @param prefix: Prefix (ifm/ofm) + @return: Names and sizes of generated C++ arrays + """ + array_names = [] + sizes = [] + + header_filename = get_header_filename(add_usecase_fname) + # In the data_folder_path there should be pairs of ifm-ofm + # It's assumed the ifm-ofm naming convention: ifm0.npy-ofm0.npy, ifm1.npy-ofm1.npy + # count = int(len(list(Path(args.data_folder_path).glob(f'{prefix}*.npy')))) + + for idx in range(count): + # Save the fm cc file + base_name = prefix + str(idx) + filename = base_name + ".npy" + array_name = base_name + add_usecase_fname + cc_filename = Path(args.source_folder_path) / (array_name + ".cc") + array_names.append(array_name) + + template_params = IofmParams( + var_name=array_name, + data_type=iofm_data_type, + ) + + write_individual_cc_file( + template_params, header_filename, filename, cc_filename, args.license_template + ) + sizes.append(get_npy_vec_size(filename)) + + return array_names, sizes + + +def get_header_filename(use_case_filename): + """ + Get the header file name from the use case file name + + @param use_case_filename: The use case file name + @return: The header file name + """ + return "TestData" + use_case_filename + ".hpp" + + +def get_cc_filename(use_case_filename): + """ + Get the cc file name from the use case file name + + @param use_case_filename: The use case file name + @return: The cc file name + """ + return "TestData" + use_case_filename + ".cc" + + +def main(args): + """ + Generate test data + @param args: Parsed args + """ add_usecase_fname = ("_" + args.usecase) if (args.usecase != "") else "" - header_filename = "TestData" + add_usecase_fname + ".hpp" - common_cc_filename = "TestData" + add_usecase_fname + ".cc" + header_filename = get_header_filename(add_usecase_fname) + common_cc_filename = get_cc_filename(add_usecase_fname) # In the data_folder_path there should be pairs of ifm-ofm # It's assumed the ifm-ofm naming convention: ifm0.npy-ofm0.npy, ifm1.npy-ofm1.npy ifms_count = int(len(list(Path(args.data_folder_path).glob('ifm*.npy')))) ofms_count = int(len(list(Path(args.data_folder_path).glob('ofm*.npy')))) - #i_ofms_count = int(len([name for name in os.listdir(os.path.join(args.data_folder_path)) if name.lower().endswith('.npy')]) / 2) - iofm_data_type = "int8_t" if ifms_count > 0: - iofm_data_type = "int8_t" if (np.load(Path(args.data_folder_path) / "ifm0.npy").dtype == np.int8) else "uint8_t" - - ifm_sizes = [] - ofm_sizes = [] + iofm_data_type = "int8_t" \ + if (np.load(str(Path(args.data_folder_path) / "ifm0.npy")).dtype == np.int8) \ + else "uint8_t" - for idx in range(ifms_count): - # Save the fm cc file - base_name = "ifm" + str(idx) - filename = base_name+".npy" - array_name = base_name + add_usecase_fname - cc_filename = Path(args.source_folder_path) / (array_name + ".cc") - ifm_array_names.append(array_name) - write_individual_cc_file(filename, cc_filename, header_filename, args.license_template, array_name, iofm_data_type) - ifm_sizes.append(get_npy_vec_size(filename)) + ifm_array_names, ifm_sizes = write_cc_files( + args, ifms_count, iofm_data_type, add_usecase_fname, prefix="ifm" + ) - for idx in range(ofms_count): - # Save the fm cc file - base_name = "ofm" + str(idx) - filename = base_name+".npy" - array_name = base_name + add_usecase_fname - cc_filename = Path(args.source_folder_path) / (array_name + ".cc") - ofm_array_names.append(array_name) - write_individual_cc_file(filename, cc_filename, header_filename, args.license_template, array_name, iofm_data_type) - ofm_sizes.append(get_npy_vec_size(filename)) + ofm_array_names, ofm_sizes = write_cc_files( + args, ofms_count, iofm_data_type, add_usecase_fname, prefix="ofm" + ) common_cc_filepath = Path(args.source_folder_path) / common_cc_filename - write_hpp_file(header_filename, common_cc_filepath, args.license_template, - ifms_count, ofms_count, ifm_array_names, ifm_sizes, ofm_array_names, ofm_sizes, iofm_data_type) + + template_params = TestDataParams( + ifm_count=ifms_count, + ofm_count=ofms_count, + ifm_var_names=ifm_array_names, + ifm_var_sizes=ifm_sizes, + ofm_var_names=ofm_array_names, + ofm_var_sizes=ofm_sizes, + data_type=iofm_data_type, + ) + + write_hpp_file( + template_params, header_filename, common_cc_filepath, args.license_template + ) if __name__ == '__main__': - if args.verbosity: - print("Running gen_test_data_cpp with args: "+str(args)) - main(args) + if parsed_args.verbosity: + print("Running gen_test_data_cpp with args: " + str(parsed_args)) + main(parsed_args) |