From 3c79893217bc632c9b0efa815091bef3c779490c Mon Sep 17 00:00:00 2001 From: alexander Date: Fri, 26 Mar 2021 21:42:19 +0000 Subject: Opensource ML embedded evaluation kit Change-Id: I12e807f19f5cacad7cef82572b6dd48252fd61fd --- scripts/py/gen_audio.py | 48 +++++++ scripts/py/gen_audio_cpp.py | 153 +++++++++++++++++++++ scripts/py/gen_default_input_cpp.py | 53 ++++++++ scripts/py/gen_fpga_mem_map.py | 192 +++++++++++++++++++++++++++ scripts/py/gen_labels_cpp.py | 81 +++++++++++ scripts/py/gen_model_cpp.py | 97 ++++++++++++++ scripts/py/gen_rgb_cpp.py | 135 +++++++++++++++++++ scripts/py/gen_test_data_cpp.py | 162 ++++++++++++++++++++++ scripts/py/gen_utils.py | 115 ++++++++++++++++ scripts/py/requirements.txt | 12 ++ scripts/py/templates/AudioClips.cc.template | 62 +++++++++ scripts/py/templates/AudioClips.hpp.template | 34 +++++ scripts/py/templates/Images.cc.template | 47 +++++++ scripts/py/templates/Images.hpp.template | 34 +++++ scripts/py/templates/Labels.cc.template | 54 ++++++++ scripts/py/templates/Labels.hpp.template | 41 ++++++ scripts/py/templates/TestData.cc.template | 51 +++++++ scripts/py/templates/TestData.hpp.template | 47 +++++++ scripts/py/templates/audio.cc.template | 25 ++++ scripts/py/templates/default.hpp.template | 28 ++++ scripts/py/templates/header_template.txt | 21 +++ scripts/py/templates/image.cc.template | 25 ++++ scripts/py/templates/testdata.cc.template | 33 +++++ scripts/py/templates/tflite.cc.template | 49 +++++++ 24 files changed, 1599 insertions(+) create mode 100644 scripts/py/gen_audio.py create mode 100644 scripts/py/gen_audio_cpp.py create mode 100644 scripts/py/gen_default_input_cpp.py create mode 100644 scripts/py/gen_fpga_mem_map.py create mode 100644 scripts/py/gen_labels_cpp.py create mode 100644 scripts/py/gen_model_cpp.py create mode 100644 scripts/py/gen_rgb_cpp.py create mode 100644 scripts/py/gen_test_data_cpp.py create mode 100644 scripts/py/gen_utils.py create mode 100644 scripts/py/requirements.txt create mode 100644 scripts/py/templates/AudioClips.cc.template create mode 100644 scripts/py/templates/AudioClips.hpp.template create mode 100644 scripts/py/templates/Images.cc.template create mode 100644 scripts/py/templates/Images.hpp.template create mode 100644 scripts/py/templates/Labels.cc.template create mode 100644 scripts/py/templates/Labels.hpp.template create mode 100644 scripts/py/templates/TestData.cc.template create mode 100644 scripts/py/templates/TestData.hpp.template create mode 100644 scripts/py/templates/audio.cc.template create mode 100644 scripts/py/templates/default.hpp.template create mode 100644 scripts/py/templates/header_template.txt create mode 100644 scripts/py/templates/image.cc.template create mode 100644 scripts/py/templates/testdata.cc.template create mode 100644 scripts/py/templates/tflite.cc.template (limited to 'scripts/py') diff --git a/scripts/py/gen_audio.py b/scripts/py/gen_audio.py new file mode 100644 index 0000000..53ed019 --- /dev/null +++ b/scripts/py/gen_audio.py @@ -0,0 +1,48 @@ +#!env/bin/python3 + +# Copyright (c) 2021 Arm Limited. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Utility script to convert an audio clip into eval platform desired spec. +""" +import soundfile as sf + +from argparse import ArgumentParser +from os import path + +from gen_utils import AudioUtils + +parser = ArgumentParser() +parser.add_argument("--audio_path", help="Audio file path", required=True) +parser.add_argument("--output_dir", help="Output directory", required=True) +parser.add_argument("--sampling_rate", type=int, help="target sampling rate.", default=16000) +parser.add_argument("--mono", type=bool, help="convert signal to mono.", default=True) +parser.add_argument("--offset", type=float, help="start reading after this time (in seconds).", default=0) +parser.add_argument("--duration", type=float, help="only load up to this much audio (in seconds).", default=0) +parser.add_argument("--res_type", type=AudioUtils.res_data_type, help=f"Resample type: {AudioUtils.res_type_list()}.", default='kaiser_best') +parser.add_argument("--min_samples", type=int, help="Minimum sample number.", default=16000) +parser.add_argument("-v", "--verbosity", action="store_true") +args = parser.parse_args() + +def main(args): + audio_data, samplerate = AudioUtils.load_resample_audio_clip(args.audio_path, + args.sampling_rate, + args.mono, args.offset, + args.duration, args.res_type, + args.min_samples) + sf.write(path.join(args.output_dir, path.basename(args.audio_path)), audio_data, samplerate) + +if __name__ == '__main__': + main(args) diff --git a/scripts/py/gen_audio_cpp.py b/scripts/py/gen_audio_cpp.py new file mode 100644 index 0000000..54fdb23 --- /dev/null +++ b/scripts/py/gen_audio_cpp.py @@ -0,0 +1,153 @@ +# Copyright (c) 2021 Arm Limited. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Utility script to convert a set of audio clip in a given location into +corresponding cpp files and a single hpp file referencing the vectors +from the cpp files. +""" +import datetime +import glob +import math +import os + +import numpy as np +from os import path +from argparse import ArgumentParser +from jinja2 import Environment, FileSystemLoader +from gen_utils import AudioUtils + +parser = ArgumentParser() +parser.add_argument("--audio_path", type=str, help="path to audio folder to convert.") +parser.add_argument("--source_folder_path", type=str, help="path to source folder to be generated.") +parser.add_argument("--header_folder_path", type=str, help="path to header folder to be generated.") +parser.add_argument("--sampling_rate", type=int, help="target sampling rate.", default=16000) +parser.add_argument("--mono", type=bool, help="convert signal to mono.", default=True) +parser.add_argument("--offset", type=float, help="start reading after this time (in seconds).", default=0) +parser.add_argument("--duration", type=float, help="only load up to this much audio (in seconds).", default=0) +parser.add_argument("--res_type", type=AudioUtils.res_data_type, help=f"Resample type: {AudioUtils.res_type_list()}.", + default='kaiser_best') +parser.add_argument("--min_samples", type=int, help="Minimum sample number.", default=16000) +parser.add_argument("--license_template", type=str, help="Header template file", + default="header_template.txt") +parser.add_argument("-v", "--verbosity", action="store_true") +args = parser.parse_args() + +env = Environment(loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), 'templates')), + trim_blocks=True, + lstrip_blocks=True) + + +def write_hpp_file(header_filepath, cc_filepath, header_template_file, num_audios, audio_filenames, audio_array_namesizes): + print(f"++ Generating {header_filepath}") + + header_template = env.get_template(header_template_file) + hdr = header_template.render(script_name=os.path.basename(__file__), + gen_time=datetime.datetime.now(), + year=datetime.datetime.now().year) + env.get_template('AudioClips.hpp.template').stream(common_template_header=hdr, + clips_count=num_audios, + varname_size=audio_array_namesizes + ) \ + .dump(str(header_filepath)) + + print(f"++ Generating {cc_filepath}") + + env.get_template('AudioClips.cc.template').stream(common_template_header=hdr, + clips_count=num_audios, + var_names=(name for name, _ in audio_array_namesizes), + clip_sizes=(size for _, size in audio_array_namesizes), + clip_names=audio_filenames) \ + .dump(str(cc_filepath)) + + +def write_individual_audio_cc_file(clip_dirpath, clip_filename, + cc_filename, header_template_file, array_name, + sampling_rate_value, mono_value, offset_value, + duration_value, res_type_value, min_len): + print(f"++ Converting {clip_filename} to {path.basename(cc_filename)}") + audio_filepath = path.join(clip_dirpath, clip_filename) + clip_data, samplerate = AudioUtils.load_resample_audio_clip(audio_filepath, + sampling_rate_value, mono_value, + offset_value, duration_value, + res_type_value, min_len) + + # Change from [-1, 1] fp32 range to int16 range. + clip_data = np.clip((clip_data * (1 << 15)), + np.iinfo(np.int16).min, + np.iinfo(np.int16).max).flatten().astype(np.int16) + + header_template = env.get_template(header_template_file) + hdr = header_template.render(script_name=os.path.basename(__file__), + gen_time=datetime.datetime.now(), + file_name=clip_filename, + year=datetime.datetime.now().year) + + hex_line_generator = (', '.join(map(hex, sub_arr)) + for sub_arr in np.array_split(clip_data, math.ceil(len(clip_data)/20))) + + env.get_template('audio.cc.template').stream(common_template_header=hdr, + size=len(clip_data), + var_name=array_name, + audio_data=hex_line_generator) \ + .dump(str(cc_filename)) + + return len(clip_data) + + +def main(args): + # Keep the count of the audio files converted + audioclip_idx = 0 + audioclip_filenames = [] + audioclip_array_names = [] + header_filename = "InputFiles.hpp" + common_cc_filename = "InputFiles.cc" + header_filepath = path.join(args.header_folder_path, header_filename) + common_cc_filepath = path.join(args.source_folder_path, common_cc_filename) + + if os.path.isdir(args.audio_path): + filepaths = sorted(glob.glob(path.join(args.audio_path, '**/*.wav'), recursive=True)) + elif os.path.isfile(args.audio_path): + filepaths = [args.audio_path] + else: + raise OSError("Directory or file does not exist.") + + for filepath in filepaths: + filename = path.basename(filepath) + clip_dirpath = path.dirname(filepath) + try: + audioclip_filenames.append(filename) + + # Save the cc file + cc_filename = path.join(args.source_folder_path, + (filename.rsplit(".")[0]).replace(" ", "_") + ".cc") + array_name = "audio" + str(audioclip_idx) + array_size = write_individual_audio_cc_file(clip_dirpath, filename, cc_filename, args.license_template, array_name, + args.sampling_rate, args.mono, args.offset, + args.duration, args.res_type, args.min_samples) + + audioclip_array_names.append((array_name, array_size)) + # Increment audio index + audioclip_idx = audioclip_idx + 1 + except: + if args.verbosity: + print(f"Failed to open {filename} as an audio.") + + write_hpp_file(header_filepath, common_cc_filepath, args.license_template, + audioclip_idx, audioclip_filenames, audioclip_array_names) + + +if __name__ == '__main__': + main(args) diff --git a/scripts/py/gen_default_input_cpp.py b/scripts/py/gen_default_input_cpp.py new file mode 100644 index 0000000..c091fd1 --- /dev/null +++ b/scripts/py/gen_default_input_cpp.py @@ -0,0 +1,53 @@ +# Copyright (c) 2021 Arm Limited. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Utility script to generate the minimum InputFiles.hpp and cpp files required by an application. +""" +import datetime +import os + +from argparse import ArgumentParser +from jinja2 import Environment, FileSystemLoader + +parser = ArgumentParser() +parser.add_argument("--header_folder_path", type=str, help="path to header folder to be generated.") +parser.add_argument("--license_template", type=str, help="Header template file", + default="header_template.txt") +args = parser.parse_args() + +env = Environment(loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), 'templates')), + trim_blocks=True, + lstrip_blocks=True) + + +def write_hpp_file(header_file_path, header_template_file): + print(f"++ Generating {header_file_path}") + header_template = env.get_template(header_template_file) + hdr = header_template.render(script_name=os.path.basename(__file__), + gen_time=datetime.datetime.now(), + year=datetime.datetime.now().year) + env.get_template('default.hpp.template').stream(common_template_header=hdr) \ + .dump(str(header_file_path)) + + +def main(args): + header_filename = "InputFiles.hpp" + header_filepath = os.path.join(args.header_folder_path, header_filename) + write_hpp_file(header_filepath, args.license_template) + + +if __name__ == '__main__': + main(args) diff --git a/scripts/py/gen_fpga_mem_map.py b/scripts/py/gen_fpga_mem_map.py new file mode 100644 index 0000000..6a2d1d2 --- /dev/null +++ b/scripts/py/gen_fpga_mem_map.py @@ -0,0 +1,192 @@ +# Copyright (c) 2021 Arm Limited. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from argparse import ArgumentParser + +""" +This file is used as part of post build steps to generate 'images.txt' file +which can be copied over onto the MPS3 board's SD card. The purpose is to +limit having to manually edit the file based on different load regions that +the build scatter file might dictate. +""" + +def is_commented(line): + if (line.startswith(";")): + return True + else: + return False + + +def is_load_rom(line): + load_region_specifiers = ['LOAD_ROM', 'LD_ROM', 'LOAD_REGION'] + + for load_specifier in load_region_specifiers: + if line.startswith(load_specifier): + return True + + return False + + +class TargetSubsystem: + + def __init__(self, target_subsystem_name: str): + """ + Constructor for target class. + Arguments: + target_subsystem_name: name of the target subsystem + """ + # Dict with mem map and binary names we expect + self.subsystems = { + "sse-200": { + "mmap_mcc" : { + # FPGA addr | MCC addr | + "0x00000000": "0x00000000", # ITCM (NS) + "0x10000000": "0x01000000", # ITCM (S) + "0x20000000": "0x02000000", # DTCM (NS) + "0x30000000": "0x03000000", # DTCM (S) + "0x60000000": "0x08000000" # DDR (NS) + }, + "bin_names": { + 0: "itcm.bin", + 1: "dram.bin" + } + }, + "sse-300": { + "mmap_mcc" : { + # FPGA addr | MCC addr | + "0x00000000": "0x00000000", # ITCM (NS) + "0x01000000": "0x02000000", # BRAM or FPGA's data SRAM (NS) + "0x60000000": "0x08000000", # DDR (NS) + "0x70000000": "0x0c000000" # DDR (S) + }, + "bin_names": { + 0: "itcm.bin", + 1: "dram.bin" + } + } + } + + self.name = target_subsystem_name + + + def is_supported(self, target_subsystem: str) -> bool: + """ + Checks if the target subsystem exists within systems + supported by this script + """ + if target_subsystem in self.subsystems.keys(): + return True + + print(f"Platforms supported: {self.subsystems.keys()}") + return False + + + def mps3_mappings(self) -> dict: + """ + Returns the FPGA <--> MCC address translations + as a dict + """ + if self.is_supported(self.name): + return self.subsystems[self.name]['mmap_mcc'] + return {} + + + def mps3_bin_names(self) -> dict: + """ + Returns expected binary names for the executable built + for Cortex-M55 or Cortex-M55+Ethos-U55 targets in the + form of a dict with index and name + """ + if self.is_supported(self.name): + return self.subsystems[self.name]['bin_names'] + + return {} + + +def main(args): + """ + Generates the output txt file with MCC to FPGA address mapping used + that is used by the MCC on FPGA to load executable regions into + correct regions in memory. + """ + # List out arguments used: + scatter_file_path = args.scatter_file_path + target_subsystem_name = args.target_subsystem + output_file_path = args.output_file_path + + target = TargetSubsystem(target_subsystem_name=target_subsystem_name) + + if target.is_supported(target_subsystem_name) != True: + print(f'Target {target_subsystem_name} not supported.') + return + + with open(scatter_file_path,'r') as scatter_file: + lines_read = scatter_file.readlines() + str_list = [] + + bin_names = None + mem_map = None + + mem_map = target.mps3_mappings() + bin_names = target.mps3_bin_names() + + str_list.append("TITLE: Arm MPS3 FPGA prototyping board Images Configuration File\n") + str_list.append("[IMAGES]\n\n") + + cnt = 0 + for line in lines_read: + if is_commented(line) or is_load_rom(line) != True: + continue + + addr = line.split()[1] + + if mem_map.get(addr, None) == None: + raise RuntimeError( + 'Translation for this address unavailable') + if cnt > len(bin_names): + raise RuntimeError( + f"bin names len exceeded: {cnt}") + + str_list.append("IMAGE" + str(cnt) + "ADDRESS: " + + mem_map[addr] + " ; MCC@" + mem_map[addr] + + " <=> FPGA@" + addr + "\n") + str_list.append("IMAGE" + str(cnt) + "UPDATE: AUTO\n") + str_list.append("IMAGE" + str(cnt) + "FILE: \SOFTWARE\\" + + bin_names[cnt] + "\n\n") + cnt += 1 + + if cnt > 0 and cnt < 33: + str_list.insert(2, + "TOTALIMAGES: {} ;Number of Images (Max: 32)\n\n".format( + cnt)) + else: + raise RuntimeError('Invalid image count') + + if os.path.exists(output_file_path): + os.remove(output_file_path) + print(''.join(str_list), file=open(output_file_path, "a")) + + +if __name__ == "__main__": + parser = ArgumentParser() + parser.add_argument("--scatter_file_path", type=str, required=True, + help="Path to the scatter file") + parser.add_argument("--target_subsystem", type=str, required=True, + help="Target subsystem in use") + parser.add_argument("--output_file_path", type=str, required=True, + help="Output file path") + args = parser.parse_args() + main(args) diff --git a/scripts/py/gen_labels_cpp.py b/scripts/py/gen_labels_cpp.py new file mode 100644 index 0000000..1be9c63 --- /dev/null +++ b/scripts/py/gen_labels_cpp.py @@ -0,0 +1,81 @@ +#!env/bin/python3 + +# Copyright (c) 2021 Arm Limited. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Utility script to convert a given text file with labels (annotations for an +NN model output vector) into a vector list initialiser. The intention is for +this script to be called as part of the build framework to auto-generate the +cpp file with labels that can be used in the application without modification. +""" +import datetime +import os +from argparse import ArgumentParser +from jinja2 import Environment, FileSystemLoader + +parser = ArgumentParser() + +# Label file path +parser.add_argument("--labels_file", type=str, help="Path to the label text file", required=True) +# Output file to be generated +parser.add_argument("--source_folder_path", type=str, help="path to source folder to be generated.", required=True) +parser.add_argument("--header_folder_path", type=str, help="path to header folder to be generated.", required=True) +parser.add_argument("--output_file_name", type=str, help="Required output file name", required=True) +# Namespaces +parser.add_argument("--namespaces", action='append', default=[]) +# License template +parser.add_argument("--license_template", type=str, help="Header template file", + default="header_template.txt") + +args = parser.parse_args() + +env = Environment(loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), 'templates')), + trim_blocks=True, + lstrip_blocks=True) + + +def main(args): + # Get the labels from text file + with open(args.labels_file, "r") as f: + labels = f.read().splitlines() + + # No labels? + if len(labels) == 0: + raise Exception(f"no labels found in {args.label_file}") + + header_template = env.get_template(args.license_template) + hdr = header_template.render(script_name=os.path.basename(__file__), + gen_time=datetime.datetime.now(), + file_name=os.path.basename(args.labels_file), + year=datetime.datetime.now().year) + + hpp_filename = os.path.join(args.header_folder_path, args.output_file_name + ".hpp") + env.get_template('Labels.hpp.template').stream(common_template_header=hdr, + filename=(args.output_file_name).upper(), + namespaces=args.namespaces) \ + .dump(str(hpp_filename)) + + + cc_filename = os.path.join(args.source_folder_path, args.output_file_name + ".cc") + env.get_template('Labels.cc.template').stream(common_template_header=hdr, + labels=labels, + labelsSize=len(labels), + namespaces=args.namespaces) \ + .dump(str(cc_filename)) + + +if __name__ == '__main__': + main(args) diff --git a/scripts/py/gen_model_cpp.py b/scripts/py/gen_model_cpp.py new file mode 100644 index 0000000..4843668 --- /dev/null +++ b/scripts/py/gen_model_cpp.py @@ -0,0 +1,97 @@ +# Copyright (c) 2021 Arm Limited. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Utility script to generate model c file that can be included in the +project directly. This should be called as part of cmake framework +should the models need to be generated at configuration stage. +""" +import datetime +import os +from argparse import ArgumentParser +from pathlib import Path +from jinja2 import Environment, FileSystemLoader + +parser = ArgumentParser() + +parser.add_argument("--tflite_path", help="Model (.tflite) path", required=True) +parser.add_argument("--output_dir", help="Output directory", required=True) +parser.add_argument('-e', '--expression', action='append', default=[], dest="expr") +parser.add_argument('--header', action='append', default=[], dest="headers") +parser.add_argument('-ns', '--namespaces', action='append', default=[], dest="namespaces") +parser.add_argument("--license_template", type=str, help="Header template file", + default="header_template.txt") +args = parser.parse_args() + +env = Environment(loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), 'templates')), + trim_blocks=True, + lstrip_blocks=True) + + +def write_tflite_data(tflite_path): + # Extract array elements + + bytes = model_hex_bytes(tflite_path) + line = '{\n' + i = 1 + while True: + try: + el = next(bytes) + line = line + el + ', ' + if i % 20 == 0: + yield line + line = '' + i += 1 + except StopIteration: + line = line[:-2] + '};\n' + yield line + break + + +def model_hex_bytes(tflite_path): + with open(tflite_path, 'rb') as tflite_model: + byte = tflite_model.read(1) + while byte != b"": + yield f'0x{byte.hex()}' + byte = tflite_model.read(1) + + +def main(args): + if not os.path.isfile(args.tflite_path): + raise Exception(f"{args.tflite_path} not found") + + # Cpp filename: + cpp_filename = Path(os.path.join(args.output_dir, os.path.basename(args.tflite_path) + ".cc")).absolute() + print(f"++ Converting {os.path.basename(args.tflite_path)} to\ + {os.path.basename(cpp_filename)}") + + os.makedirs(cpp_filename.parent, exist_ok=True) + + header_template = env.get_template(args.license_template) + + hdr = header_template.render(script_name=os.path.basename(__file__), + file_name=os.path.basename(args.tflite_path), + gen_time=datetime.datetime.now(), + year=datetime.datetime.now().year) + + env.get_template('tflite.cc.template').stream(common_template_header=hdr, + model_data=write_tflite_data(args.tflite_path), + expressions=args.expr, + additional_headers=args.headers, + namespaces=args.namespaces).dump(str(cpp_filename)) + + +if __name__ == '__main__': + main(args) diff --git a/scripts/py/gen_rgb_cpp.py b/scripts/py/gen_rgb_cpp.py new file mode 100644 index 0000000..1a2e09b --- /dev/null +++ b/scripts/py/gen_rgb_cpp.py @@ -0,0 +1,135 @@ +# Copyright (c) 2021 Arm Limited. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Utility script to convert a set of RGB images in a given location into +corresponding cpp files and a single hpp file referencing the vectors +from the cpp files. +""" +import datetime +import glob +import math +import os +import numpy as np + +from argparse import ArgumentParser +from PIL import Image, UnidentifiedImageError +from jinja2 import Environment, FileSystemLoader + +parser = ArgumentParser() +parser.add_argument("--image_path", type=str, help="path to images folder or image file to convert.") +parser.add_argument("--source_folder_path", type=str, help="path to source folder to be generated.") +parser.add_argument("--header_folder_path", type=str, help="path to header folder to be generated.") +parser.add_argument("--image_size", type=int, nargs=2, help="Size (width and height) of the converted images.") +parser.add_argument("--license_template", type=str, help="Header template file", + default="header_template.txt") +args = parser.parse_args() + +env = Environment(loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), 'templates')), + trim_blocks=True, + lstrip_blocks=True) + + +def write_hpp_file(header_file_path, cc_file_path, header_template_file, num_images, image_filenames, + image_array_names, image_size): + print(f"++ Generating {header_file_path}") + header_template = env.get_template(header_template_file) + hdr = header_template.render(script_name=os.path.basename(__file__), + gen_time=datetime.datetime.now(), + year=datetime.datetime.now().year) + env.get_template('Images.hpp.template').stream(common_template_header=hdr, + imgs_count=num_images, + img_size=str(image_size[0] * image_size[1] * 3), + var_names=image_array_names) \ + .dump(str(header_file_path)) + + env.get_template('Images.cc.template').stream(common_template_header=hdr, + var_names=image_array_names, + img_names=image_filenames) \ + .dump(str(cc_file_path)) + + +def write_individual_img_cc_file(image_filename, cc_filename, header_template_file, original_image, + image_size, array_name): + print(f"++ Converting {image_filename} to {os.path.basename(cc_filename)}") + + header_template = env.get_template(header_template_file) + hdr = header_template.render(script_name=os.path.basename(__file__), + gen_time=datetime.datetime.now(), + file_name=os.path.basename(image_filename), + year=datetime.datetime.now().year) + + original_image.thumbnail(image_size) + delta_w = abs(image_size[0] - original_image.size[0]) + delta_h = abs(image_size[1] - original_image.size[1]) + resized_image = Image.new('RGB', args.image_size, (255, 255, 255, 0)) + resized_image.paste(original_image, (int(delta_w / 2), int(delta_h / 2))) + + # Convert the image and write it to the cc file + rgb_data = np.array(resized_image, dtype=np.uint8).flatten() + hex_line_generator = (', '.join(map(hex, sub_arr)) + for sub_arr in np.array_split(rgb_data, math.ceil(len(rgb_data) / 20))) + env.get_template('image.cc.template').stream(common_template_header=hdr, + var_name=array_name, + img_data=hex_line_generator) \ + .dump(str(cc_filename)) + + +def main(args): + # Keep the count of the images converted + image_idx = 0 + image_filenames = [] + image_array_names = [] + + + if os.path.isdir(args.image_path): + filepaths = sorted(glob.glob(os.path.join(args.image_path, '**/*.*'), recursive=True)) + elif os.path.isfile(args.image_path): + filepaths = [args.image_path] + else: + raise OSError("Directory or file does not exist.") + + for filepath in filepaths: + filename = os.path.basename(filepath) + + try: + original_image = Image.open(filepath).convert("RGB") + except UnidentifiedImageError: + print(f"-- Skipping file {filepath} due to unsupported image format.") + continue + + image_filenames.append(filename) + + # Save the cc file + cc_filename = os.path.join(args.source_folder_path, + (filename.rsplit(".")[0]).replace(" ", "_") + ".cc") + array_name = "im" + str(image_idx) + image_array_names.append(array_name) + write_individual_img_cc_file(filename, cc_filename, args.license_template, + original_image, args.image_size, array_name) + + # Increment image index + image_idx = image_idx + 1 + + header_filename = "InputFiles.hpp" + header_filepath = os.path.join(args.header_folder_path, header_filename) + common_cc_filename = "InputFiles.cc" + common_cc_filepath = os.path.join(args.source_folder_path, common_cc_filename) + write_hpp_file(header_filepath, common_cc_filepath, args.license_template, + image_idx, image_filenames, image_array_names, args.image_size) + + +if __name__ == '__main__': + main(args) diff --git a/scripts/py/gen_test_data_cpp.py b/scripts/py/gen_test_data_cpp.py new file mode 100644 index 0000000..7cc5f11 --- /dev/null +++ b/scripts/py/gen_test_data_cpp.py @@ -0,0 +1,162 @@ +# Copyright (c) 2021 Arm Limited. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Utility script to convert a set of pairs of npy files in a given location into +corresponding cpp files and a single hpp file referencing the vectors +from the cpp files. +""" +import datetime +import math +import os +import numpy as np + +from argparse import ArgumentParser +from jinja2 import Environment, FileSystemLoader + +parser = ArgumentParser() +parser.add_argument("--data_folder_path", type=str, help="path to ifm-ofm npy folder to convert.") +parser.add_argument("--source_folder_path", type=str, help="path to source folder to be generated.") +parser.add_argument("--header_folder_path", type=str, help="path to header folder to be generated.") +parser.add_argument("--usecase", type=str, default="", help="Test data file suffix.") +parser.add_argument("--namespaces", action='append', default=[]) +parser.add_argument("--license_template", type=str, help="Header template file", + default="header_template.txt") +parser.add_argument("-v", "--verbosity", action="store_true") + +args = parser.parse_args() + +env = Environment(loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), 'templates')), + trim_blocks=True, + lstrip_blocks=True) + + +def write_hpp_file(header_filename, cc_file_path, header_template_file, num_iofms, + ifm_array_names, ifm_size, ofm_array_names, ofm_size, iofm_data_type): + header_file_path = os.path.join(args.header_folder_path, header_filename) + + print(f"++ Generating {header_file_path}") + header_template = env.get_template(header_template_file) + hdr = header_template.render(script_name=os.path.basename(__file__), + gen_time=datetime.datetime.now(), + year=datetime.datetime.now().year) + env.get_template('TestData.hpp.template').stream(common_template_header=hdr, + fm_count=num_iofms, + ifm_var_names=ifm_array_names, + ifm_var_size=ifm_size, + ofm_var_names=ofm_array_names, + ofm_var_size=ofm_size, + data_type=iofm_data_type, + namespaces=args.namespaces) \ + .dump(str(header_file_path)) + + env.get_template('TestData.cc.template').stream(common_template_header=hdr, + include_h=header_filename, + ifm_var_names=ifm_array_names, + ofm_var_names=ofm_array_names, + data_type=iofm_data_type, + namespaces=args.namespaces) \ + .dump(str(cc_file_path)) + + +def write_individual_cc_file(filename, cc_filename, header_filename, header_template_file, array_name, iofm_data_type): + print(f"++ Converting {filename} to {os.path.basename(cc_filename)}") + header_template = env.get_template(header_template_file) + hdr = header_template.render(script_name=os.path.basename(__file__), + gen_time=datetime.datetime.now(), + file_name=os.path.basename(filename), + year=datetime.datetime.now().year) + + # Convert the image and write it to the cc file + fm_data = (np.load(os.path.join(args.data_folder_path, filename))).flatten() + type(fm_data.dtype) + hex_line_generator = (', '.join(map(hex, sub_arr)) + for sub_arr in np.array_split(fm_data, math.ceil(len(fm_data) / 20))) + + env.get_template('testdata.cc.template').stream(common_template_header=hdr, + include_h=header_filename, + var_name=array_name, + fm_data=hex_line_generator, + data_type=iofm_data_type, + namespaces=args.namespaces) \ + .dump(str(cc_filename)) + + +def get_npy_vec_size(filename: str) -> int: + """ + Gets the size of the array in the npy file + Args: + filename: npy file path. + Return: + size in bytes + """ + data = np.load(os.path.join(args.data_folder_path, filename)) + return (data.size * data.dtype.itemsize) + + +def main(args): + # Keep the count of the images converted + ifm_array_names = [] + ofm_array_names = [] + + add_usecase_fname = ("_" + args.usecase) if (args.usecase is not "") else "" + header_filename = "TestData" + add_usecase_fname + ".hpp" + common_cc_filename = "TestData" + add_usecase_fname + ".cc" + + # In the data_folder_path there should be pairs of ifm-ofm + # It's assumed the ifm-ofm nameing convention: ifm0.npy-ofm0.npy, ifm1.npy-ofm1.npy + i_ofms_count = int(len([name for name in os.listdir(os.path.join(args.data_folder_path)) if name.lower().endswith('.npy')]) / 2) + + iofm_data_type = "int8_t" + if (i_ofms_count > 0): + iofm_data_type = "int8_t" if (np.load(os.path.join(args.data_folder_path, "ifm0.npy")).dtype == np.int8) else "uint8_t" + + ifm_size = -1 + ofm_size = -1 + + for idx in range(i_ofms_count): + # Save the fm cc file + base_name = "ifm" + str(idx) + filename = base_name+".npy" + array_name = base_name + add_usecase_fname + cc_filename = os.path.join(args.source_folder_path, array_name + ".cc") + ifm_array_names.append(array_name) + write_individual_cc_file(filename, cc_filename, header_filename, args.license_template, array_name, iofm_data_type) + if ifm_size == -1: + ifm_size = get_npy_vec_size(filename) + elif ifm_size != get_npy_vec_size(filename): + raise Exeception(f"ifm size changed for index {idx}") + + # Save the fm cc file + base_name = "ofm" + str(idx) + filename = base_name+".npy" + array_name = base_name + add_usecase_fname + cc_filename = os.path.join(args.source_folder_path, array_name + ".cc") + ofm_array_names.append(array_name) + write_individual_cc_file(filename, cc_filename, header_filename, args.license_template, array_name, iofm_data_type) + if ofm_size == -1: + ofm_size = get_npy_vec_size(filename) + elif ofm_size != get_npy_vec_size(filename): + raise Exeception(f"ofm size changed for index {idx}") + + common_cc_filepath = os.path.join(args.source_folder_path, common_cc_filename) + write_hpp_file(header_filename, common_cc_filepath, args.license_template, + i_ofms_count, ifm_array_names, ifm_size, ofm_array_names, ofm_size, iofm_data_type) + + +if __name__ == '__main__': + if args.verbosity: + print("Running gen_test_data_cpp with args: "+str(args)) + main(args) diff --git a/scripts/py/gen_utils.py b/scripts/py/gen_utils.py new file mode 100644 index 0000000..4a56646 --- /dev/null +++ b/scripts/py/gen_utils.py @@ -0,0 +1,115 @@ +#!env/bin/python3 + +# Copyright (c) 2021 Arm Limited. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import soundfile as sf +import resampy +import numpy as np + + +class AudioUtils: + @staticmethod + def res_data_type(res_type_value): + """ + Returns the input string if is one of the valid resample type + """ + import argparse + if res_type_value not in AudioUtils.res_type_list(): + raise argparse.ArgumentTypeError(f"{res_type_value} not valid. Supported only {AudioUtils.res_type_list()}") + return res_type_value + + @staticmethod + def res_type_list(): + """ + Returns the resample type list + """ + return ['kaiser_best', 'kaiser_fast'] + + @staticmethod + def load_resample_audio_clip(path, target_sr=16000, mono=True, offset=0.0, duration=0, res_type='kaiser_best', + min_len=16000): + """ + Load and resample an audio clip with the given desired specs. + + Parameters: + ---------- + path (string): Path to the input audio clip. + target_sr (int, optional): Target sampling rate. Positive number are considered valid, + if zero or negative the native sampling rate of the file will be preserved. Default is 16000. + mono (bool, optional): Specify if the audio file needs to be converted to mono. Default is True. + offset (float, optional): Target sampling rate. Default is 0.0. + duration (int, optional): Target duration. Positive number are considered valid, + if zero or negative the duration of the file will be preserved. Default is 0. + res_type (int, optional): Resample type to use, Default is 'kaiser_best'. + min_len (int, optional): Minimun lenght of the output audio time series. Default is 16000. + + Returns: + ---------- + y (np.ndarray): Output audio time series of shape shape=(n,) or (2, n). + sr (int): A scalar number > 0 that represent the sampling rate of `y` + """ + try: + with sf.SoundFile(path) as audio_file: + origin_sr = audio_file.samplerate + + if offset: + # Seek to the start of the target read + audio_file.seek(int(offset * origin_sr)) + + if duration > 0: + num_frame_duration = int(duration * origin_sr) + else: + num_frame_duration = -1 + + # Load the target number of frames + y = audio_file.read(frames=num_frame_duration, dtype=np.float32, always_2d=False).T + + except: + print(f"Failed to open {path} as an audio.") + + # Convert to mono if requested and if audio has more than one dimension + if mono and (y.ndim > 1): + y = np.mean(y, axis=0) + + if not (origin_sr == target_sr) and (target_sr > 0): + ratio = float(target_sr) / origin_sr + axis = -1 + n_samples = int(np.ceil(y.shape[axis] * ratio)) + + # Resample using resampy + y_rs = resampy.resample(y, origin_sr, target_sr, filter=res_type, axis=axis) + n_rs_samples = y_rs.shape[axis] + + # Adjust the size + if n_rs_samples > n_samples: + slices = [slice(None)] * y_rs.ndim + slices[axis] = slice(0, n_samples) + y = y_rs[tuple(slices)] + elif n_rs_samples < n_samples: + lengths = [(0, 0)] * y_rs.ndim + lengths[axis] = (0, n_samples - n_rs_samples) + y = np.pad(y_rs, lengths, 'constant', constant_values=(0)) + + sr = target_sr + else: + sr = origin_sr + + # Pad if necessary and min lenght is setted (min_len> 0) + if (y.shape[0] < min_len) and (min_len > 0): + sample_to_pad = min_len - y.shape[0] + y = np.pad(y, (0, sample_to_pad), 'constant', constant_values=(0)) + + return y, sr diff --git a/scripts/py/requirements.txt b/scripts/py/requirements.txt new file mode 100644 index 0000000..6330f58 --- /dev/null +++ b/scripts/py/requirements.txt @@ -0,0 +1,12 @@ +cffi==1.14.2 +Jinja2==2.11.2 +llvmlite==0.33.0 +MarkupSafe==1.1.1 +numba==0.50.1 +numpy==1.17.4 +Pillow==7.0.0 +pycparser==2.20 +resampy==0.2.2 +scipy==1.5.2 +six==1.15.0 +SoundFile==0.10.3.post1 diff --git a/scripts/py/templates/AudioClips.cc.template b/scripts/py/templates/AudioClips.cc.template new file mode 100644 index 0000000..edf46bc --- /dev/null +++ b/scripts/py/templates/AudioClips.cc.template @@ -0,0 +1,62 @@ +{# + Copyright (c) 2021 Arm Limited. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +#} +{{common_template_header}} + +#include "InputFiles.hpp" + +static const char *audio_clip_filenames[] = { +{% for name in clip_names %} + "{{name}}", +{% endfor %} +}; + +static const int16_t *audio_clip_arrays[] = { + {{ var_names|join(',\n\t') }} +}; + + +static const size_t audio_clip_sizes[NUMBER_OF_FILES] = { + {{ clip_sizes|join(',\n\t') }} +}; + + +const char* get_filename(const uint32_t idx) +{ + if (idx < NUMBER_OF_FILES) { + return audio_clip_filenames[idx]; + } + return nullptr; +} + + +const int16_t* get_audio_array(const uint32_t idx) +{ + if (idx < NUMBER_OF_FILES) { + return audio_clip_arrays[idx]; + } + return nullptr; +} + + +uint32_t get_audio_array_size(const uint32_t idx) +{ + if (idx < NUMBER_OF_FILES) { + return audio_clip_sizes[idx]; + } + return 0; +} + diff --git a/scripts/py/templates/AudioClips.hpp.template b/scripts/py/templates/AudioClips.hpp.template new file mode 100644 index 0000000..eb0beda --- /dev/null +++ b/scripts/py/templates/AudioClips.hpp.template @@ -0,0 +1,34 @@ +{# + Copyright (c) 2021 Arm Limited. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +#} +{{common_template_header}} + +#ifndef GENERATED_AUDIOCLIPS_H +#define GENERATED_AUDIOCLIPS_H + +#include +#include + +#define NUMBER_OF_FILES ({{clips_count}}U) +{% for var_name, size in varname_size %} +extern const int16_t {{var_name}}[{{size}}]; +{% endfor %} + +const char* get_filename(const uint32_t idx); +const int16_t* get_audio_array(const uint32_t idx); +uint32_t get_audio_array_size(const uint32_t idx); + +#endif /* GENERATED_AUDIOCLIPS_H */ diff --git a/scripts/py/templates/Images.cc.template b/scripts/py/templates/Images.cc.template new file mode 100644 index 0000000..6e86f98 --- /dev/null +++ b/scripts/py/templates/Images.cc.template @@ -0,0 +1,47 @@ +{# + Copyright (c) 2021 Arm Limited. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +#} +{{common_template_header}} + +#include "InputFiles.hpp" + +static const char *img_filenames[] = { +{% for name in img_names %} + "{{name}}", +{% endfor %} +}; + +static const uint8_t *img_arrays[] = { + {{ var_names|join(',\n\t') }} +}; + +const char* get_filename(const uint32_t idx) +{ + if (idx < NUMBER_OF_FILES) { + return img_filenames[idx]; + } + return nullptr; +} + + +const uint8_t* get_img_array(const uint32_t idx) +{ + if (idx < NUMBER_OF_FILES) { + return img_arrays[idx]; + } + return nullptr; +} + diff --git a/scripts/py/templates/Images.hpp.template b/scripts/py/templates/Images.hpp.template new file mode 100644 index 0000000..89ce39e --- /dev/null +++ b/scripts/py/templates/Images.hpp.template @@ -0,0 +1,34 @@ +{# + Copyright (c) 2021 Arm Limited. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +#} +{{common_template_header}} + +#ifndef GENERATED_IMAGES_H +#define GENERATED_IMAGES_H + +#include + +#define NUMBER_OF_FILES ({{imgs_count}}U) +#define IMAGE_DATA_SIZE ({{img_size}}U) + +{% for var_name in var_names %} +extern const uint8_t {{var_name}}[IMAGE_DATA_SIZE]; +{% endfor %} + +const char* get_filename(const uint32_t idx); +const uint8_t* get_img_array(const uint32_t idx); + +#endif /* GENERATED_IMAGES_H */ diff --git a/scripts/py/templates/Labels.cc.template b/scripts/py/templates/Labels.cc.template new file mode 100644 index 0000000..f1ec1b5 --- /dev/null +++ b/scripts/py/templates/Labels.cc.template @@ -0,0 +1,54 @@ +{# + Copyright (c) 2021 Arm Limited. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +#} +{{common_template_header}} + +#include "BufAttributes.hpp" + +#include +#include + +{% for namespace in namespaces %} +namespace {{namespace}} { +{% endfor %} + +static const char * labelsVec[] LABELS_ATTRIBUTE = { +{% for label in labels %} + "{{label}}", +{% endfor %} +}; + +bool GetLabelsVector(std::vector& labels) +{ + constexpr size_t labelsSz = {{labelsSize}}; + labels.clear(); + + if (!labelsSz) { + return false; + } + + labels.reserve(labelsSz); + + for (size_t i = 0; i < labelsSz; ++i) { + labels.emplace_back(labelsVec[i]); + } + + return true; +} + +{% for namespace in namespaces %} +} /* namespace {{name_space}} */ +{% endfor %} diff --git a/scripts/py/templates/Labels.hpp.template b/scripts/py/templates/Labels.hpp.template new file mode 100644 index 0000000..c16a983 --- /dev/null +++ b/scripts/py/templates/Labels.hpp.template @@ -0,0 +1,41 @@ +{# + Copyright (c) 2021 Arm Limited. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +#} +{{common_template_header}} + +#ifndef {{filename}}_HPP +#define {{filename}}_HPP + +#include +#include + +{% for namespace in namespaces %} +namespace {{namespace}} { +{% endfor %} + +/** + * @brief Gets the label vector corresponding to the model + * @param[out] labels Vector of strings. + * @return true if successful, false otherwise. + */ +extern bool GetLabelsVector(std::vector& labels); + + +{% for namespace in namespaces %} +} /* namespace {{namespace}} */ +{% endfor %} + +#endif /* {{filename}}_HPP */ diff --git a/scripts/py/templates/TestData.cc.template b/scripts/py/templates/TestData.cc.template new file mode 100644 index 0000000..1acd14d --- /dev/null +++ b/scripts/py/templates/TestData.cc.template @@ -0,0 +1,51 @@ +{# + Copyright (c) 2021 Arm Limited. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +#} +{{common_template_header}} + +#include "{{include_h}}" + +{% for namespace in namespaces %} +namespace {{namespace}} { +{% endfor %} + +static const {{data_type}} *ifm_arrays[] = { + {{ ifm_var_names|join(',\n\t') }} +}; + +static const {{data_type}} *ofm_arrays[] = { + {{ ofm_var_names|join(',\n\t') }} +}; + +const {{data_type}}* get_ifm_data_array(const uint32_t idx) +{ + if (idx < NUMBER_OF_FM_FILES) { + return ifm_arrays[idx]; + } + return nullptr; +} + +const {{data_type}}* get_ofm_data_array(const uint32_t idx) +{ + if (idx < NUMBER_OF_FM_FILES) { + return ofm_arrays[idx]; + } + return nullptr; +} + +{% for namespace in namespaces %} +} /* namespace {{namespace}} */ +{% endfor %} diff --git a/scripts/py/templates/TestData.hpp.template b/scripts/py/templates/TestData.hpp.template new file mode 100644 index 0000000..cdedd48 --- /dev/null +++ b/scripts/py/templates/TestData.hpp.template @@ -0,0 +1,47 @@ +{# + Copyright (c) 2021 Arm Limited. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +#} +{{common_template_header}} + +#ifndef GENERATED_TEST_DATA_H +#define GENERATED_TEST_DATA_H + +#include + +{% for namespace in namespaces %} +namespace {{namespace}} { +{% endfor %} + +#define NUMBER_OF_FM_FILES ({{fm_count}}U) +#define IFM_DATA_SIZE ({{ifm_var_size}}U) +#define OFM_DATA_SIZE ({{ofm_var_size}}U) + +{% for ifm_var_name in ifm_var_names %} +extern const {{data_type}} {{ifm_var_name}}[IFM_DATA_SIZE]; +{% endfor %} + +{% for ofm_var_name in ofm_var_names %} +extern const {{data_type}} {{ofm_var_name}}[OFM_DATA_SIZE]; +{% endfor %} + +const {{data_type}}* get_ifm_data_array(const uint32_t idx); +const {{data_type}}* get_ofm_data_array(const uint32_t idx); + +{% for namespace in namespaces %} +} /* namespace {{namespace}} */ +{% endfor %} + +#endif /* GENERATED_TEST_DATA_H */ diff --git a/scripts/py/templates/audio.cc.template b/scripts/py/templates/audio.cc.template new file mode 100644 index 0000000..f1e29ef --- /dev/null +++ b/scripts/py/templates/audio.cc.template @@ -0,0 +1,25 @@ +{# + Copyright (c) 2021 Arm Limited. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +#} +{{common_template_header}} + +#include "InputFiles.hpp" +#include "BufAttributes.hpp" +#include + +const int16_t {{var_name}} [{{size}}] IFM_BUF_ATTRIBUTE = { + {{audio_data|join(',\n\t')}} +}; \ No newline at end of file diff --git a/scripts/py/templates/default.hpp.template b/scripts/py/templates/default.hpp.template new file mode 100644 index 0000000..acba891 --- /dev/null +++ b/scripts/py/templates/default.hpp.template @@ -0,0 +1,28 @@ +{# + Copyright (c) 2021 Arm Limited. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +#} +{{common_template_header}} + +#ifndef DEFAULT_GENERATED_INPUT_H +#define DEFAULT_GENERATED_INPUT_H + +#include + +#define NUMBER_OF_FILES (0U) + +const char* get_filename(const uint32_t idx); + +#endif /* DEFAULT_GENERATED_INPUT_H */ diff --git a/scripts/py/templates/header_template.txt b/scripts/py/templates/header_template.txt new file mode 100644 index 0000000..0dac4be --- /dev/null +++ b/scripts/py/templates/header_template.txt @@ -0,0 +1,21 @@ +/* + * Copyright (c) {{year}}, Arm Limited and affiliates. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/********************* Autogenerated file. DO NOT EDIT ******************* + * Generated from {{script_name}} tool {% if file_name %}and {{file_name}}{% endif %} file. + * Date: {{gen_time}} + ***************************************************************************/ diff --git a/scripts/py/templates/image.cc.template b/scripts/py/templates/image.cc.template new file mode 100644 index 0000000..010daa1 --- /dev/null +++ b/scripts/py/templates/image.cc.template @@ -0,0 +1,25 @@ +{# + Copyright (c) 2021 Arm Limited. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +#} +{{common_template_header}} + +#include "InputFiles.hpp" +#include "BufAttributes.hpp" +#include + +const uint8_t {{var_name}}[] IFM_BUF_ATTRIBUTE = { + {{img_data|join(',\n\t')}} +}; diff --git a/scripts/py/templates/testdata.cc.template b/scripts/py/templates/testdata.cc.template new file mode 100644 index 0000000..e3c1dc6 --- /dev/null +++ b/scripts/py/templates/testdata.cc.template @@ -0,0 +1,33 @@ +{# + Copyright (c) 2021 Arm Limited. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +#} +{{common_template_header}} + +#include "{{include_h}}" +#include "BufAttributes.hpp" +#include + +{% for namespace in namespaces %} +namespace {{namespace}} { +{% endfor %} + +const {{data_type}} {{var_name}} [{{size}}] IFM_BUF_ATTRIBUTE = { + {{fm_data|join(',\n\t')}} +}; + +{% for namespace in namespaces %} +} /* namespace {{namespace}} */ +{% endfor %} diff --git a/scripts/py/templates/tflite.cc.template b/scripts/py/templates/tflite.cc.template new file mode 100644 index 0000000..97bdec5 --- /dev/null +++ b/scripts/py/templates/tflite.cc.template @@ -0,0 +1,49 @@ +{# + Copyright (c) 2021 Arm Limited. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +#} +{{common_template_header}} + +#include "Model.hpp" +{% for header in additional_headers %} +#include "{{header}}" +{% endfor %} + +{% for namespace in namespaces %} +namespace {{namespace}} { +{% endfor %} + +{% for expression in expressions %} +{{expression}}; +{% endfor %} + +static const uint8_t nn_model[] MODEL_TFLITE_ATTRIBUTE = +{% for model_hex_line in model_data %} +{{model_hex_line}} +{% endfor %} + +const uint8_t * GetModelPointer() +{ + return nn_model; +} + +size_t GetModelLen() +{ + return sizeof(nn_model); +} + +{% for namespace in namespaces %} +} /* namespace {{namespace}} */ +{% endfor %} -- cgit v1.2.1