MLECO-3036: Update to use Pathlib in Python scripts

* Pathlib used in Python scripts over os
* Bug fix for build_default.py
* Minor code style updates
 
Signed-off-by: Richard Burton <richard.burton@arm.com>
Change-Id: I5fc2e582a84443c3fb79250eb711b960d63ed8fd

9 files changed, 80 insertions, 81 deletions

diff --git a/scripts/py/check_update_resources_downloaded.py b/scripts/py/check_update_resources_downloaded.py
index 44e9bd9..021f1b1 100644
--- a/scripts/py/check_update_resources_downloaded.py
+++ b/scripts/py/check_update_resources_downloaded.py
@@ -13,11 +13,12 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
+
 import json
-import os
 import sys
 import hashlib
 from argparse import ArgumentParser
+from pathlib import Path
 
 
 def get_md5sum_for_file(filepath: str) -> str:
@@ -51,11 +52,9 @@ def check_update_resources_downloaded(
     set_up_script_path (string):       Specifies the path to set_up_default_resources.py file.
     """
 
-    metadata_file_path = os.path.join(
-        resource_downloaded_dir, "resources_downloaded_metadata.json"
-    )
+    metadata_file_path = Path(resource_downloaded_dir) / "resources_downloaded_metadata.json"
 
-    if os.path.isfile(metadata_file_path):
+    if metadata_file_path.is_file():
         with open(metadata_file_path) as metadata_json:
 
             metadata_dict = json.load(metadata_json)
@@ -95,8 +94,8 @@ if __name__ == "__main__":
         required=True)
     args = parser.parse_args()
 
-    # Check validity of script path
-    if not os.path.isfile(args.setup_script_path):
+    # Check validity of script path.
+    if not Path(args.setup_script_path).is_file():
         raise ValueError(f'Invalid script path: {args.setup_script_path}')
 
     # Check the resources are downloaded as expected
diff --git a/scripts/py/gen_audio.py b/scripts/py/gen_audio.py
index 53ed019..e997499 100644
--- a/scripts/py/gen_audio.py
+++ b/scripts/py/gen_audio.py
@@ -36,6 +36,7 @@ parser.add_argument("--min_samples", type=int, help="Minimum sample number.", de
 parser.add_argument("-v", "--verbosity", action="store_true")
 args = parser.parse_args()
 
+
 def main(args):
     audio_data, samplerate = AudioUtils.load_resample_audio_clip(args.audio_path,
                                                 args.sampling_rate,
@@ -44,5 +45,6 @@ def main(args):
                                                 args.min_samples)
     sf.write(path.join(args.output_dir, path.basename(args.audio_path)), audio_data, samplerate)
 
+
 if __name__ == '__main__':
     main(args)
diff --git a/scripts/py/gen_audio_cpp.py b/scripts/py/gen_audio_cpp.py
index e7155c7..b8928f0 100644
--- a/scripts/py/gen_audio_cpp.py
+++ b/scripts/py/gen_audio_cpp.py
@@ -21,12 +21,12 @@ from the cpp files.
 import datetime
 import glob
 import math
-import os
+from pathlib import Path
+from argparse import ArgumentParser
 
 import numpy as np
-from os import path
-from argparse import ArgumentParser
 from jinja2 import Environment, FileSystemLoader
+
 from gen_utils import AudioUtils
 
 parser = ArgumentParser()
@@ -45,7 +45,7 @@ parser.add_argument("--license_template", type=str, help="Header template file",
 parser.add_argument("-v", "--verbosity", action="store_true")
 args = parser.parse_args()
 
-env = Environment(loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), 'templates')),
+env = Environment(loader=FileSystemLoader(Path(__file__).parent / 'templates'),
                   trim_blocks=True,
                   lstrip_blocks=True)
 
@@ -54,7 +54,7 @@ def write_hpp_file(header_filepath, cc_filepath, header_template_file, num_audio
     print(f"++ Generating {header_filepath}")
 
     header_template = env.get_template(header_template_file)
-    hdr = header_template.render(script_name=os.path.basename(__file__),
+    hdr = header_template.render(script_name=Path(__file__).name,
                                  gen_time=datetime.datetime.now(),
                                  year=datetime.datetime.now().year)
     env.get_template('AudioClips.hpp.template').stream(common_template_header=hdr,
@@ -77,8 +77,8 @@ def write_individual_audio_cc_file(clip_dirpath, clip_filename,
                                    cc_filename, header_template_file, array_name,
                                    sampling_rate_value, mono_value, offset_value,
                                    duration_value, res_type_value, min_len):
-    print(f"++ Converting {clip_filename} to {path.basename(cc_filename)}")
-    audio_filepath = path.join(clip_dirpath, clip_filename)
+    print(f"++ Converting {clip_filename} to {Path(cc_filename).name}")
+    audio_filepath = Path(clip_dirpath) / clip_filename
     clip_data, samplerate = AudioUtils.load_resample_audio_clip(audio_filepath,
                                                                 sampling_rate_value, mono_value,
                                                                 offset_value, duration_value,
@@ -90,7 +90,7 @@ def write_individual_audio_cc_file(clip_dirpath, clip_filename,
                         np.iinfo(np.int16).max).flatten().astype(np.int16)
 
     header_template = env.get_template(header_template_file)
-    hdr = header_template.render(script_name=os.path.basename(__file__),
+    hdr = header_template.render(script_name=Path(__file__).name,
                                  gen_time=datetime.datetime.now(),
                                  file_name=clip_filename,
                                  year=datetime.datetime.now().year)
@@ -114,25 +114,24 @@ def main(args):
     audioclip_array_names = []
     header_filename = "InputFiles.hpp"
     common_cc_filename = "InputFiles.cc"
-    header_filepath = path.join(args.header_folder_path, header_filename)
-    common_cc_filepath = path.join(args.source_folder_path, common_cc_filename)
+    header_filepath = Path(args.header_folder_path) / header_filename
+    common_cc_filepath = Path(args.source_folder_path) / common_cc_filename
 
-    if os.path.isdir(args.audio_path):
-        filepaths = sorted(glob.glob(path.join(args.audio_path, '**/*.wav'), recursive=True))
-    elif os.path.isfile(args.audio_path):
+    if Path(args.audio_path).is_dir():
+        filepaths = sorted(glob.glob(str(Path(args.audio_path) / '**/*.wav'), recursive=True))
+    elif Path(args.audio_path).is_file():
         filepaths = [args.audio_path]
     else:
         raise OSError("Directory or file does not exist.")
 
     for filepath in filepaths:
-        filename = path.basename(filepath)
-        clip_dirpath = path.dirname(filepath)
+        filename = Path(filepath).name
+        clip_dirpath = Path(filepath).parent
         try:
             audioclip_filenames.append(filename)
 
             # Save the cc file
-            cc_filename = path.join(args.source_folder_path,
-                                    (filename.rsplit(".")[0]).replace(" ", "_") + ".cc")
+            cc_filename = Path(args.source_folder_path) / (Path(filename).stem.replace(" ", "_") + ".cc")
             array_name = "audio" + str(audioclip_idx)
             array_size = write_individual_audio_cc_file(clip_dirpath, filename, cc_filename, args.license_template, array_name,
                                                         args.sampling_rate, args.mono, args.offset,
diff --git a/scripts/py/gen_default_input_cpp.py b/scripts/py/gen_default_input_cpp.py
index c091fd1..e42f4ae 100644
--- a/scripts/py/gen_default_input_cpp.py
+++ b/scripts/py/gen_default_input_cpp.py
@@ -17,9 +17,9 @@
 Utility script to generate the minimum InputFiles.hpp and cpp files required by an application.
 """
 import datetime
-import os
-
+from pathlib import Path
 from argparse import ArgumentParser
+
 from jinja2 import Environment, FileSystemLoader
 
 parser = ArgumentParser()
@@ -28,7 +28,7 @@ parser.add_argument("--license_template", type=str, help="Header template file",
                     default="header_template.txt")
 args = parser.parse_args()
 
-env = Environment(loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), 'templates')),
+env = Environment(loader=FileSystemLoader(Path(__file__).parent / 'templates'),
                   trim_blocks=True,
                   lstrip_blocks=True)
 
@@ -36,7 +36,7 @@ env = Environment(loader=FileSystemLoader(os.path.join(os.path.dirname(__file__)
 def write_hpp_file(header_file_path, header_template_file):
     print(f"++ Generating {header_file_path}")
     header_template = env.get_template(header_template_file)
-    hdr = header_template.render(script_name=os.path.basename(__file__),
+    hdr = header_template.render(script_name=Path(__file__).name,
                                  gen_time=datetime.datetime.now(),
                                  year=datetime.datetime.now().year)
     env.get_template('default.hpp.template').stream(common_template_header=hdr) \
@@ -45,7 +45,7 @@ def write_hpp_file(header_file_path, header_template_file):
 
 def main(args):
     header_filename = "InputFiles.hpp"
-    header_filepath = os.path.join(args.header_folder_path, header_filename)
+    header_filepath = Path(args.header_folder_path) / header_filename
     write_hpp_file(header_filepath, args.license_template)
 
 
diff --git a/scripts/py/gen_labels_cpp.py b/scripts/py/gen_labels_cpp.py
index 1be9c63..25c4bc3 100644
--- a/scripts/py/gen_labels_cpp.py
+++ b/scripts/py/gen_labels_cpp.py
@@ -22,8 +22,9 @@ this script to be called as part of the build framework to auto-generate the
 cpp file with labels that can be used in the application without modification.
 """
 import datetime
-import os
+from pathlib import Path
 from argparse import ArgumentParser
+
 from jinja2 import Environment, FileSystemLoader
 
 parser = ArgumentParser()
@@ -42,7 +43,7 @@ parser.add_argument("--license_template", type=str, help="Header template file",
 
 args = parser.parse_args()
 
-env = Environment(loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), 'templates')),
+env = Environment(loader=FileSystemLoader(Path(__file__).parent / 'templates'),
                   trim_blocks=True,
                   lstrip_blocks=True)
 
@@ -57,19 +58,18 @@ def main(args):
         raise Exception(f"no labels found in {args.label_file}")
 
     header_template = env.get_template(args.license_template)
-    hdr = header_template.render(script_name=os.path.basename(__file__),
+    hdr = header_template.render(script_name=Path(__file__).name,
                                  gen_time=datetime.datetime.now(),
-                                 file_name=os.path.basename(args.labels_file),
+                                 file_name=Path(args.labels_file).name,
                                  year=datetime.datetime.now().year)
 
-    hpp_filename = os.path.join(args.header_folder_path, args.output_file_name + ".hpp")
+    hpp_filename = Path(args.header_folder_path) / (args.output_file_name + ".hpp")
     env.get_template('Labels.hpp.template').stream(common_template_header=hdr,
-                                                   filename=(args.output_file_name).upper(),
+                                                   filename=args.output_file_name.upper(),
                                                    namespaces=args.namespaces) \
         .dump(str(hpp_filename))
 
-
-    cc_filename = os.path.join(args.source_folder_path, args.output_file_name + ".cc")
+    cc_filename = Path(args.source_folder_path) / (args.output_file_name + ".cc")
     env.get_template('Labels.cc.template').stream(common_template_header=hdr,
                                                   labels=labels,
                                                   labelsSize=len(labels),
diff --git a/scripts/py/gen_model_cpp.py b/scripts/py/gen_model_cpp.py
index c43c93a..de71992 100644
--- a/scripts/py/gen_model_cpp.py
+++ b/scripts/py/gen_model_cpp.py
@@ -19,9 +19,9 @@ project directly. This should be called as part of cmake framework
 should the models need to be generated at configuration stage.
 """
 import datetime
-import os
 from argparse import ArgumentParser
 from pathlib import Path
+
 from jinja2 import Environment, FileSystemLoader
 import binascii
 
@@ -36,7 +36,7 @@ parser.add_argument("--license_template", type=str, help="Header template file",
                     default="header_template.txt")
 args = parser.parse_args()
 
-env = Environment(loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), 'templates')),
+env = Environment(loader=FileSystemLoader(Path(__file__).parent / 'templates'),
                   trim_blocks=True,
                   lstrip_blocks=True)
 
@@ -70,20 +70,20 @@ def get_tflite_data(tflite_path: str) -> list:
 
 
 def main(args):
-    if not os.path.isfile(args.tflite_path):
+    if not Path(args.tflite_path).is_file():
         raise Exception(f"{args.tflite_path} not found")
 
     # Cpp filename:
-    cpp_filename = Path(os.path.join(args.output_dir, os.path.basename(args.tflite_path) + ".cc")).absolute()
-    print(f"++ Converting {os.path.basename(args.tflite_path)} to\
-    {os.path.basename(cpp_filename)}")
+    cpp_filename = (Path(args.output_dir) / (Path(args.tflite_path).name + ".cc")).resolve()
+    print(f"++ Converting {Path(args.tflite_path).name} to\
+    {cpp_filename.name}")
 
-    os.makedirs(cpp_filename.parent, exist_ok=True)
+    cpp_filename.parent.mkdir(exist_ok=True)
 
     header_template = env.get_template(args.license_template)
 
-    hdr = header_template.render(script_name=os.path.basename(__file__),
-                                 file_name=os.path.basename(args.tflite_path),
+    hdr = header_template.render(script_name=Path(__file__).name,
+                                 file_name=Path(args.tflite_path).name,
                                  gen_time=datetime.datetime.now(),
                                  year=datetime.datetime.now().year)
 
diff --git a/scripts/py/gen_rgb_cpp.py b/scripts/py/gen_rgb_cpp.py
index c53fbd7..88ff81e 100644
--- a/scripts/py/gen_rgb_cpp.py
+++ b/scripts/py/gen_rgb_cpp.py
@@ -21,10 +21,10 @@ from the cpp files.
 import datetime
 import glob
 import math
-import os
-import numpy as np
-
+from pathlib import Path
 from argparse import ArgumentParser
+
+import numpy as np
 from PIL import Image, UnidentifiedImageError
 from jinja2 import Environment, FileSystemLoader
 
@@ -37,7 +37,7 @@ parser.add_argument("--license_template", type=str, help="Header template file",
                     default="header_template.txt")
 args = parser.parse_args()
 
-env = Environment(loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), 'templates')),
+env = Environment(loader=FileSystemLoader(Path(__file__).parent / 'templates'),
                   trim_blocks=True,
                   lstrip_blocks=True)
 
@@ -46,7 +46,7 @@ def write_hpp_file(header_file_path, cc_file_path, header_template_file, num_ima
                    image_array_names, image_size):
     print(f"++ Generating {header_file_path}")
     header_template = env.get_template(header_template_file)
-    hdr = header_template.render(script_name=os.path.basename(__file__),
+    hdr = header_template.render(script_name=Path(__file__).name,
                                  gen_time=datetime.datetime.now(),
                                  year=datetime.datetime.now().year)
     env.get_template('Images.hpp.template').stream(common_template_header=hdr,
@@ -63,12 +63,12 @@ def write_hpp_file(header_file_path, cc_file_path, header_template_file, num_ima
 
 def write_individual_img_cc_file(image_filename, cc_filename, header_template_file, original_image,
                                  image_size, array_name):
-    print(f"++ Converting {image_filename} to {os.path.basename(cc_filename)}")
+    print(f"++ Converting {image_filename} to {cc_filename.name}")
 
     header_template = env.get_template(header_template_file)
-    hdr = header_template.render(script_name=os.path.basename(__file__),
+    hdr = header_template.render(script_name=Path(__file__).name,
                                  gen_time=datetime.datetime.now(),
-                                 file_name=os.path.basename(image_filename),
+                                 file_name=image_filename,
                                  year=datetime.datetime.now().year)
     # IFM size
     ifm_width = image_size[0]
@@ -104,16 +104,15 @@ def main(args):
     image_filenames = []
     image_array_names = []
 
-
-    if os.path.isdir(args.image_path):
-        filepaths = sorted(glob.glob(os.path.join(args.image_path, '**/*.*'), recursive=True))
-    elif os.path.isfile(args.image_path):
+    if Path(args.image_path).is_dir():
+        filepaths = sorted(glob.glob(str(Path(args.image_path) / '**/*.*'), recursive=True))
+    elif Path(args.image_path).is_file():
         filepaths = [args.image_path]
     else:
         raise OSError("Directory or file does not exist.")
 
     for filepath in filepaths:
-        filename = os.path.basename(filepath)
+        filename = Path(filepath).name
 
         try:
             original_image = Image.open(filepath).convert("RGB")
@@ -124,8 +123,7 @@ def main(args):
         image_filenames.append(filename)
 
         # Save the cc file
-        cc_filename = os.path.join(args.source_folder_path,
-                                   (filename.rsplit(".")[0]).replace(" ", "_") + ".cc")
+        cc_filename = Path(args.source_folder_path) / (Path(filename).stem.replace(" ", "_") + ".cc")
         array_name = "im" + str(image_idx)
         image_array_names.append(array_name)
         write_individual_img_cc_file(filename, cc_filename, args.license_template,
@@ -135,9 +133,9 @@ def main(args):
         image_idx = image_idx + 1
 
     header_filename = "InputFiles.hpp"
-    header_filepath = os.path.join(args.header_folder_path, header_filename)
+    header_filepath = Path(args.header_folder_path) / header_filename
     common_cc_filename = "InputFiles.cc"
-    common_cc_filepath = os.path.join(args.source_folder_path, common_cc_filename)
+    common_cc_filepath = Path(args.source_folder_path) / common_cc_filename
 
     if len(image_filenames) > 0:
         write_hpp_file(header_filepath, common_cc_filepath, args.license_template,
diff --git a/scripts/py/gen_test_data_cpp.py b/scripts/py/gen_test_data_cpp.py
index dae181d..f1cb30f 100644
--- a/scripts/py/gen_test_data_cpp.py
+++ b/scripts/py/gen_test_data_cpp.py
@@ -39,18 +39,18 @@ parser.add_argument("-v", "--verbosity", action="store_true")
 
 args = parser.parse_args()
 
-env = Environment(loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), 'templates')),
+env = Environment(loader=FileSystemLoader(Path(__file__).parent / 'templates'),
                   trim_blocks=True,
                   lstrip_blocks=True)
 
 
 def write_hpp_file(header_filename, cc_file_path, header_template_file, num_ifms, num_ofms,
                    ifm_array_names, ifm_sizes, ofm_array_names, ofm_sizes, iofm_data_type):
-    header_file_path = os.path.join(args.header_folder_path, header_filename)
+    header_file_path = Path(args.header_folder_path) / header_filename
 
     print(f"++ Generating {header_file_path}")
     header_template = env.get_template(header_template_file)
-    hdr = header_template.render(script_name=os.path.basename(__file__),
+    hdr = header_template.render(script_name=Path(__file__).name,
                                  gen_time=datetime.datetime.now(),
                                  year=datetime.datetime.now().year)
     env.get_template('TestData.hpp.template').stream(common_template_header=hdr,
@@ -74,15 +74,15 @@ def write_hpp_file(header_filename, cc_file_path, header_template_file, num_ifms
 
 
 def write_individual_cc_file(filename, cc_filename, header_filename, header_template_file, array_name, iofm_data_type):
-    print(f"++ Converting {filename} to {os.path.basename(cc_filename)}")
+    print(f"++ Converting {filename} to {cc_filename.name}")
     header_template = env.get_template(header_template_file)
-    hdr = header_template.render(script_name=os.path.basename(__file__),
+    hdr = header_template.render(script_name=Path(__file__).name,
                                  gen_time=datetime.datetime.now(),
-                                 file_name=os.path.basename(filename),
+                                 file_name=filename,
                                  year=datetime.datetime.now().year)
 
     # Convert the image and write it to the cc file
-    fm_data = (np.load(os.path.join(args.data_folder_path, filename))).flatten()
+    fm_data = (np.load(Path(args.data_folder_path) / filename)).flatten()
     type(fm_data.dtype)
     hex_line_generator = (', '.join(map(hex, sub_arr))
                           for sub_arr in np.array_split(fm_data, math.ceil(len(fm_data) / 20)))
@@ -104,8 +104,8 @@ def get_npy_vec_size(filename: str) -> int:
     Return:
         size in bytes
     """
-    data = np.load(os.path.join(args.data_folder_path, filename))
-    return (data.size * data.dtype.itemsize)
+    data = np.load(Path(args.data_folder_path) / filename)
+    return data.size * data.dtype.itemsize
 
 
 def main(args):
@@ -126,7 +126,7 @@ def main(args):
 
     iofm_data_type = "int8_t"
     if ifms_count > 0:
-        iofm_data_type = "int8_t" if (np.load(os.path.join(args.data_folder_path, "ifm0.npy")).dtype == np.int8) else "uint8_t"
+        iofm_data_type = "int8_t" if (np.load(Path(args.data_folder_path) / "ifm0.npy").dtype == np.int8) else "uint8_t"
 
     ifm_sizes = []
     ofm_sizes = []
@@ -136,7 +136,7 @@ def main(args):
         base_name = "ifm" + str(idx)
         filename = base_name+".npy"
         array_name = base_name + add_usecase_fname
-        cc_filename = os.path.join(args.source_folder_path, array_name + ".cc")
+        cc_filename = Path(args.source_folder_path) / (array_name + ".cc")
         ifm_array_names.append(array_name)
         write_individual_cc_file(filename, cc_filename, header_filename, args.license_template, array_name, iofm_data_type)
         ifm_sizes.append(get_npy_vec_size(filename))
@@ -146,12 +146,12 @@ def main(args):
         base_name = "ofm" + str(idx)
         filename = base_name+".npy"
         array_name = base_name + add_usecase_fname
-        cc_filename = os.path.join(args.source_folder_path, array_name + ".cc")
+        cc_filename = Path(args.source_folder_path) / (array_name + ".cc")
         ofm_array_names.append(array_name)
         write_individual_cc_file(filename, cc_filename, header_filename, args.license_template, array_name, iofm_data_type)
         ofm_sizes.append(get_npy_vec_size(filename))
 
-    common_cc_filepath = os.path.join(args.source_folder_path, common_cc_filename)
+    common_cc_filepath = Path(args.source_folder_path) / common_cc_filename
     write_hpp_file(header_filename, common_cc_filepath, args.license_template,
                    ifms_count, ofms_count, ifm_array_names, ifm_sizes, ofm_array_names, ofm_sizes, iofm_data_type)
 
diff --git a/scripts/py/rnnoise_dump_extractor.py b/scripts/py/rnnoise_dump_extractor.py
index 947a75a..ff79dcb 100644
--- a/scripts/py/rnnoise_dump_extractor.py
+++ b/scripts/py/rnnoise_dump_extractor.py
@@ -12,7 +12,6 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
-
 """
 This script can be used with the noise reduction use case to save
 the dumped noise reduced audio to a wav file.
@@ -20,14 +19,15 @@ the dumped noise reduced audio to a wav file.
 Example use:
 python rnnoise_dump_extractor.py --dump_file output.bin --output_dir ./denoised_wavs/
 """
+
 import soundfile as sf
 import numpy as np
 
 import argparse
 from os import path
-
 import struct
 
+
 def extract(fp, output_dir, export_npy):
     while True:
         filename_length = struct.unpack("i", fp.read(4))[0]
@@ -47,19 +47,20 @@ def extract(fp, output_dir, export_npy):
         if export_npy:
             output_file_name += ".npy"
             pack_format = "{}h".format(int(audio_clip_length/2))
-            npdata = np.array(struct.unpack(pack_format,audio_clip)).astype(np.int16)
+            npdata = np.array(struct.unpack(pack_format, audio_clip)).astype(np.int16)
             np.save(output_file_name, npdata)
             print("{} written to disk".format(output_file_name))
 
+
 def main(args):
     extract(args.dump_file, args.output_dir, args.export_npy)
 
+
 parser = argparse.ArgumentParser()
 parser.add_argument("--dump_file", type=argparse.FileType('rb'), help="Dump file with audio files to extract.", required=True)
 parser.add_argument("--output_dir", help="Output directory, Warning: Duplicated file names will be overwritten.", required=True)
 parser.add_argument("--export_npy", help="Export the audio buffer in NumPy format", action="store_true")
 args = parser.parse_args()
 
-if __name__=="__main__":
+if __name__ == "__main__":
     main(args)
-