From 66b4a6a8ca1ee55e5b7f05bae2543cf99fe22d6d Mon Sep 17 00:00:00 2001 From: Gunes Bayir Date: Sat, 1 Jul 2023 22:55:42 +0100 Subject: Setup pre-commit and include code formatting scripts This patch - includes our code formatting scripts used in our precommit pipeline - sets up pre-commit framework to help contributors validate their patches This has several benefits: - our repository becomes more inclusive to external contributions - we can use several hooks available online efficiently, w/o implementing our own - it speeds up our development flow and, it is completely optional. The pre-commit configuration includes running the following: - our code formatting scripts - CMake and Bazel build file generation scripts - hooks that check trailing whitespace, end of files, committed large files etc. The number of checks can easily be extended using pre-commit framework. Change-Id: I06bf1259715579d321f89820726a8301504c36d9 Signed-off-by: Gunes Bayir Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/10064 Reviewed-by: Jakub Sujak Tested-by: Arm Jenkins Comments-Addressed: Arm Jenkins Benchmark: Arm Jenkins --- scripts/ensure_single_eol.py | 48 +++++ scripts/format_code.py | 464 ++++++++++++++++++++++++++++++++++++++++ scripts/format_doxygen.py | 2 +- scripts/generate_android_bp.py | 213 ++++++++++++++++++ scripts/generate_build_files.py | 2 - scripts/modules/Shell.py | 105 +++++++++ 6 files changed, 831 insertions(+), 3 deletions(-) create mode 100755 scripts/ensure_single_eol.py create mode 100755 scripts/format_code.py create mode 100755 scripts/generate_android_bp.py create mode 100755 scripts/modules/Shell.py (limited to 'scripts') diff --git a/scripts/ensure_single_eol.py b/scripts/ensure_single_eol.py new file mode 100755 index 0000000000..0eb105e091 --- /dev/null +++ b/scripts/ensure_single_eol.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2023 Arm Limited. +# +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +This script ensures a file ends with exactly one end-of-line character +""" + +import argparse +import pathlib as pl +import os + + +def main(args): + f_p = pl.Path(args.file) + with open(f_p, "r") as f: + lines = f.read() + lines = lines.rstrip(os.linesep) + lines += os.linesep + with open(f_p, "w") as f: + f.write(lines) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("file") + args = parser.parse_args() + main(args) diff --git a/scripts/format_code.py b/scripts/format_code.py new file mode 100755 index 0000000000..fa572cf51c --- /dev/null +++ b/scripts/format_code.py @@ -0,0 +1,464 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2023 Arm Limited. +# +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import argparse +import datetime +import difflib +import filecmp +import logging +import os +import re +import subprocess +import sys + +from modules.Shell import Shell + +logger = logging.getLogger("format_code") + +ASTYLE_PARAMETERS ="--style=ansi \ + --indent=spaces \ + --indent-switches \ + --indent-col1-comments \ + --min-conditional-indent=0 \ + --max-instatement-indent=120 \ + --pad-oper \ + --align-pointer=name \ + --align-reference=name \ + --break-closing-brackets \ + --keep-one-line-statements \ + --max-code-length=200 \ + --mode=c \ + --lineend=linux \ + --indent-preprocessor \ + " + +exceptions = [ + "src/core/NEON/kernels/assembly/gemm", + "src/core/NEON/kernels/assembly/arm", + "/winograd/", + "/convolution/", + "/arm_gemm/", + "/arm_conv/", + "compute_kernel_writer/" +] + +def adjust_copyright_year(copyright_years, curr_year): + ret_copyright_year = str() + # Read last year in the Copyright + last_year = int(copyright_years[-4:]) + if last_year == curr_year: + ret_copyright_year = copyright_years + elif last_year == (curr_year - 1): + # Create range if latest year on the copyright is the previous + if len(copyright_years) > 4 and copyright_years[-5] == "-": + # Range already exists, update year to current + ret_copyright_year = copyright_years[:-5] + "-" + str(curr_year) + else: + # Create a new range + ret_copyright_year = copyright_years + "-" + str(curr_year) + else: + ret_copyright_year = copyright_years + ", " + str(curr_year) + return ret_copyright_year + +def check_copyright( filename ): + f = open(filename, "r") + content = f.readlines() + f.close() + f = open(filename, "w") + year = datetime.datetime.now().year + ref = open("scripts/copyright_mit.txt","r").readlines() + + # Need to handle python files separately + if("SConstruct" in filename or "SConscript" in filename): + start = 2 + if("SConscript" in filename): + start = 3 + m = re.match("(# Copyright \(c\) )(.*\d{4})( [Arm|ARM].*)", content[start]) + line = m.group(1) + + if m.group(2): # Is there a year already? + # Yes: adjust accordingly + line += adjust_copyright_year(m.group(2), year) + else: + # No: add current year + line += str(year) + line += m.group(3).replace("ARM", "Arm") + if("SConscript" in filename): + f.write('#!/usr/bin/python\n') + + f.write('# -*- coding: utf-8 -*-\n\n') + f.write(line+"\n") + # Copy the rest of the file's content: + f.write("".join(content[start + 1:])) + f.close() + + return + + # This only works until year 9999 + m = re.match("(.*Copyright \(c\) )(.*\d{4})( [Arm|ARM].*)", content[1]) + start =len(ref)+2 + if content[0] != "/*\n" or not m: + start = 0 + f.write("/*\n * Copyright (c) %d Arm Limited.\n" % year) + else: + logger.debug("Found Copyright start") + logger.debug("\n\t".join([ g or "" for g in m.groups()])) + line = m.group(1) + + if m.group(2): # Is there a year already? + # Yes: adjust accordingly + line += adjust_copyright_year(m.group(2), year) + else: + # No: add current year + line += str(year) + line += m.group(3).replace("ARM", "Arm") + f.write("/*\n"+line+"\n") + logger.debug(line) + # Write out the rest of the Copyright header: + for i in range(1, len(ref)): + line = ref[i] + f.write(" *") + if line.rstrip() != "": + f.write(" %s" % line) + else: + f.write("\n") + f.write(" */\n") + # Copy the rest of the file's content: + f.write("".join(content[start:])) + f.close() + +def check_license(filename): + """ + Check that the license file is up-to-date + """ + f = open(filename, "r") + content = f.readlines() + f.close() + + f = open(filename, "w") + f.write("".join(content[:2])) + + year = datetime.datetime.now().year + # This only works until year 9999 + m = re.match("(.*Copyright \(c\) )(.*\d{4})( [Arm|ARM].*)", content[2]) + + if not m: + f.write("Copyright (c) {} Arm Limited\n".format(year)) + else: + updated_year = adjust_copyright_year(m.group(2), year) + f.write("Copyright (c) {} Arm Limited\n".format(updated_year)) + + # Copy the rest of the file's content: + f.write("".join(content[3:])) + f.close() + + +class OtherChecksRun: + def __init__(self, folder, error_diff=False, strategy="all"): + self.folder = folder + self.error_diff=error_diff + self.strategy = strategy + + def error_on_diff(self, msg): + retval = 0 + if self.error_diff: + diff = self.shell.run_single_to_str("git diff") + if len(diff) > 0: + retval = -1 + logger.error(diff) + logger.error("\n"+msg) + return retval + + def run(self): + retval = 0 + self.shell = Shell() + self.shell.save_cwd() + this_dir = os.path.dirname(__file__) + self.shell.cd(self.folder) + self.shell.prepend_env("PATH","%s/../bin" % this_dir) + + to_check = "" + if self.strategy != "all": + to_check, skip_copyright = FormatCodeRun.get_files(self.folder, self.strategy) + #FIXME: Exclude shaders! + + logger.info("Running ./scripts/format_doxygen.py") + logger.debug(self.shell.run_single_to_str("./scripts/format_doxygen.py %s" % " ".join(to_check))) + retval = self.error_on_diff("Doxygen comments badly formatted (check above diff output for more details) try to run ./scripts/format_doxygen.py on your patch and resubmit") + if retval == 0: + logger.info("Running ./scripts/include_functions_kernels.py") + logger.debug(self.shell.run_single_to_str("python ./scripts/include_functions_kernels.py")) + retval = self.error_on_diff("Some kernels or functions are not included in their corresponding master header (check above diff output to see which includes are missing)") + if retval == 0: + try: + logger.info("Running ./scripts/check_bad_style.sh") + logger.debug(self.shell.run_single_to_str("./scripts/check_bad_style.sh")) + #logger.debug(self.shell.run_single_to_str("./scripts/check_bad_style.sh %s" % " ".join(to_check))) + except subprocess.CalledProcessError as e: + logger.error("Command %s returned:\n%s" % (e.cmd, e.output)) + retval -= 1 + + if retval != 0: + raise Exception("format-code failed with error code %d" % retval) + +class FormatCodeRun: + @staticmethod + def get_files(folder, strategy="git-head"): + shell = Shell() + shell.cd(folder) + skip_copyright = False + if strategy == "git-head": + cmd = "git diff-tree --no-commit-id --name-status -r HEAD | grep \"^[AMRT]\" | cut -f 2" + elif strategy == "git-diff": + cmd = "git diff --name-status --cached -r HEAD | grep \"^[AMRT]\" | cut -f 2" + else: + cmd = "git ls-tree -r HEAD --name-only" + # Skip copyright checks when running on all files because we don't know when they were last modified + # Therefore we can't tell if their copyright dates are correct + skip_copyright = True + + grep_folder = "grep -e \"^\\(arm_compute\\|src\\|examples\\|tests\\|utils\\|support\\)/\"" + grep_extension = "grep -e \"\\.\\(cpp\\|h\\|inl\\|cl\\|cs\\|hpp\\)$\"" + list_files = shell.run_single_to_str(cmd+" | { "+ grep_folder+" | "+grep_extension + " || true; }") + to_check = [ f for f in list_files.split("\n") if len(f) > 0] + + # Check for scons files as they are excluded from the above list + list_files = shell.run_single_to_str(cmd+" | { grep -e \"SC\" || true; }") + to_check += [ f for f in list_files.split("\n") if len(f) > 0] + + return (to_check, skip_copyright) + + def __init__(self, files, folder, error_diff=False, skip_copyright=False): + self.files = files + self.folder = folder + self.skip_copyright = skip_copyright + self.error_diff=error_diff + + def error_on_diff(self, msg): + retval = 0 + if self.error_diff: + diff = self.shell.run_single_to_str("git diff") + if len(diff) > 0: + retval = -1 + logger.error(diff) + logger.error("\n"+msg) + return retval + + def run(self): + if len(self.files) < 1: + logger.debug("No file: early exit") + retval = 0 + self.shell = Shell() + self.shell.save_cwd() + this_dir = os.path.dirname(__file__) + try: + self.shell.cd(self.folder) + self.shell.prepend_env("PATH","%s/../bin" % this_dir) + clang_format = "clang-format -i -style=file " + astyle = "astyle -n -q %s " % (ASTYLE_PARAMETERS) + + if sys.platform == 'darwin': + # this platform explicitly needs an extension for the temporary file + sed = "sed -i '.log' 's/\\t/ /g' " + else: + sed = "sed -i 's/\\t/ /g' " + + single_eol = "%s/ensure_single_eol.py " % this_dir + for f in self.files: + skip_this_file = False + for e in exceptions: + if e in f: + logger.warning("Skipping '%s' file: %s" % (e,f)) + skip_this_file = True + break + if skip_this_file: + continue + + logger.info("Formatting %s" % f) + if not self.skip_copyright: + check_copyright(f) + cmds = [ + sed + f, + clang_format + f, + astyle + f, + single_eol + f + ] + + if sys.platform == 'darwin': + # the temporary file creted by 'sed' will be removed here + cmds.append(f"rm {f}.log") + + for cmd in cmds: + output = self.shell.run_single_to_str(cmd) + if len(output) > 0: + logger.info(output) + + check_license("LICENSE") + + except subprocess.CalledProcessError as e: + retval = -1 + logger.error(e) + logger.error("OUTPUT= %s" % e.output) + + retval += self.error_on_diff("See above for clang-tidy errors") + + if retval != 0: + raise Exception("format-code failed with error code %d" % retval) + +class GenerateAndroidBP: + def __init__(self, folder): + self.folder = folder + self.bp_output_file = "Generated_Android.bp" + + def run(self): + retval = 0 + self.shell = Shell() + self.shell.save_cwd() + this_dir = os.path.dirname(__file__) + + logger.debug("Running Android.bp check") + try: + self.shell.cd(self.folder) + cmd = "%s/generate_android_bp.py --folder %s --output_file %s" % (this_dir, self.folder, self.bp_output_file) + output = self.shell.run_single_to_str(cmd) + if len(output) > 0: + logger.info(output) + except subprocess.CalledProcessError as e: + retval = -1 + logger.error(e) + logger.error("OUTPUT= %s" % e.output) + + # Compare the genereated file with the one in the review + if not filecmp.cmp(self.bp_output_file, self.folder + "/Android.bp"): + is_mismatched = True + + with open(self.bp_output_file, 'r') as generated_file: + with open(self.folder + "/Android.bp", 'r') as review_file: + diff = list(difflib.unified_diff(generated_file.readlines(), review_file.readlines(), + fromfile='Generated_Android.bp', tofile='Android.bp')) + + # If the only mismatch in Android.bp file is the copyright year, + # the content of the file is considered unchanged and we don't need to update + # the copyright year. This will resolve the issue that emerges every new year. + num_added_lines = 0 + num_removed_lines = 0 + last_added_line = "" + last_removed_line = "" + expect_add_line = False + + for line in diff: + if line.startswith("-") and not line.startswith("---"): + num_removed_lines += 1 + if num_removed_lines > 1: + break + last_removed_line = line + expect_add_line = True + elif line.startswith("+") and not line.startswith("+++"): + num_added_lines += 1 + if num_added_lines > 1: + break + if expect_add_line: + last_added_line = line + else: + expect_add_line = False + + if num_added_lines == 1 and num_removed_lines == 1: + re_copyright = re.compile("^(?:\+|\-)// Copyright © ([0-9]+)\-([0-9]+) Arm Ltd. All rights reserved.\n$") + generated_matches = re_copyright.search(last_removed_line) + review_matches = re_copyright.search(last_added_line) + + if generated_matches is not None and review_matches is not None: + if generated_matches.group(1) == review_matches.group(1) and \ + int(generated_matches.group(2)) > int(review_matches.group(2)): + is_mismatched = False + + if is_mismatched: + logger.error("Lines with '-' need to be added to Android.bp") + logger.error("Lines with '+' need to be removed from Android.bp") + + for line in diff: + logger.error(line.rstrip()) + if is_mismatched: + raise Exception("Android bp file is not updated") + + if retval != 0: + raise Exception("generate Android bp file failed with error code %d" % retval) + +def run_fix_code_formatting( files="git-head", folder=".", num_threads=1, error_on_diff=True): + try: + retval = 0 + + # Genereate Android.bp file and test it + gen_android_bp = GenerateAndroidBP(folder) + gen_android_bp.run() + + to_check, skip_copyright = FormatCodeRun.get_files(folder, files) + other_checks = OtherChecksRun(folder,error_on_diff, files) + other_checks.run() + + logger.debug(to_check) + num_files = len(to_check) + per_thread = max( num_files / num_threads,1) + start=0 + logger.info("Files to format:\n\t%s" % "\n\t".join(to_check)) + + for i in range(num_threads): + if i == num_threads -1: + end = num_files + else: + end= min(start+per_thread, num_files) + sub = to_check[start:end] + logger.debug("[%d] [%d,%d] %s" % (i, start, end, sub)) + start = end + format_code_run = FormatCodeRun(sub, folder, skip_copyright=skip_copyright) + format_code_run.run() + + return retval + except Exception as e: + logger.error("Exception caught in run_fix_code_formatting: %s" % e) + return -1 + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description="Build & run pre-commit tests", + ) + + file_sources=["git-diff","git-head","all"] + parser.add_argument("-D", "--debug", action='store_true', help="Enable script debugging output") + parser.add_argument("--error_on_diff", action='store_true', help="Show diff on error and stop") + parser.add_argument("--files", nargs='?', metavar="source", choices=file_sources, help="Which files to run fix_code_formatting on, choices=%s" % file_sources, default="git-head") + parser.add_argument("--folder", metavar="path", help="Folder in which to run fix_code_formatting", default=".") + + args = parser.parse_args() + + logging_level = logging.INFO + if args.debug: + logging_level = logging.DEBUG + + logging.basicConfig(level=logging_level) + + logger.debug("Arguments passed: %s" % str(args.__dict__)) + + exit(run_fix_code_formatting(args.files, args.folder, 1, error_on_diff=args.error_on_diff)) diff --git a/scripts/format_doxygen.py b/scripts/format_doxygen.py index 5882958fc5..8ac5e630b9 100755 --- a/scripts/format_doxygen.py +++ b/scripts/format_doxygen.py @@ -81,7 +81,7 @@ def process_comment(fd, comment, first_param, last_param): if __name__ == "__main__": n_file=0 - if len(sys.argv) == 1: + if len(sys.argv) == 2 and sys.argv[1] == '--all': paths = [] for top_level in ["./arm_compute", "./src", "./examples", "./tests", "./utils", "./framework", "./support"]: diff --git a/scripts/generate_android_bp.py b/scripts/generate_android_bp.py new file mode 100755 index 0000000000..350b52f01c --- /dev/null +++ b/scripts/generate_android_bp.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2023 Arm Limited. +# +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import argparse +import os +from jinja2 import Template +import datetime + +# Paths to exclude +excluded_paths = ["build", + "compute_kernel_writer/build", + "compute_kernel_writer/src", + "compute_kernel_writer/validation", + "docs/", + "documentation/", + "examples/", + "opencl-1.2-stubs/", + "release_repository/", + "opengles-3.1-stubs/", + "scripts/", + "tests/", + "/GLES_COMPUTE/", + "/graph/", + "/sve/", + "/SVE/", + "/sve2/", + "/SVE2/" + ] + +excluded_files = ["TracePoint.cpp"] + +# Android bp template to render +year = datetime.datetime.now().year + +bp_tm = Template( +"""// +// Copyright © 2020-""" + str(year) + """ Arm Ltd. All rights reserved. +// SPDX-License-Identifier: MIT +// + +// OpenCL sources are NOT required by ArmNN or its Android NNAPI driver and are used for CI purposes only. +opencl_srcs = [ + {% for cl_src in cl_srcs -%} + "{{ cl_src }}", + {% endfor %} +] + +bootstrap_go_package { + name: "arm_compute_library_nn_driver", + pkgPath: "arm_compute_library_nn_driver", + deps: [ + "blueprint", + "blueprint-pathtools", + "blueprint-proptools", + "soong", + "soong-android", + "soong-cc", + ], + srcs: [ + "scripts/arm_compute_library_nn_driver.go", + ], + pluginFor: [ "soong_build" ], +} + +arm_compute_library_defaults { + name: "acl-default-cppflags", + cppflags: [ + "-std=c++14", + "-fexceptions", + "-DBOOST_NO_AUTO_PTR", + "-DEMBEDDED_KERNELS", + "-DARM_COMPUTE_ASSERTS_ENABLED", + "-DARM_COMPUTE_CPP_SCHEDULER", + "-DENABLE_NEON", + "-DARM_COMPUTE_ENABLE_NEON", + "-Wno-unused-parameter", + "-DNO_DOT_IN_TOOLCHAIN", + "-Wno-implicit-fallthrough", + "-fPIC", + "-DACL_INTERNAL_TEST_CKW_IN_DF" + ], + rtti: true, +} + +cc_library_static { + name: "arm_compute_library", + defaults: ["acl-default-cppflags"], + proprietary: true, + local_include_dirs: ["build/android-arm64v8a/src/core", + "build/android-arm64v8a/src/core/CL", + "compute_kernel_writer/prototype/include", + "compute_kernel_writer/prototype", + "src/core/common", + "src/core/helpers", + "src/core/NEON/kernels/arm_gemm", + "src/core/NEON/kernels/assembly", + "src/core/NEON/kernels/convolution/common", + "src/core/NEON/kernels/convolution/winograd", + "src/cpu/kernels/assembly"], + export_include_dirs: [".", "./include"], + srcs: [ + {% for src in srcs -%} + "{{ src }}", + {% endfor %} + ], + arch: { + arm: { + srcs: [ + {% for arm_src in arm_srcs -%} + "{{ arm_src }}", + {% endfor %} + ], + }, + arm64: { + srcs: [ + {% for arm64_src in arm64_srcs -%} + "{{ arm64_src }}", + {% endfor %} + ], + }, + }, + rtti: true, +} +""") + + +def generate_bp_file(cpp_files, opencl_files): + arm_files = [f for f in cpp_files if "a32_" in f] + arm64_files = [f for f in cpp_files if any(a64 in f for a64 in ["a64_", "sve_", 'sme_', 'sme2_'])] + gen_files = [x for x in cpp_files if x not in arm_files + arm64_files] + + arm_files.sort() + arm64_files.sort() + gen_files.sort() + opencl_files.sort() + + bp_file = bp_tm.render(srcs=gen_files, + arm_srcs=arm_files, + arm64_srcs=arm64_files, + cl_srcs=opencl_files) + return bp_file + + +def list_all_files(repo_path): + """ Gets the list of files to include to the Android.bp + + :param repo_path: Path of the repository + :return: The filtered list of useful filess + """ + if not repo_path.endswith('/'): + repo_path = repo_path + "/" + + # Get cpp files + cpp_files = [] + cl_files = [] + for path, subdirs, files in os.walk(repo_path): + for file in files: + if file.endswith(".cpp"): + cpp_files.append(os.path.join(path, file)) + elif file.endswith(".cl"): + cl_files.append(os.path.join(path, file)) + # Include CL headers + if "src/core/CL/cl_kernels" in path and file.endswith(".h"): + cl_files.append(os.path.join(path, file)) + # Filter out unused cpp files + filtered_cpp_files = [] + for cpp_file in cpp_files: + if any(ep in cpp_file for ep in excluded_paths) or any(ef in cpp_file for ef in excluded_files): + continue + filtered_cpp_files.append(cpp_file.replace(repo_path, "")) + # Filter out unused cl files + filtered_cl_files = [] + for cl_file in cl_files: + if any(ep in cl_file for ep in excluded_paths): + continue + filtered_cl_files.append(cl_file.replace(repo_path, "")) + + return filtered_cpp_files, filtered_cl_files + + +if __name__ == "__main__": + # Parse arguments + parser = argparse.ArgumentParser('Generate Android.bp file for ComputeLibrary') + parser.add_argument('--folder', default=".", metavar="folder", dest='folder', type=str, required=False, help='Compute Library source path') + parser.add_argument('--output_file', metavar="output_file", default='Android.bp', type=str, required=False, help='Specify Android bp output file') + args = parser.parse_args() + + cpp_files, opencl_files = list_all_files(args.folder) + bp_file = generate_bp_file(cpp_files, opencl_files) + + with open(args.output_file, 'w') as f: + f.write(bp_file) diff --git a/scripts/generate_build_files.py b/scripts/generate_build_files.py index 0ca27179eb..61da7f23ed 100644 --- a/scripts/generate_build_files.py +++ b/scripts/generate_build_files.py @@ -280,14 +280,12 @@ if "__main__" in __name__: if args.bazel: bazel_build_string = build_from_template_bazel( graph_files, lib_files_sve, lib_files_sve2, lib_files) - print(bazel_build_string) with open("src/BUILD.bazel", "w") as fp: fp.write(bazel_build_string) if args.cmake: cmake_build_string = build_from_template_cmake( graph_files, lib_files_sve, lib_files_sve2, lib_files) - print(cmake_build_string) with open("src/CMakeLists.txt", "w") as fp: fp.write(cmake_build_string) diff --git a/scripts/modules/Shell.py b/scripts/modules/Shell.py new file mode 100755 index 0000000000..f3fd0bd242 --- /dev/null +++ b/scripts/modules/Shell.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python + +# Copyright (c) 2023 Arm Limited. +# +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import os +import logging +import subprocess + +logger = logging.getLogger("Shell") + +class Shell: + def __init__(self, is_interactive=False): + self.line="" + self.env=os.environ.copy() + self.initial_path = self.env["PATH"] + self.save_cwd() + self.is_interactive = is_interactive + + def reset_path(self): + self.env["PATH"]=self.initial_path + + def set_env(self, key, value): + self.env[key] = value + + def append_env(self, key, value): + logger.debug("Appending '%s' to '%s'" % (value, key)) + if key not in list(self.env.keys()): + self.set_env(key,value) + else: + self.env[key] += ":"+value + def prepend_env(self, key, value): + logger.debug("Prepending '%s' to '%s'" % (value, key)) + if key not in list(self.env.keys()): + self.set_env(key,value) + else: + self.env[key] = value+":"+self.env[key] + def run(self, cmd): + if isinstance(cmd, list): + for c in cmd: + self.run_single(c) + else: + self.run_single(cmd) + def run_to_str(self, cmd): + out = "" + if isinstance(cmd, list): + for c in cmd: + out += self.run_single_to_str(c) + else: + out = self.run_single_to_str(cmd) + return out + def cd(self, dirname): + os.chdir(dirname) + + def save_cwd(self): + self.cwd = os.getcwd() + + def restore_cwd(self): + self.cd( self.cwd ) + + def run_single_interactive(self,cmd): + subprocess.check_call(cmd, env=self.env,stderr=subprocess.STDOUT, shell=True) + logger.debug("%s returned" % cmd) + + def run_single(self,cmd): + if self.is_interactive: + self.run_single_interactive(cmd) + else: + self.run_single_to_str(cmd) + + def run_single_to_str_no_output_check(self,cmd): + try: + out = subprocess.check_output(cmd, env=self.env, stderr=subprocess.STDOUT, shell=True) + except subprocess.CalledProcessError as cpe: + out = cpe.output + if (len(out.strip()) > 0): + logger.debug(out) + logger.debug("%s returned" % cmd) + return out + + def run_single_to_str(self,cmd): + out = subprocess.check_output(cmd, env=self.env, stderr=subprocess.STDOUT, shell=True).decode('utf-8') + if (len(out.strip()) > 0): + logger.debug(out) + logger.debug("%s returned" % cmd) + return out -- cgit v1.2.1