From 6a2ac465896ce6f980f0f3d374d6d401b8acb900 Mon Sep 17 00:00:00 2001 From: Kshitij Sisodia Date: Tue, 1 Mar 2022 17:36:06 +0000 Subject: MLECO-2881: Using MD5 sum for downloaded resources check. Removing git as a requirement for the checks performed for downloaded resources. Another minor change is renaming of RTE_components.h to RTE_Components.h as per the convention. Change-Id: If93f80f2f5dfa6a3f143259904c33b3b6d3a6e7c Signed-off-by: Kshitij Sisodia --- scripts/py/check_update_resources_downloaded.py | 60 +++++++++++++--------- set_up_default_resources.py | 25 ++++----- source/hal/cmsis_device/CMakeLists.txt | 3 +- source/hal/cmsis_device/include/RTE_Components.h | 24 +++++++++ source/hal/cmsis_device/include/RTE_components.h | 24 --------- .../components/ethosu_npu_init/ethosu_npu_init.c | 2 +- .../hal/platform/mps3/include/platform_drivers.h | 2 +- source/hal/platform/mps3/source/include/smm_mps3.h | 2 +- .../hal/platform/simple/include/platform_drivers.h | 2 +- .../simple/include/timer_simple_platform.h | 2 +- .../platform/simple/source/timer_simple_platform.c | 2 +- 11 files changed, 78 insertions(+), 70 deletions(-) create mode 100644 source/hal/cmsis_device/include/RTE_Components.h delete mode 100644 source/hal/cmsis_device/include/RTE_components.h diff --git a/scripts/py/check_update_resources_downloaded.py b/scripts/py/check_update_resources_downloaded.py index c016665..44e9bd9 100644 --- a/scripts/py/check_update_resources_downloaded.py +++ b/scripts/py/check_update_resources_downloaded.py @@ -15,11 +15,30 @@ # limitations under the License. import json import os -import subprocess import sys +import hashlib from argparse import ArgumentParser +def get_md5sum_for_file(filepath: str) -> str: + """ + Function to calculate md5sum for contents of a given file. + + Parameters: + ---------- + filepath (string): Path to the required file. + + Returns: + ------- + Hex digest represented as string. + """ + md5_sum = hashlib.md5() + with open(filepath, mode='rb') as f: + buf = f.read() + md5_sum.update(buf) + return md5_sum.hexdigest() + + def check_update_resources_downloaded( resource_downloaded_dir: str, set_up_script_path: str ): @@ -40,20 +59,25 @@ def check_update_resources_downloaded( with open(metadata_file_path) as metadata_json: metadata_dict = json.load(metadata_json) - set_up_script_hash = metadata_dict["set_up_script_hash"] - command = f"git log -1 --pretty=tformat:%H {set_up_script_path}" + md5_key = 'set_up_script_md5sum' + set_up_script_md5sum_metadata = '' + + if md5_key in metadata_dict.keys(): + set_up_script_md5sum_metadata = metadata_dict["set_up_script_md5sum"] - proc = subprocess.run( - command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True - ) - git_commit_hash = proc.stdout.decode("utf-8").strip("\n") - proc.check_returncode() + set_up_script_md5sum_current = get_md5sum_for_file(set_up_script_path) - if set_up_script_hash == git_commit_hash: + if set_up_script_md5sum_current == set_up_script_md5sum_metadata: return 0 + # Return code 1 if the resources need to be refreshed. + print('Error: hash mismatch!') + print(f'Metadata: {set_up_script_md5sum_metadata}') + print(f'Current : {set_up_script_md5sum_current}') return 1 + # Return error code 2 if the file doesn't exists. + print(f'Error: could not find {metadata_file_path}') return 2 @@ -75,18 +99,8 @@ if __name__ == "__main__": if not os.path.isfile(args.setup_script_path): raise ValueError(f'Invalid script path: {args.setup_script_path}') - # Check if the repo root directory is a git repository - root_file_dir = os.path.dirname(os.path.abspath(args.setup_script_path)) - is_git_repo = os.path.exists(os.path.join(root_file_dir, ".git")) - - # if we have a git repo then check the resources are downloaded, - # otherwise it's considered a prerequisite to have run - # the set_up_default_resources.py - status = ( - check_update_resources_downloaded( - args.resource_downloaded_dir, args.setup_script_path - ) - if is_git_repo - else 0 - ) + # Check the resources are downloaded as expected + status = check_update_resources_downloaded( + args.resource_downloaded_dir, + args.setup_script_path) sys.exit(status) diff --git a/set_up_default_resources.py b/set_up_default_resources.py index 48e6a67..899fbe6 100755 --- a/set_up_default_resources.py +++ b/set_up_default_resources.py @@ -27,6 +27,7 @@ from argparse import ArgumentParser from argparse import ArgumentTypeError from collections import namedtuple from urllib.error import URLError +from scripts.py.check_update_resources_downloaded import get_md5sum_for_file json_uc_res = [ @@ -356,6 +357,8 @@ def set_up_resources( greater than 0 is provided, this will be taken as the cache size. If 0, the default values, as per the NPU config requirements, are used. + check_clean_folder (bool): Indicates whether the resources folder needs to + be checked for updates and cleaned. """ # Paths current_file_dir = os.path.dirname(os.path.abspath(__file__)) @@ -369,14 +372,8 @@ def set_up_resources( metadata_dict = dict() vela_version = "3.2.0" - # Check if the current directory is a git repository - is_git_repo = os.path.exists(os.path.join(current_file_dir, ".git")) - git_commit_hash = "" - setup_script_hash_changed = False - if is_git_repo: - # If the current directory is a git script then extract the set_up_default_resorces.py hash - command = f"git log -1 --pretty=tformat:%H {os.path.abspath(__file__)}" - git_commit_hash = call_command(command, False) + setup_script_hash_verified = False + setup_script_hash = get_md5sum_for_file(os.path.abspath(__file__)) try: # 1.1 Does the download dir exist? @@ -398,11 +395,9 @@ def set_up_resources( remove_tree_dir(download_dir) metadata_dict = dict() else: - # Check if the set_up_default_resorces.py has changed from last setup, only if this is a git repo - if is_git_repo: - setup_script_hash_changed = not ( - metadata_dict["set_up_script_hash"] == git_commit_hash - ) + # Check if the set_up_default_resorces.py has changed from last setup + setup_script_hash_verified = ( + metadata_dict.get('set_up_script_md5sum') == setup_script_hash) else: raise @@ -444,7 +439,7 @@ def set_up_resources( except OSError as e: if e.errno == errno.EEXIST: # The usecase_name download dir exist - if setup_script_hash_changed: + if check_clean_folder and not setup_script_hash_verified: for idx, metadata_uc_url_prefix in enumerate( [ f @@ -588,7 +583,7 @@ def set_up_resources( # 4. Collect and write metadata logging.info("Collecting and write metadata.") metadata_dict["ethosu_vela_version"] = vela_version - metadata_dict["set_up_script_hash"] = git_commit_hash.strip("\n") + metadata_dict["set_up_script_md5sum"] = setup_script_hash.strip("\n") metadata_dict["resources_info"] = json_uc_res with open(metadata_file_path, "w") as metadata_file: diff --git a/source/hal/cmsis_device/CMakeLists.txt b/source/hal/cmsis_device/CMakeLists.txt index b98feb2..4e5ce72 100644 --- a/source/hal/cmsis_device/CMakeLists.txt +++ b/source/hal/cmsis_device/CMakeLists.txt @@ -64,8 +64,7 @@ target_link_options( # Check if semihosting configuration is available if (COMMAND configure_semihosting) - option(USE_SEMIHOSTING "Enable/disable semihosting option" OFF) - configure_semihosting(${CMSIS_DEVICE_TARGET} ${USE_SEMIHOSTING}) + configure_semihosting(${CMSIS_DEVICE_TARGET} OFF) endif() # 4 Display status: diff --git a/source/hal/cmsis_device/include/RTE_Components.h b/source/hal/cmsis_device/include/RTE_Components.h new file mode 100644 index 0000000..8988e9b --- /dev/null +++ b/source/hal/cmsis_device/include/RTE_Components.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2022 Arm Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef RTE_COMPONENTS_H +#define RTE_COMPONENTS_H + +#if defined(CPU_HEADER_FILE) +#include CPU_HEADER_FILE /* Cortex M system header file from CMSIS. */ +#endif /* CPU_HEADER_FILE */ + +#endif /* RTE_COMPONENTS_H */ diff --git a/source/hal/cmsis_device/include/RTE_components.h b/source/hal/cmsis_device/include/RTE_components.h deleted file mode 100644 index 8988e9b..0000000 --- a/source/hal/cmsis_device/include/RTE_components.h +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2022 Arm Limited. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef RTE_COMPONENTS_H -#define RTE_COMPONENTS_H - -#if defined(CPU_HEADER_FILE) -#include CPU_HEADER_FILE /* Cortex M system header file from CMSIS. */ -#endif /* CPU_HEADER_FILE */ - -#endif /* RTE_COMPONENTS_H */ diff --git a/source/hal/components/ethosu_npu_init/ethosu_npu_init.c b/source/hal/components/ethosu_npu_init/ethosu_npu_init.c index 2fed693..161d613 100644 --- a/source/hal/components/ethosu_npu_init/ethosu_npu_init.c +++ b/source/hal/components/ethosu_npu_init/ethosu_npu_init.c @@ -17,7 +17,7 @@ #include "ethosu_npu_init.h" -#include "RTE_components.h" /* For CPU related defintiions */ +#include "RTE_Components.h" /* For CPU related defintiions */ #include "peripheral_memmap.h" /* Peripheral memory map definitions. */ #include "peripheral_irqs.h" /* IRQ numbers for this platform. */ #include "log_macros.h" /* Logging functions */ diff --git a/source/hal/platform/mps3/include/platform_drivers.h b/source/hal/platform/mps3/include/platform_drivers.h index da3af1e..a706ed4 100644 --- a/source/hal/platform/mps3/include/platform_drivers.h +++ b/source/hal/platform/mps3/include/platform_drivers.h @@ -22,7 +22,7 @@ /* Platform components */ #include "timer_mps3.h" /* Timer functions. */ -#include "RTE_components.h" /* For CPU related defintiions */ +#include "RTE_Components.h" /* For CPU related defintiions */ #include "glcd_mps3.h" /* LCD functions. */ /** Platform definitions. TODO: These should be removed. */ diff --git a/source/hal/platform/mps3/source/include/smm_mps3.h b/source/hal/platform/mps3/source/include/smm_mps3.h index 9e848e4..bc93fdf 100644 --- a/source/hal/platform/mps3/source/include/smm_mps3.h +++ b/source/hal/platform/mps3/source/include/smm_mps3.h @@ -19,7 +19,7 @@ #include "peripheral_memmap.h" /* Peripheral memory map definitions. */ -#include "RTE_components.h" +#include "RTE_Components.h" #if defined ( __CC_ARM ) #pragma anon_unions diff --git a/source/hal/platform/simple/include/platform_drivers.h b/source/hal/platform/simple/include/platform_drivers.h index 4140f76..c1a6c6a 100644 --- a/source/hal/platform/simple/include/platform_drivers.h +++ b/source/hal/platform/simple/include/platform_drivers.h @@ -23,7 +23,7 @@ /* Platform components */ #include "stubs/glcd.h" /* LCD stubs to support use cases that use LCD */ #include "timer_simple_platform.h" /* timer implementation */ -#include "RTE_components.h" /* For CPU related defintiions */ +#include "RTE_Components.h" /* For CPU related defintiions */ /** Platform definitions. TODO: These should be removed. */ #include "peripheral_memmap.h" /* Peripheral memory map definitions. */ diff --git a/source/hal/platform/simple/include/timer_simple_platform.h b/source/hal/platform/simple/include/timer_simple_platform.h index 5f3c26b..683a207 100644 --- a/source/hal/platform/simple/include/timer_simple_platform.h +++ b/source/hal/platform/simple/include/timer_simple_platform.h @@ -18,7 +18,7 @@ #define TIMER_SIMPLE_PLATFORM_H #include -#include "RTE_components.h" +#include "RTE_Components.h" /* Container for timestamp for simple platform. */ typedef struct _generic_time_counter { diff --git a/source/hal/platform/simple/source/timer_simple_platform.c b/source/hal/platform/simple/source/timer_simple_platform.c index b1f3194..f7917b0 100644 --- a/source/hal/platform/simple/source/timer_simple_platform.c +++ b/source/hal/platform/simple/source/timer_simple_platform.c @@ -17,7 +17,7 @@ #include "timer_simple_platform.h" #include "log_macros.h" /* Logging macros */ -#include "RTE_components.h" /* For CPU related defintiions */ +#include "RTE_Components.h" /* For CPU related defintiions */ #include -- cgit v1.2.1