From 46c9477ded912f26ddf0a761c728d23f7d616004 Mon Sep 17 00:00:00 2001 From: wilisa01 Date: Wed, 8 Feb 2023 09:56:14 +0000 Subject: MLBEDSW-7211: Convert fixup_asymmetric_weights to supported ops check Changed default behaviour to place int8 ops with asymmetric quantization on cpu, and added an option to force symmetric quantization Change-Id: Ib9b717aaf61eae78833254ca3dfa745f4f253dc6 Signed-off-by: wilisa01 --- ethosu/vela/compiler_driver.py | 8 ++++-- ethosu/vela/graph_optimiser.py | 6 ++--- ethosu/vela/tflite_graph_optimiser.py | 49 +++++++++++++++++++++++++++-------- ethosu/vela/vela.py | 8 +++++- 4 files changed, 54 insertions(+), 17 deletions(-) (limited to 'ethosu/vela') diff --git a/ethosu/vela/compiler_driver.py b/ethosu/vela/compiler_driver.py index 439b954a..d2892096 100644 --- a/ethosu/vela/compiler_driver.py +++ b/ethosu/vela/compiler_driver.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2020-2022 Arm Limited and/or its affiliates +# SPDX-FileCopyrightText: Copyright 2020-2023 Arm Limited and/or its affiliates # # SPDX-License-Identifier: Apache-2.0 # @@ -65,6 +65,7 @@ class CompilerOptions: show_cpu_operations=False, tensor_allocator=TensorAllocator.Greedy, timing=False, + force_symmetric_int_weights=False, output_dir="outputs", cpu_tensor_alignment=Tensor.AllocationQuantum, hillclimb_max_iterations=None, @@ -84,6 +85,7 @@ class CompilerOptions: self.show_cpu_operations = show_cpu_operations self.tensor_allocator = tensor_allocator self.timing = timing + self.force_symmetric_int_weights = force_symmetric_int_weights self.output_dir = output_dir self.cpu_tensor_alignment = cpu_tensor_alignment self.hillclimb_max_iterations = hillclimb_max_iterations @@ -157,7 +159,9 @@ def compiler_driver(nng, arch, options, scheduler_options, network_type, output_ for sg in nng.subgraphs: visit_graph_post_order(sg.output_tensors, arch, [], [_record_operator]) - nng = graph_optimiser.optimise_graph(nng, arch, network_type, options.verbose_graph) + nng = graph_optimiser.optimise_graph( + nng, arch, network_type, options.verbose_graph, options.force_symmetric_int_weights + ) assert verify_graph_health(nng) if options.verbose_quantization: diff --git a/ethosu/vela/graph_optimiser.py b/ethosu/vela/graph_optimiser.py index 4505cf58..984a0167 100644 --- a/ethosu/vela/graph_optimiser.py +++ b/ethosu/vela/graph_optimiser.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2020-2021 Arm Limited and/or its affiliates +# SPDX-FileCopyrightText: Copyright 2020-2021, 2023 Arm Limited and/or its affiliates # # SPDX-License-Identifier: Apache-2.0 # @@ -25,13 +25,13 @@ from .tflite_graph_optimiser import tflite_optimise_graph from .tosa_graph_optimiser import tosa_optimise_graph -def optimise_graph(nng, arch, network_type, verbose_graph=False): +def optimise_graph(nng, arch, network_type, verbose_graph=False, force_symmetric_int_weights=False): if verbose_graph: nng.print_graph("Before Graph Optimization") if network_type == NetworkType.TFLite: # TensorFlow Lite graph optimization - nng = tflite_optimise_graph(nng, arch) + nng = tflite_optimise_graph(nng, arch, force_symmetric_int_weights) else: # TOSA graph optimization nng = tosa_optimise_graph(nng, arch) diff --git a/ethosu/vela/tflite_graph_optimiser.py b/ethosu/vela/tflite_graph_optimiser.py index 2026892e..3a49309d 100644 --- a/ethosu/vela/tflite_graph_optimiser.py +++ b/ethosu/vela/tflite_graph_optimiser.py @@ -1583,16 +1583,42 @@ def fixup_bias_tensors(op, arch, nng, dtype=None): return op -def fixup_asymmetric_weights(op, arch, nng): - if op.run_on_npu and (op.type.is_conv2d_op() or op.type.is_depthwise_conv2d_op()): - if op.ifm.dtype == DataType.int8: +def detect_asymmetric_weights(op): + # Check all ops (cpu and npu) + if op.type.is_conv2d_op() or op.type.is_depthwise_conv2d_op(): + if op.ifm.dtype in (DataType.int8, DataType.int16): if not np.all(op.weights.quantization.zero_point == 0): - print(f"Warning: {op.type} '{op.name}' has asymmetric weights, zero points have been adjusted.") - op.weights.quantization.zero_point *= 0 + print(f"Warning: Op {op.type} '{op.name}' has asymmetric weights.", end=" ") + return True + return False + + +def fixup_asymmetric_weights(op, arch, nng): + if detect_asymmetric_weights(op): + if op.run_on_npu: + print("Zero points have been adjusted.") + op.weights.quantization.zero_point *= 0 + return op + +def check_asymmetric_weights(op, arch, nng): + # This function can modify the run_on_npu flag which causes an operator to be placed on the CPU. It is usually only + # set by the supported operator checks. Therefore, it should be run immediately after those checks to avoid the + # possibility of other graph optimiser functions modify the operator (that is later run on the CPU) + if detect_asymmetric_weights(op): + if op.run_on_npu: + print("To run the operator on Ethos-U use the option --force-symmetric-int-weights") + op.run_on_npu = False return op +def fixup_or_check_asymmetric_weights(force_symmetric_int_weights): + if force_symmetric_int_weights: + return fixup_asymmetric_weights + else: + return check_asymmetric_weights + + def convert_mean_to_depthwise_conv_or_avgpool(op, arch, nng): if op.type == Op.Mean and op.run_on_npu: inp, axis = op.inputs @@ -1871,9 +1897,13 @@ def supported_operator_check(op, arch, nng): return op -def tflite_optimise_graph(nng, arch): +def tflite_optimise_graph(nng, arch, force_symmetric_int_weights): # Compile time static optimisations - optimisation_list = [optimise_quantize, convert_shape_op_to_constant_tensor] + optimisation_list = [ + optimise_quantize, + convert_shape_op_to_constant_tensor, + fixup_or_check_asymmetric_weights(force_symmetric_int_weights), + ] for idx, sg in enumerate(nng.subgraphs): nng.subgraphs[idx] = rewrite_graph.rewrite_graph_pre_order( @@ -1886,10 +1916,7 @@ def tflite_optimise_graph(nng, arch): ) # Pre-processing step - pre_process_list = [ - supported_operator_check, - set_ifm_ofm_op_shapes, - ] + pre_process_list = [supported_operator_check, set_ifm_ofm_op_shapes] for idx, sg in enumerate(nng.subgraphs): nng.subgraphs[idx] = rewrite_graph.rewrite_graph_pre_order( diff --git a/ethosu/vela/vela.py b/ethosu/vela/vela.py index b5dfd80c..c44c7894 100644 --- a/ethosu/vela/vela.py +++ b/ethosu/vela/vela.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2020-2022 Arm Limited and/or its affiliates +# SPDX-FileCopyrightText: Copyright 2020-2023 Arm Limited and/or its affiliates # # SPDX-License-Identifier: Apache-2.0 # @@ -376,6 +376,11 @@ def main(args=None): "--show-cpu-operations", action="store_true", help="Show the operations that fall back to the CPU" ) parser.add_argument("--timing", action="store_true", help="Time the compiler doing operations") + parser.add_argument( + "--force-symmetric-int-weights", + action="store_true", + help="Forces all zero points to 0 for signed integer weights", + ) parser.add_argument( "--accelerator-config", type=str, @@ -553,6 +558,7 @@ def main(args=None): show_cpu_operations=args.show_cpu_operations, tensor_allocator=args.tensor_allocator, timing=args.timing, + force_symmetric_int_weights=args.force_symmetric_int_weights, output_dir=args.output_dir, cpu_tensor_alignment=args.cpu_tensor_alignment, hillclimb_max_iterations=args.hillclimb_max_iterations, -- cgit v1.2.1