#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # Copyright (c) 2024 Arm Limited. # # SPDX-License-Identifier: MIT # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to # deal in the Software without restriction, including without limitation the # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or # sell copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. """ Updates the Doxygen documentation pages with a table of operators supported by Compute Library. The script builds up a table in XML format internally containing the different operators and their respective supported compute backends, data types and layouts, and the equivalent operator in the Android Neural Networks API. The list of operators is pulled from the OperatorList.h header file and further implementation details are provided in the function headers for the backend-specific operator e.g., NEStridedSlice.h. Usage: python update_supported_ops.py """ import argparse import logging import re from enum import Enum from pathlib import Path class States(Enum): INIT = 0 DESCRIPTION = 1 DESCRIPTION_END = 2 IN_CLASS = 3 DATA_TYPE_START = 4 DATA_TYPE_END = 5 NN_OPERATOR = 6 NN_OPERATOR_END = 7 SKIP_OPERATOR = 8 DATA_LAYOUT_START = 9 DATA_LAYOUT_END = 10 class OperatorsTable: def __init__(self): self.project_dir = Path(__file__).resolve().parents[1] # ComputeLibrary directory self.xml = "" def generate_operator_list(self): operator_list_head_file = self.project_dir / "arm_compute" / "runtime" / "OperatorList.h" neon_file_name_prefix = str(self.project_dir / "arm_compute" / "runtime" / "NEON" / "functions" / "NE") cl_file_name_prefix = str(self.project_dir / "arm_compute" / "runtime" / "CL" / "functions" / "CL") logging.debug(operator_list_head_file) f = open(operator_list_head_file, 'r') # Iterates over the lines of the file state = States.INIT operator_desc = "" nn_op_list = [] for line in f: # /** ActivationLayer # * # * Description: # * Function to simulate an activation layer with the specified activation function. # * # * Equivalent Android NNAPI Op: # * ANEURALNETWORKS_ELU # * ANEURALNETWORKS_HARD_SWISH # * ANEURALNETWORKS_LOGISTIC # * ANEURALNETWORKS_RELU # * ANEURALNETWORKS_RELU1 # * ANEURALNETWORKS_RELU6 # * ANEURALNETWORKS_TANH # * # */ # Check for "/**" of the start of the operator r = re.search('^\s*/\*\*(.*)', line) if r and state == States.INIT: # Skip below ones if re.search('.*\(not ported\)', line): state = States.SKIP_OPERATOR continue if re.search('.*\(only CL\)', line): state = States.SKIP_OPERATOR continue if re.search('.*\(no CL\)', line): state = States.SKIP_OPERATOR continue if re.search('.*\(skip\)', line): state = States.SKIP_OPERATOR continue # Check" */" r = re.match('\s*\*/\s*$', line) if r and state == States.SKIP_OPERATOR: state = States.INIT continue # Check " *" r = re.match('\s*\*\s*$', line) if r and state == States.SKIP_OPERATOR: continue # Check non " *" lines r = re.search('^\s*\*(.*)', line) if r and state == States.SKIP_OPERATOR: continue # Check for "/**" of the start of the operator r = re.search('^\s*/\*\*(.*)', line) if r and state == States.INIT: tmp = r.groups()[0] class_name = tmp.strip() logging.debug(class_name) continue # Check whether "Description: " exists r = re.search('\s*\*\s*Description:\s*', line) if r and state == States.INIT: state = States.DESCRIPTION continue # Treat description ends with a blank line only with " *" r = re.match('\s*\*\s*$', line) if r and state == States.DESCRIPTION: logging.debug(operator_desc) state = States.DESCRIPTION_END continue # Find continuing class description in the following lines r = re.search('^\s*\*(.*)', line) if r and state == States.DESCRIPTION: tmp = r.groups()[0] operator_desc = operator_desc + ' ' + tmp.strip() continue # Check whether "Equivalent AndroidNN Op: " exists r = re.search('\s*\*\s*Equivalent Android NNAPI Op:\s*', line) if r and state == States.DESCRIPTION_END: state = States.NN_OPERATOR continue # Treat AndroidNN Op ends with a blank line only with " *" r = re.match('\s*\*\s*$', line) if r and state == States.NN_OPERATOR: logging.debug(nn_op_list) state = States.NN_OPERATOR_END # Check NE#class_name neon_file_name = neon_file_name_prefix + class_name + ".h" logging.debug(neon_file_name) # Check CL#class_name cl_file_name = cl_file_name_prefix + class_name + ".h" logging.debug(cl_file_name) # Check whether CL/Neon file exists if Path(neon_file_name).is_file() and Path(cl_file_name).is_file(): if neon_file_name.find("NEElementwiseOperations.h") != -1: logging.debug(neon_file_name) self.generate_operator_common_info(class_name, operator_desc, nn_op_list, "13") elif neon_file_name.find("NEElementwiseUnaryLayer.h") != -1: logging.debug(neon_file_name) self.generate_operator_common_info(class_name, operator_desc, nn_op_list, "8") else: self.generate_operator_common_info(class_name, operator_desc, nn_op_list, "2") self.generate_operator_info(neon_file_name) self.generate_operator_cl_begin() self.generate_operator_info(cl_file_name) else: if neon_file_name.find("NELogical.h") != -1: logging.debug(neon_file_name) self.generate_operator_common_info(class_name, operator_desc, nn_op_list, "3") else: self.generate_operator_common_info(class_name, operator_desc, nn_op_list, "1") if Path(neon_file_name).is_file(): self.generate_operator_info(neon_file_name) if Path(cl_file_name).is_file(): self.generate_operator_info(cl_file_name) continue # Find continuing AndroidNN Op in the following lines r = re.search('^\s*\*(.*)', line) if r and state == States.NN_OPERATOR: tmp = r.groups()[0] nn_op = tmp.strip() nn_op_list.append(nn_op) continue # Treat operator ends with a blank line only with " */" r = re.match('\s*\*/\s*$', line) if r and state == States.NN_OPERATOR_END: operator_desc = "" nn_op_list = [] state = States.INIT continue f.close() def generate_operator_info(self, file_name): logging.debug(file_name) f = open(file_name, 'r') # iterates over the lines of the file state = States.INIT data_type_list = [] data_layout_list = [] io_list = [] class_no = 0 for line in f: # Locate class definition by "class...: public IFunction", # There are also exceptions, which will need to support in later version r = re.match("\s*class\s+(\S+)\s*:\s*(public)*", line) if r and state == States.INIT: class_name = r.groups()[0] logging.debug("class name is %s" % (class_name)) state = States.IN_CLASS continue r = re.match("\s*\}\;", line) if r and state == States.IN_CLASS: state = States.INIT continue # * Valid data layouts: # * - All r = re.search('\s*\*\s*Valid data layouts:', line) if r and state == States.IN_CLASS: state = States.DATA_LAYOUT_START continue # Treat data configuration ends with a blank line only with " *" r = re.match('\s*\*\s*$', line) if r and state == States.DATA_LAYOUT_START: state = States.DATA_LAYOUT_END continue # Data layout continues r = re.search('\s*\*\s*\-\s*(.*)', line) if r and state == States.DATA_LAYOUT_START: tmp = r.groups()[0] tmp = tmp.strip() logging.debug(tmp) data_layout_list.append(tmp) # * Valid data type configurations: # * |src0 |dst | # * |:--------------|:--------------| # * |QASYMM8 |QASYMM8 | # * |QASYMM8_SIGNED |QASYMM8_SIGNED | # * |QSYMM16 |QSYMM16 | # * |F16 |F16 | # * |F32 |F32 | r = re.search('\s*\*\s*Valid data type configurations:\s*', line) if r and state == States.DATA_LAYOUT_END: state = States.DATA_TYPE_START logging.debug(line) continue # Treat data configuration ends with a blank line only with " *" r = re.match('\s*\*\s*$', line) if r and state == States.DATA_TYPE_START: logging.debug(class_name) logging.debug(data_layout_list) logging.debug(io_list) logging.debug(data_type_list) class_no = class_no + 1 if class_no > 1: logging.debug(class_no) self.generate_operator_cl_begin() self.generate_operator_dl_dt_info(class_name, data_layout_list, io_list, data_type_list) state = States.INIT data_type_list = [] data_layout_list = [] continue # Data type continues r = re.search('\s*\*(.*)', line) if r and state == States.DATA_TYPE_START: tmp = r.groups()[0] tmp = tmp.strip() if re.search('\|\:\-\-\-', tmp): # Skip the table split row "|:-----" continue else: tmp = tmp.strip() if re.search('.*(src|input|dst)', tmp): io_list = tmp.split('|') else: data_type = tmp.split('|') logging.debug(data_type) data_type_list.append(data_type) continue f.close() def generate_operator_cl_begin(self): self.xml += "\n" def generate_operator_common_info(self, class_name, operator_desc, nn_op_list, rowspan): tmp = "\n" # Store class name tmp += " " + class_name + "\n" tmp += " " + operator_desc + "\n" tmp += " \n" tmp += " \n" self.xml += tmp def generate_operator_dl_dt_info(self, class_name, data_layout, io_list, data_type_list): tmp = " " + class_name + "\n" # Store data layout info tmp += " \n" tmp += " \n" tmp += " \n" # Store data type table tmp += " \n" tmp += " " for io in io_list: # Make sure it's not empty string if len(io) != 0: tmp += "" for i in item: # Make sure it's not empty string if len(i) != 0: tmp += "
" tmp += io.strip() tmp += "\n" for item in data_type_list: tmp += "
" tmp += i.strip() tmp += "\n" tmp += "
\n" self.xml += tmp def generate_table_prefix(self): tmp = "\n" tmp += "\n" tmp += "\n" tmp += "
Function\n" tmp += " Description\n" tmp += " Equivalent Android NNAPI Op\n" tmp += " Backends\n" tmp += " Data Layouts\n" tmp += " Data Types\n" self.xml += tmp def generate_table_ending(self): self.xml += "
\n" def dump_xml(self): print(self.xml) def update_dox_file(self): operator_list_dox = self.project_dir / "docs" / "user_guide" / "operator_list.dox" with open(operator_list_dox, "r") as f: dox_content = f.read() # Check that there is only one non-indented table (This table should be the operator list) x = re.findall("\n", dox_content) y = re.findall("\n
", dox_content) if len(x) != 1 or len(y) != 1: raise RuntimeError("Invalid .dox file") repl_str = "\n" + self.xml[:-1] # Extra / removed "\n" characters needed to make up for search regex new_file = re.sub("\n(.|\n)*\n<\/table>", repl_str, dox_content) with open(operator_list_dox, "w") as f: f.write(new_file) print("Successfully updated operator_list.dox with the XML table of supported operators.") if __name__ == "__main__": parser = argparse.ArgumentParser( description="Updates the Compute Library documentation with a table of supported operators." ) parser.add_argument( "--dump_xml", type=bool, default=False, required=False, help="Dump the supported operators table XML to stdout", ) parser.add_argument( "--debug", type=bool, default=False, required=False, help="Enables logging, helpful for debugging. Default: False", ) args = parser.parse_args() if args.debug: logging.basicConfig(format="%(message)s", level=logging.DEBUG) table_xml = OperatorsTable() table_xml.generate_table_prefix() table_xml.generate_operator_list() table_xml.generate_table_ending() table_xml.update_dox_file() if args.dump_xml: table_xml.dump_xml()