diff options
Diffstat (limited to 'scripts/update_supported_ops.py')
| -rw-r--r-- | scripts/update_supported_ops.py | 414 |
1 files changed, 414 insertions, 0 deletions
diff --git a/scripts/update_supported_ops.py b/scripts/update_supported_ops.py new file mode 100644 index 0000000000..c39c8bc827 --- /dev/null +++ b/scripts/update_supported_ops.py @@ -0,0 +1,414 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (c) 2024 Arm Limited. +# +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +Updates the Doxygen documentation pages with a table of operators supported by Compute Library. + +The script builds up a table in XML format internally containing the different operators and their respective supported +compute backends, data types and layouts, and the equivalent operator in the Android Neural Networks API. The list of +operators is pulled from the OperatorList.h header file and further implementation details are provided in the function +headers for the backend-specific operator e.g., NEStridedSlice.h. + +Usage: + python update_supported_ops.py +""" + +import argparse +import logging +import re +from enum import Enum +from pathlib import Path + + +class States(Enum): + INIT = 0 + DESCRIPTION = 1 + DESCRIPTION_END = 2 + IN_CLASS = 3 + DATA_TYPE_START = 4 + DATA_TYPE_END = 5 + NN_OPERATOR = 6 + NN_OPERATOR_END = 7 + SKIP_OPERATOR = 8 + DATA_LAYOUT_START = 9 + DATA_LAYOUT_END = 10 + + +class OperatorsTable: + def __init__(self): + self.project_dir = Path(__file__).resolve().parents[1] # ComputeLibrary directory + self.xml = "" + + def generate_operator_list(self): + operator_list_head_file = self.project_dir / "arm_compute" / "runtime" / "OperatorList.h" + neon_file_name_prefix = str(self.project_dir / "arm_compute" / "runtime" / "NEON" / "functions" / "NE") + cl_file_name_prefix = str(self.project_dir / "arm_compute" / "runtime" / "CL" / "functions" / "CL") + + logging.debug(operator_list_head_file) + + f = open(operator_list_head_file, 'r') + # Iterates over the lines of the file + state = States.INIT + operator_desc = "" + nn_op_list = [] + for line in f: + # /** ActivationLayer + # * + # * Description: + # * Function to simulate an activation layer with the specified activation function. + # * + # * Equivalent Android NNAPI Op: + # * ANEURALNETWORKS_ELU + # * ANEURALNETWORKS_HARD_SWISH + # * ANEURALNETWORKS_LOGISTIC + # * ANEURALNETWORKS_RELU + # * ANEURALNETWORKS_RELU1 + # * ANEURALNETWORKS_RELU6 + # * ANEURALNETWORKS_TANH + # * + # */ + # Check for "/**" of the start of the operator + r = re.search('^\s*/\*\*(.*)', line) + if r and state == States.INIT: + # Skip below ones + if re.search('.*\(not ported\)', line): + state = States.SKIP_OPERATOR + continue + if re.search('.*\(only CL\)', line): + state = States.SKIP_OPERATOR + continue + if re.search('.*\(no CL\)', line): + state = States.SKIP_OPERATOR + continue + if re.search('.*\(skip\)', line): + state = States.SKIP_OPERATOR + continue + # Check" */" + r = re.match('\s*\*/\s*$', line) + if r and state == States.SKIP_OPERATOR: + state = States.INIT + continue + # Check " *" + r = re.match('\s*\*\s*$', line) + if r and state == States.SKIP_OPERATOR: + continue + # Check non " *" lines + r = re.search('^\s*\*(.*)', line) + if r and state == States.SKIP_OPERATOR: + continue + + # Check for "/**" of the start of the operator + r = re.search('^\s*/\*\*(.*)', line) + if r and state == States.INIT: + tmp = r.groups()[0] + class_name = tmp.strip() + logging.debug(class_name) + continue + + # Check whether "Description: " exists + r = re.search('\s*\*\s*Description:\s*', line) + if r and state == States.INIT: + state = States.DESCRIPTION + continue + # Treat description ends with a blank line only with " *" + r = re.match('\s*\*\s*$', line) + if r and state == States.DESCRIPTION: + logging.debug(operator_desc) + state = States.DESCRIPTION_END + continue + # Find continuing class description in the following lines + r = re.search('^\s*\*(.*)', line) + if r and state == States.DESCRIPTION: + tmp = r.groups()[0] + operator_desc = operator_desc + ' ' + tmp.strip() + continue + + # Check whether "Equivalent AndroidNN Op: " exists + r = re.search('\s*\*\s*Equivalent Android NNAPI Op:\s*', line) + if r and state == States.DESCRIPTION_END: + state = States.NN_OPERATOR + continue + # Treat AndroidNN Op ends with a blank line only with " *" + r = re.match('\s*\*\s*$', line) + if r and state == States.NN_OPERATOR: + logging.debug(nn_op_list) + state = States.NN_OPERATOR_END + # Check NE#class_name + neon_file_name = neon_file_name_prefix + class_name + ".h" + logging.debug(neon_file_name) + # Check CL#class_name + cl_file_name = cl_file_name_prefix + class_name + ".h" + logging.debug(cl_file_name) + # Check whether CL/Neon file exists + if Path(neon_file_name).is_file() and Path(cl_file_name).is_file(): + if neon_file_name.find("NEElementwiseOperations.h") != -1: + logging.debug(neon_file_name) + self.generate_operator_common_info(class_name, operator_desc, nn_op_list, "13") + elif neon_file_name.find("NEElementwiseUnaryLayer.h") != -1: + logging.debug(neon_file_name) + self.generate_operator_common_info(class_name, operator_desc, nn_op_list, "8") + else: + self.generate_operator_common_info(class_name, operator_desc, nn_op_list, "2") + self.generate_operator_info(neon_file_name) + self.generate_operator_cl_begin() + self.generate_operator_info(cl_file_name) + else: + if neon_file_name.find("NELogical.h") != -1: + logging.debug(neon_file_name) + self.generate_operator_common_info(class_name, operator_desc, nn_op_list, "3") + else: + self.generate_operator_common_info(class_name, operator_desc, nn_op_list, "1") + if Path(neon_file_name).is_file(): + self.generate_operator_info(neon_file_name) + if Path(cl_file_name).is_file(): + self.generate_operator_info(cl_file_name) + continue + + # Find continuing AndroidNN Op in the following lines + r = re.search('^\s*\*(.*)', line) + if r and state == States.NN_OPERATOR: + tmp = r.groups()[0] + nn_op = tmp.strip() + nn_op_list.append(nn_op) + continue + + # Treat operator ends with a blank line only with " */" + r = re.match('\s*\*/\s*$', line) + if r and state == States.NN_OPERATOR_END: + operator_desc = "" + nn_op_list = [] + state = States.INIT + continue + f.close() + + def generate_operator_info(self, file_name): + logging.debug(file_name) + f = open(file_name, 'r') + # iterates over the lines of the file + state = States.INIT + data_type_list = [] + data_layout_list = [] + io_list = [] + class_no = 0 + for line in f: + # Locate class definition by "class...: public IFunction", + # There are also exceptions, which will need to support in later version + r = re.match("\s*class\s+(\S+)\s*:\s*(public)*", line) + if r and state == States.INIT: + class_name = r.groups()[0] + logging.debug("class name is %s" % (class_name)) + state = States.IN_CLASS + continue + + r = re.match("\s*\}\;", line) + if r and state == States.IN_CLASS: + state = States.INIT + continue + + # * Valid data layouts: + # * - All + r = re.search('\s*\*\s*Valid data layouts:', line) + if r and state == States.IN_CLASS: + state = States.DATA_LAYOUT_START + continue + # Treat data configuration ends with a blank line only with " *" + r = re.match('\s*\*\s*$', line) + if r and state == States.DATA_LAYOUT_START: + state = States.DATA_LAYOUT_END + continue + # Data layout continues + r = re.search('\s*\*\s*\-\s*(.*)', line) + if r and state == States.DATA_LAYOUT_START: + tmp = r.groups()[0] + tmp = tmp.strip() + logging.debug(tmp) + data_layout_list.append(tmp) + + # * Valid data type configurations: + # * |src0 |dst | + # * |:--------------|:--------------| + # * |QASYMM8 |QASYMM8 | + # * |QASYMM8_SIGNED |QASYMM8_SIGNED | + # * |QSYMM16 |QSYMM16 | + # * |F16 |F16 | + # * |F32 |F32 | + r = re.search('\s*\*\s*Valid data type configurations:\s*', line) + if r and state == States.DATA_LAYOUT_END: + state = States.DATA_TYPE_START + logging.debug(line) + continue + # Treat data configuration ends with a blank line only with " *" + r = re.match('\s*\*\s*$', line) + if r and state == States.DATA_TYPE_START: + logging.debug(class_name) + logging.debug(data_layout_list) + logging.debug(io_list) + logging.debug(data_type_list) + class_no = class_no + 1 + if class_no > 1: + logging.debug(class_no) + self.generate_operator_cl_begin() + self.generate_operator_dl_dt_info(class_name, data_layout_list, io_list, data_type_list) + state = States.INIT + data_type_list = [] + data_layout_list = [] + continue + # Data type continues + r = re.search('\s*\*(.*)', line) + if r and state == States.DATA_TYPE_START: + tmp = r.groups()[0] + tmp = tmp.strip() + if re.search('\|\:\-\-\-', tmp): + # Skip the table split row "|:-----" + continue + else: + tmp = tmp.strip() + if re.search('.*(src|input|dst)', tmp): + io_list = tmp.split('|') + else: + data_type = tmp.split('|') + logging.debug(data_type) + data_type_list.append(data_type) + continue + + f.close() + + def generate_operator_cl_begin(self): + self.xml += "<tr>\n" + + def generate_operator_common_info(self, class_name, operator_desc, nn_op_list, rowspan): + tmp = "<tr>\n" + # Store class name + tmp += " <td rowspan=\"" + rowspan + "\">" + class_name + "\n" + tmp += " <td rowspan=\"" + rowspan + "\" style=\"width:200px;\">" + operator_desc + "\n" + tmp += " <td rowspan=\"" + rowspan + "\">\n" + tmp += " <ul>\n" + for item in nn_op_list: + tmp += " <li>" + tmp += item.strip() + tmp += "\n" + tmp += " </ul>\n" + self.xml += tmp + + def generate_operator_dl_dt_info(self, class_name, data_layout, io_list, data_type_list): + tmp = " <td>" + class_name + "\n" + # Store data layout info + tmp += " <td>\n" + tmp += " <ul>\n" + for item in data_layout: + tmp += " <li>" + tmp += item.strip() + tmp += "\n" + tmp += " </ul>\n" + tmp += " <td>\n" + # Store data type table + tmp += " <table>\n" + tmp += " <tr>" + for io in io_list: + # Make sure it's not empty string + if len(io) != 0: + tmp += "<th>" + tmp += io.strip() + tmp += "\n" + for item in data_type_list: + tmp += " <tr>" + for i in item: + # Make sure it's not empty string + if len(i) != 0: + tmp += "<td>" + tmp += i.strip() + tmp += "\n" + tmp += " </table>\n" + self.xml += tmp + + def generate_table_prefix(self): + tmp = "<table>\n" + tmp += "<caption id=\"multi_row\"></caption>\n" + tmp += "<tr>\n" + tmp += " <th>Function\n" + tmp += " <th>Description\n" + tmp += " <th>Equivalent Android NNAPI Op\n" + tmp += " <th>Backends\n" + tmp += " <th>Data Layouts\n" + tmp += " <th>Data Types\n" + self.xml += tmp + + def generate_table_ending(self): + self.xml += "</table>\n" + + def dump_xml(self): + print(self.xml) + + def update_dox_file(self): + operator_list_dox = self.project_dir / "docs" / "user_guide" / "operator_list.dox" + + with open(operator_list_dox, "r") as f: + dox_content = f.read() + + # Check that there is only one non-indented table (This table should be the operator list) + x = re.findall("\n<table>", dox_content) + y = re.findall("\n</table>", dox_content) + if len(x) != 1 or len(y) != 1: + raise RuntimeError("Invalid .dox file") + + repl_str = "\n" + self.xml[:-1] # Extra / removed "\n" characters needed to make up for search regex + new_file = re.sub("\n<table>(.|\n)*\n<\/table>", repl_str, dox_content) + + with open(operator_list_dox, "w") as f: + f.write(new_file) + print("Successfully updated operator_list.dox with the XML table of supported operators.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Updates the Compute Library documentation with a table of supported operators." + ) + parser.add_argument( + "--dump_xml", + type=bool, + default=False, + required=False, + help="Dump the supported operators table XML to stdout", + ) + parser.add_argument( + "--debug", + type=bool, + default=False, + required=False, + help="Enables logging, helpful for debugging. Default: False", + ) + args = parser.parse_args() + + if args.debug: + logging.basicConfig(format="%(message)s", level=logging.DEBUG) + + table_xml = OperatorsTable() + table_xml.generate_table_prefix() + table_xml.generate_operator_list() + table_xml.generate_table_ending() + table_xml.update_dox_file() + + if args.dump_xml: + table_xml.dump_xml() |