Add desc.json schema validation with compliance/data_gen extensions

Rename scripts utlities to tosa_* such as tosa_json2numpy

Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
Change-Id: Ie8d584e2afb189fb74cf96b39590c7c27444ba14

5 files changed, 437 insertions, 0 deletions

diff --git a/scripts/schemavalidation/__init__.py b/scripts/schemavalidation/__init__.py
new file mode 100644
index 0000000..5f57249
--- /dev/null
+++ b/scripts/schemavalidation/__init__.py
@@ -0,0 +1,3 @@
+"""Namespace."""
+# Copyright (c) 2023 Arm Limited.
+# SPDX-License-Identifier: Apache-2.0
diff --git a/scripts/schemavalidation/compliance-config.schema.json b/scripts/schemavalidation/compliance-config.schema.json
new file mode 100644
index 0000000..677a2a8
--- /dev/null
+++ b/scripts/schemavalidation/compliance-config.schema.json
@@ -0,0 +1,79 @@
+ {
+    "$comment": "Copyright (c) 2023, ARM Limited.",
+    "$comment": "SPDX-License-Identifier: Apache-2.0",
+    "$id": "compliance-config.schema.json",
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "title": "compliance-config",
+    "description": "Compliance configuration for verifying TOSA tensor data outputs from a test",
+    "type": "object",
+    "properties": {
+        "version": {
+            "description": "version string X.Y of this config data",
+            "type": "string"
+        },
+        "tensors": {
+            "type": "object",
+            "description": "dictionary of output tensors - keys are the tosa network names",
+            "patternProperties":
+            {
+                "^.*$":
+                {
+                    "description": "per named tensor config for compliance checking",
+                    "type": "object",
+                    "properties": {
+                        "mode": {
+                            "description": "verifier mode EXACT, DOT_PRODUCT, ULP, REDUCE_PRODUCT, or FP_SPECIAL",
+                            "type": "string"
+                        },
+                        "ulp_info": {
+                            "description": "info required for the ULP mode",
+                            "type": "object",
+                            "properties":
+                            {
+                                "ulp": {
+                                    "description": "ulp range limit - such as 1 or 0.5",
+                                    "type": "string"
+                                }
+                            },
+                            "required": [ "ulp" ]
+                        },
+                        "dot_product_info": {
+                            "description": "info required for the DOT_PRODUCT mode",
+                            "type": "object",
+                            "properties":
+                            {
+                                "s": {
+                                    "description": "test set 0 to 5 or -1 for other data",
+                                    "type": "integer"
+                                },
+                                "ks": {
+                                    "description": "kernel size for this dot product operation",
+                                    "type": "integer"
+                                },
+                                "data_type": {
+                                    "description": "tensor data type, such as: BOOL, INT16, FP32",
+                                    "type": "string"
+                                }
+                            },
+                            "additionalProperties": false,
+                            "required": [
+                                "s",
+                                "ks",
+                                "data_type"
+                            ]
+                        }
+                    },
+                    "additionalProperties": false,
+                    "required": [
+                        "mode"
+                    ]
+                }
+            }
+        }
+    },
+    "additionalProperties": false,
+    "required": [
+        "version",
+        "tensors"
+    ]
+}
diff --git a/scripts/schemavalidation/datagen-config.schema.json b/scripts/schemavalidation/datagen-config.schema.json
new file mode 100644
index 0000000..01f9fad
--- /dev/null
+++ b/scripts/schemavalidation/datagen-config.schema.json
@@ -0,0 +1,165 @@
+ {
+    "$comment": "Copyright (c) 2023, ARM Limited.",
+    "$comment": "SPDX-License-Identifier: Apache-2.0",
+    "$id": "datagen-config.schema.json",
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "title": "datagen-config",
+    "description": "Data generator configuration for generating TOSA tensor data",
+    "type": "object",
+    "properties": {
+        "version": {
+            "description": "version string X.Y of this config data",
+            "type": "string"
+        },
+        "tensors": {
+            "type": "object",
+            "description": "dictionary of input tensors - keys are the tosa network names",
+            "patternProperties":
+            {
+                "^.*$":
+                {
+                    "description": "per named tensor config for data generation",
+                    "type": "object",
+                    "properties": {
+                        "generator": {
+                            "description": "data generator name - PSEUDO_RANDOM, DOT_PRODUCT or OP_SPECIFIC",
+                            "type": "string"
+                        },
+                        "data_type": {
+                            "description": "tensor data type, such as: BOOL, INT16, FP32",
+                            "type": "string"
+                        },
+                        "input_type": {
+                            "description": "input is variable or constant",
+                            "type": "string"
+                        },
+                        "shape": {
+                            "description": "tensor shape array (rank 0 should be [])",
+                            "type": "array",
+                            "items": {
+                                "description": "shape dimension",
+                                "type": "integer",
+                                "minimum": 1
+                            }
+                        },
+                        "op": {
+                            "description": "operator name the tensor is initially used with",
+                            "type": "string"
+                        },
+                        "input_pos": {
+                            "description": "tensor input position for the operator (starts at 0)",
+                            "type": "integer",
+                            "minimum": 0
+                        },
+                        "pseudo_random_info": {
+                            "description": "info required for the PSEUDO_RANDOM generator",
+                            "type": "object",
+                            "properties":
+                            {
+                                "rng_seed": {
+                                    "description": "random number generator seed",
+                                    "type": "integer"
+                                },
+                                "range": {
+                                    "type": "array",
+                                    "description": "range of random numbers from low to high inclusively",
+                                    "minItems": 2,
+                                    "maxItems": 2,
+                                    "items": {
+                                        "description": "[low value, high value] as strings to allow ints and floats",
+                                        "type": "string"
+                                    }
+                                }
+                            },
+                            "additionalProperties": false,
+                            "required": [ "rng_seed" ]
+                        },
+                        "dot_product_info": {
+                            "description": "info required for the DOT_PRODUCT generator",
+                            "type": "object",
+                            "properties":
+                            {
+                                "s": {
+                                    "description": "test set 0 to 5 or -1 for other data",
+                                    "type": "integer"
+                                },
+                                "ks": {
+                                    "description": "kernel size for this dot product operation",
+                                    "type": "integer"
+                                },
+                                "acc_type": {
+                                    "description": "operator accumulator type (like tensor data_type)",
+                                    "type": "string"
+                                },
+                                "kernel": {
+                                    "type": "array",
+                                    "description": "kernel x, y sizes (for avg_pool2d)",
+                                    "minItems": 2,
+                                    "maxItems": 2,
+                                    "items": {
+                                        "description": "kernel dimension",
+                                        "type": "integer",
+                                        "minimum": 1
+                                    }
+                                },
+                                "axis": {
+                                    "description": "axis dimension (for reduce_sum)",
+                                    "type": "integer",
+                                    "minimum": 0
+                                }
+                            },
+                            "additionalProperties": false,
+                            "required": [
+                                "s",
+                                "ks",
+                                "acc_type"
+                            ]
+                        },
+                        "op_specific_info": {
+                            "description": "info required for the OP_SPECIFIC generator",
+                            "type": "object",
+                            "properties":
+                            {
+                                "sub_generator": {
+                                    "description": "sub generator type for this op - FULL, SPECIAL or BOUNDARY",
+                                    "type": "string"
+                                },
+                                "offset": {
+                                    "description": "starting offset within the test data",
+                                    "type": "integer",
+                                    "minimum": 0
+                                },
+                                "attributes": {
+                                    "description": "attribute data from the op needed to compute the data",
+                                    "type": "object",
+                                    "properties": {
+                                        "TBD": {
+                                            "description": "Probably needed for RESCALE and MUL",
+                                            "type": "string"
+                                        }
+                                    }
+                                }
+                            },
+                            "additionalProperties": false,
+                            "required": [ "sub_generator" ]
+                        }
+                    },
+                    "additionalProperties": false,
+                    "required": [
+                        "generator",
+                        "data_type",
+                        "input_type",
+                        "shape",
+                        "op",
+                        "input_pos"
+                    ]
+                }
+            }
+        }
+    },
+    "additionalProperties": false,
+    "required": [
+        "version",
+        "tensors"
+    ]
+}
diff --git a/scripts/schemavalidation/desc.schema.json b/scripts/schemavalidation/desc.schema.json
new file mode 100644
index 0000000..226fc68
--- /dev/null
+++ b/scripts/schemavalidation/desc.schema.json
@@ -0,0 +1,111 @@
+ {
+    "$comment": "Copyright (c) 2023, ARM Limited.",
+    "$comment": "SPDX-License-Identifier: Apache-2.0",
+    "$id": "desc.schema.json",
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "title": "desc",
+    "description": "Test description",
+    "type": "object",
+    "properties": {
+        "tosa_file": {
+            "description": "tosa flatbuffer file",
+            "type": "string"
+        },
+        "ifm_name": {
+            "description": "list of input tensors",
+            "type": "array",
+            "items": {
+                "description": "tensor name",
+                "type": "string"
+            }
+        },
+        "ifm_file": {
+            "description": "list of input files, one for each tensor in ifm_name",
+            "type": "array",
+            "items": {
+                "description": "file name",
+                "type": "string"
+            }
+        },
+        "ofm_name": {
+            "description": "list of output tensors",
+            "type": "array",
+            "minItems": 1,
+            "items": {
+                "description": "tensor name",
+                "type": "string"
+            }
+        },
+        "ofm_file": {
+            "description": "list of output files, one for each tensor in ofm_name",
+            "type": "array",
+            "minItems": 1,
+            "items": {
+                "description": "file name",
+                "type": "string"
+            }
+        },
+        "expected_failure": {
+            "description": "if the test is expected to fail - used for test filtering",
+            "type": "boolean"
+        },
+        "expected_failure_desc": {
+            "description": "summary of expected failure test",
+            "type": "string"
+        },
+        "expected_return_code": {
+            "description": "DEPRECATED (unused): expected return code from ref-model",
+            "type": "integer"
+        },
+        "expected_result_file": {
+            "description": "DEPRECATED: list of expected result files, one for each tensor in ofm_name",
+            "type": "array",
+            "minItems": 1,
+            "items": {
+                "description": "file name",
+                "type": "string"
+            }
+        },
+        "profile": {
+            "description": "list of profiles this test is valid for",
+            "type": "array",
+            "minItems": 1,
+            "items": {
+                "description": "profile name - tosa_bi or tosa_mi",
+                "type": "string"
+            }
+        },
+        "tags": {
+            "description": "list of string tags that can be used for test filtering",
+            "type": "array",
+            "items": {
+                "description": "test tag for filtering",
+                "type": "string"
+            }
+        },
+        "meta": {
+            "description": "meta-data for test libraries",
+            "type": "object",
+            "properties":
+            {
+                "data_gen": {
+                    "description": "data generation meta-data",
+                    "$ref": "datagen-config.schema.json"
+                },
+                "compliance": {
+                    "description": "compliance meta-data",
+                    "$ref": "compliance-config.schema.json"
+                }
+            },
+            "additionalProperties": false
+        }
+    },
+    "additionalProperties": false,
+    "required": [
+        "tosa_file",
+        "ifm_name",
+        "ifm_file",
+        "ofm_name",
+        "ofm_file"
+    ]
+}
diff --git a/scripts/schemavalidation/schemavalidation.py b/scripts/schemavalidation/schemavalidation.py
new file mode 100644
index 0000000..2848edf
--- /dev/null
+++ b/scripts/schemavalidation/schemavalidation.py
@@ -0,0 +1,79 @@
+# Copyright (c) 2023, ARM Limited.
+# SPDX-License-Identifier: Apache-2.0
+"""Validates desc.json against its JSON schema."""
+import json
+from pathlib import Path
+
+from jsonschema import validate
+from referencing import Registry
+from referencing import Resource
+
+TD_SCHEMA_FULL = "full"
+TD_SCHEMA_DATA_GEN = "data_gen"
+TD_SCHEMA_COMPLIANCE = "compliance"
+
+# Default file info
+SCRIPT = Path(__file__).absolute()
+TD_SCHEMAS = {
+    TD_SCHEMA_DATA_GEN: "datagen-config.schema.json",
+    TD_SCHEMA_COMPLIANCE: "compliance-config.schema.json",
+    TD_SCHEMA_FULL: "desc.schema.json",
+}
+
+
+class TestDescSchemaValidator:
+    def __init__(self):
+        """Initialize by loading all the schemas and setting up a registry."""
+        self.registry = Registry()
+        self.schemas = {}
+        for key, name in TD_SCHEMAS.items():
+            schema_path = SCRIPT.parent / name
+            with schema_path.open("r") as fd:
+                schema = json.load(fd)
+            self.schemas[key] = schema
+            if key != TD_SCHEMA_FULL:
+                resource = Resource.from_contents(schema)
+                self.registry = self.registry.with_resource(uri=name, resource=resource)
+
+    def validate_config(self, config, schema_type=TD_SCHEMA_FULL):
+        """Validate the whole (or partial) config versus the relevant schema."""
+        validate(config, self.schemas[schema_type], registry=self.registry)
+
+
+def main(argv=None):
+    """Command line interface for the schema validation."""
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "path", type=Path, help="the path to the test directory to validate"
+    )
+    args = parser.parse_args(argv)
+    test_path = args.path
+
+    if not test_path.is_dir():
+        print(f"ERROR: Invalid directory - {test_path}")
+        return 2
+
+    test_desc_path = test_path / "desc.json"
+
+    if not test_desc_path.is_file():
+        print(f"ERROR: No test description found: {test_desc_path}")
+        return 2
+
+    # Load the JSON desc.json
+    try:
+        with test_desc_path.open("r") as fd:
+            test_desc = json.load(fd)
+    except Exception as e:
+        print(f"ERROR: Loading {test_desc_path} - {repr(e)}")
+        return 2
+
+    sv = TestDescSchemaValidator()
+    sv.validate_config(test_desc, TD_SCHEMA_FULL)
+
+    return 0
+
+
+if __name__ == "__main__":
+    exit(main())