Add desc.json schema validation with compliance/data_gen extensions

Rename scripts utlities to tosa_* such as tosa_json2numpy

Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
Change-Id: Ie8d584e2afb189fb74cf96b39590c7c27444ba14

8 files changed, 557 insertions, 9 deletions

diff --git a/README.md b/README.md
index 063b231..4b74b6e 100644
--- a/README.md
+++ b/README.md
@@ -489,12 +489,13 @@ tosa_verif_conformance_generator        \
 
 Included in this repository are some support utilities used by the test runner:
 
-* `json2numpy` - converts from JSON format to numpy array or the reverse operation.
-* `json2fbbin` - converts from JSON flatbuffer format to flatbuffer
+* `tosa_json2numpy` - converts from JSON format to numpy array or the reverse operation.
+* `tosa_json2fbbin` - converts from JSON flatbuffer format to flatbuffer
     binary format or the reverse operation. This is dependent on the FlatBuffers
     command `flatc` - see the section on the FlatBuffers compiler below.
 * `tosa_verif_result_check` - compares two results files.
-* `convert2conformance` - converts a unit test into a conformance suitable test.
+* `tosa_convert2conformance` - converts a unit test into a conformance suitable test.
+* `tosa_schemavalidation` - validates a test desc.json against the schema.
 
 Please see the respective `--help` of each utility for more information on using
 them standalone.
@@ -502,8 +503,8 @@ them standalone.
 ### FlatBuffers compiler
 
 The FlatBuffers compiler tool (`flatc`) is only needed if you want to use
-json2fbbin to convert the TOSA flatbuffer binary test files (.tosa) to JSON
-or from JSON to binary.
+tosa_json2fbbin to convert the TOSA flatbuffer binary test files (.tosa) to
+JSON or from JSON to binary.
 It is best to use the flatbuffer version that comes with the reference model.
 After following the reference model compilation instructions, you can build
 the FlatBuffers tool using:
diff --git a/scripts/schemavalidation/__init__.py b/scripts/schemavalidation/__init__.py
new file mode 100644
index 0000000..5f57249
--- /dev/null
+++ b/scripts/schemavalidation/__init__.py
@@ -0,0 +1,3 @@
+"""Namespace."""
+# Copyright (c) 2023 Arm Limited.
+# SPDX-License-Identifier: Apache-2.0
diff --git a/scripts/schemavalidation/compliance-config.schema.json b/scripts/schemavalidation/compliance-config.schema.json
new file mode 100644
index 0000000..677a2a8
--- /dev/null
+++ b/scripts/schemavalidation/compliance-config.schema.json
@@ -0,0 +1,79 @@
+ {
+    "$comment": "Copyright (c) 2023, ARM Limited.",
+    "$comment": "SPDX-License-Identifier: Apache-2.0",
+    "$id": "compliance-config.schema.json",
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "title": "compliance-config",
+    "description": "Compliance configuration for verifying TOSA tensor data outputs from a test",
+    "type": "object",
+    "properties": {
+        "version": {
+            "description": "version string X.Y of this config data",
+            "type": "string"
+        },
+        "tensors": {
+            "type": "object",
+            "description": "dictionary of output tensors - keys are the tosa network names",
+            "patternProperties":
+            {
+                "^.*$":
+                {
+                    "description": "per named tensor config for compliance checking",
+                    "type": "object",
+                    "properties": {
+                        "mode": {
+                            "description": "verifier mode EXACT, DOT_PRODUCT, ULP, REDUCE_PRODUCT, or FP_SPECIAL",
+                            "type": "string"
+                        },
+                        "ulp_info": {
+                            "description": "info required for the ULP mode",
+                            "type": "object",
+                            "properties":
+                            {
+                                "ulp": {
+                                    "description": "ulp range limit - such as 1 or 0.5",
+                                    "type": "string"
+                                }
+                            },
+                            "required": [ "ulp" ]
+                        },
+                        "dot_product_info": {
+                            "description": "info required for the DOT_PRODUCT mode",
+                            "type": "object",
+                            "properties":
+                            {
+                                "s": {
+                                    "description": "test set 0 to 5 or -1 for other data",
+                                    "type": "integer"
+                                },
+                                "ks": {
+                                    "description": "kernel size for this dot product operation",
+                                    "type": "integer"
+                                },
+                                "data_type": {
+                                    "description": "tensor data type, such as: BOOL, INT16, FP32",
+                                    "type": "string"
+                                }
+                            },
+                            "additionalProperties": false,
+                            "required": [
+                                "s",
+                                "ks",
+                                "data_type"
+                            ]
+                        }
+                    },
+                    "additionalProperties": false,
+                    "required": [
+                        "mode"
+                    ]
+                }
+            }
+        }
+    },
+    "additionalProperties": false,
+    "required": [
+        "version",
+        "tensors"
+    ]
+}
diff --git a/scripts/schemavalidation/datagen-config.schema.json b/scripts/schemavalidation/datagen-config.schema.json
new file mode 100644
index 0000000..01f9fad
--- /dev/null
+++ b/scripts/schemavalidation/datagen-config.schema.json
@@ -0,0 +1,165 @@
+ {
+    "$comment": "Copyright (c) 2023, ARM Limited.",
+    "$comment": "SPDX-License-Identifier: Apache-2.0",
+    "$id": "datagen-config.schema.json",
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "title": "datagen-config",
+    "description": "Data generator configuration for generating TOSA tensor data",
+    "type": "object",
+    "properties": {
+        "version": {
+            "description": "version string X.Y of this config data",
+            "type": "string"
+        },
+        "tensors": {
+            "type": "object",
+            "description": "dictionary of input tensors - keys are the tosa network names",
+            "patternProperties":
+            {
+                "^.*$":
+                {
+                    "description": "per named tensor config for data generation",
+                    "type": "object",
+                    "properties": {
+                        "generator": {
+                            "description": "data generator name - PSEUDO_RANDOM, DOT_PRODUCT or OP_SPECIFIC",
+                            "type": "string"
+                        },
+                        "data_type": {
+                            "description": "tensor data type, such as: BOOL, INT16, FP32",
+                            "type": "string"
+                        },
+                        "input_type": {
+                            "description": "input is variable or constant",
+                            "type": "string"
+                        },
+                        "shape": {
+                            "description": "tensor shape array (rank 0 should be [])",
+                            "type": "array",
+                            "items": {
+                                "description": "shape dimension",
+                                "type": "integer",
+                                "minimum": 1
+                            }
+                        },
+                        "op": {
+                            "description": "operator name the tensor is initially used with",
+                            "type": "string"
+                        },
+                        "input_pos": {
+                            "description": "tensor input position for the operator (starts at 0)",
+                            "type": "integer",
+                            "minimum": 0
+                        },
+                        "pseudo_random_info": {
+                            "description": "info required for the PSEUDO_RANDOM generator",
+                            "type": "object",
+                            "properties":
+                            {
+                                "rng_seed": {
+                                    "description": "random number generator seed",
+                                    "type": "integer"
+                                },
+                                "range": {
+                                    "type": "array",
+                                    "description": "range of random numbers from low to high inclusively",
+                                    "minItems": 2,
+                                    "maxItems": 2,
+                                    "items": {
+                                        "description": "[low value, high value] as strings to allow ints and floats",
+                                        "type": "string"
+                                    }
+                                }
+                            },
+                            "additionalProperties": false,
+                            "required": [ "rng_seed" ]
+                        },
+                        "dot_product_info": {
+                            "description": "info required for the DOT_PRODUCT generator",
+                            "type": "object",
+                            "properties":
+                            {
+                                "s": {
+                                    "description": "test set 0 to 5 or -1 for other data",
+                                    "type": "integer"
+                                },
+                                "ks": {
+                                    "description": "kernel size for this dot product operation",
+                                    "type": "integer"
+                                },
+                                "acc_type": {
+                                    "description": "operator accumulator type (like tensor data_type)",
+                                    "type": "string"
+                                },
+                                "kernel": {
+                                    "type": "array",
+                                    "description": "kernel x, y sizes (for avg_pool2d)",
+                                    "minItems": 2,
+                                    "maxItems": 2,
+                                    "items": {
+                                        "description": "kernel dimension",
+                                        "type": "integer",
+                                        "minimum": 1
+                                    }
+                                },
+                                "axis": {
+                                    "description": "axis dimension (for reduce_sum)",
+                                    "type": "integer",
+                                    "minimum": 0
+                                }
+                            },
+                            "additionalProperties": false,
+                            "required": [
+                                "s",
+                                "ks",
+                                "acc_type"
+                            ]
+                        },
+                        "op_specific_info": {
+                            "description": "info required for the OP_SPECIFIC generator",
+                            "type": "object",
+                            "properties":
+                            {
+                                "sub_generator": {
+                                    "description": "sub generator type for this op - FULL, SPECIAL or BOUNDARY",
+                                    "type": "string"
+                                },
+                                "offset": {
+                                    "description": "starting offset within the test data",
+                                    "type": "integer",
+                                    "minimum": 0
+                                },
+                                "attributes": {
+                                    "description": "attribute data from the op needed to compute the data",
+                                    "type": "object",
+                                    "properties": {
+                                        "TBD": {
+                                            "description": "Probably needed for RESCALE and MUL",
+                                            "type": "string"
+                                        }
+                                    }
+                                }
+                            },
+                            "additionalProperties": false,
+                            "required": [ "sub_generator" ]
+                        }
+                    },
+                    "additionalProperties": false,
+                    "required": [
+                        "generator",
+                        "data_type",
+                        "input_type",
+                        "shape",
+                        "op",
+                        "input_pos"
+                    ]
+                }
+            }
+        }
+    },
+    "additionalProperties": false,
+    "required": [
+        "version",
+        "tensors"
+    ]
+}
diff --git a/scripts/schemavalidation/desc.schema.json b/scripts/schemavalidation/desc.schema.json
new file mode 100644
index 0000000..226fc68
--- /dev/null
+++ b/scripts/schemavalidation/desc.schema.json
@@ -0,0 +1,111 @@
+ {
+    "$comment": "Copyright (c) 2023, ARM Limited.",
+    "$comment": "SPDX-License-Identifier: Apache-2.0",
+    "$id": "desc.schema.json",
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "title": "desc",
+    "description": "Test description",
+    "type": "object",
+    "properties": {
+        "tosa_file": {
+            "description": "tosa flatbuffer file",
+            "type": "string"
+        },
+        "ifm_name": {
+            "description": "list of input tensors",
+            "type": "array",
+            "items": {
+                "description": "tensor name",
+                "type": "string"
+            }
+        },
+        "ifm_file": {
+            "description": "list of input files, one for each tensor in ifm_name",
+            "type": "array",
+            "items": {
+                "description": "file name",
+                "type": "string"
+            }
+        },
+        "ofm_name": {
+            "description": "list of output tensors",
+            "type": "array",
+            "minItems": 1,
+            "items": {
+                "description": "tensor name",
+                "type": "string"
+            }
+        },
+        "ofm_file": {
+            "description": "list of output files, one for each tensor in ofm_name",
+            "type": "array",
+            "minItems": 1,
+            "items": {
+                "description": "file name",
+                "type": "string"
+            }
+        },
+        "expected_failure": {
+            "description": "if the test is expected to fail - used for test filtering",
+            "type": "boolean"
+        },
+        "expected_failure_desc": {
+            "description": "summary of expected failure test",
+            "type": "string"
+        },
+        "expected_return_code": {
+            "description": "DEPRECATED (unused): expected return code from ref-model",
+            "type": "integer"
+        },
+        "expected_result_file": {
+            "description": "DEPRECATED: list of expected result files, one for each tensor in ofm_name",
+            "type": "array",
+            "minItems": 1,
+            "items": {
+                "description": "file name",
+                "type": "string"
+            }
+        },
+        "profile": {
+            "description": "list of profiles this test is valid for",
+            "type": "array",
+            "minItems": 1,
+            "items": {
+                "description": "profile name - tosa_bi or tosa_mi",
+                "type": "string"
+            }
+        },
+        "tags": {
+            "description": "list of string tags that can be used for test filtering",
+            "type": "array",
+            "items": {
+                "description": "test tag for filtering",
+                "type": "string"
+            }
+        },
+        "meta": {
+            "description": "meta-data for test libraries",
+            "type": "object",
+            "properties":
+            {
+                "data_gen": {
+                    "description": "data generation meta-data",
+                    "$ref": "datagen-config.schema.json"
+                },
+                "compliance": {
+                    "description": "compliance meta-data",
+                    "$ref": "compliance-config.schema.json"
+                }
+            },
+            "additionalProperties": false
+        }
+    },
+    "additionalProperties": false,
+    "required": [
+        "tosa_file",
+        "ifm_name",
+        "ifm_file",
+        "ofm_name",
+        "ofm_file"
+    ]
+}
diff --git a/scripts/schemavalidation/schemavalidation.py b/scripts/schemavalidation/schemavalidation.py
new file mode 100644
index 0000000..2848edf
--- /dev/null
+++ b/scripts/schemavalidation/schemavalidation.py
@@ -0,0 +1,79 @@
+# Copyright (c) 2023, ARM Limited.
+# SPDX-License-Identifier: Apache-2.0
+"""Validates desc.json against its JSON schema."""
+import json
+from pathlib import Path
+
+from jsonschema import validate
+from referencing import Registry
+from referencing import Resource
+
+TD_SCHEMA_FULL = "full"
+TD_SCHEMA_DATA_GEN = "data_gen"
+TD_SCHEMA_COMPLIANCE = "compliance"
+
+# Default file info
+SCRIPT = Path(__file__).absolute()
+TD_SCHEMAS = {
+    TD_SCHEMA_DATA_GEN: "datagen-config.schema.json",
+    TD_SCHEMA_COMPLIANCE: "compliance-config.schema.json",
+    TD_SCHEMA_FULL: "desc.schema.json",
+}
+
+
+class TestDescSchemaValidator:
+    def __init__(self):
+        """Initialize by loading all the schemas and setting up a registry."""
+        self.registry = Registry()
+        self.schemas = {}
+        for key, name in TD_SCHEMAS.items():
+            schema_path = SCRIPT.parent / name
+            with schema_path.open("r") as fd:
+                schema = json.load(fd)
+            self.schemas[key] = schema
+            if key != TD_SCHEMA_FULL:
+                resource = Resource.from_contents(schema)
+                self.registry = self.registry.with_resource(uri=name, resource=resource)
+
+    def validate_config(self, config, schema_type=TD_SCHEMA_FULL):
+        """Validate the whole (or partial) config versus the relevant schema."""
+        validate(config, self.schemas[schema_type], registry=self.registry)
+
+
+def main(argv=None):
+    """Command line interface for the schema validation."""
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "path", type=Path, help="the path to the test directory to validate"
+    )
+    args = parser.parse_args(argv)
+    test_path = args.path
+
+    if not test_path.is_dir():
+        print(f"ERROR: Invalid directory - {test_path}")
+        return 2
+
+    test_desc_path = test_path / "desc.json"
+
+    if not test_desc_path.is_file():
+        print(f"ERROR: No test description found: {test_desc_path}")
+        return 2
+
+    # Load the JSON desc.json
+    try:
+        with test_desc_path.open("r") as fd:
+            test_desc = json.load(fd)
+    except Exception as e:
+        print(f"ERROR: Loading {test_desc_path} - {repr(e)}")
+        return 2
+
+    sv = TestDescSchemaValidator()
+    sv.validate_config(test_desc, TD_SCHEMA_FULL)
+
+    return 0
+
+
+if __name__ == "__main__":
+    exit(main())
diff --git a/setup.cfg b/setup.cfg
index 6cd1489..a8adcf4 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,4 +1,4 @@
-# Copyright (c) 2021-2022 Arm Limited.
+# Copyright (c) 2021-2023 Arm Limited.
 # SPDX-License-Identifier: Apache-2.0
 
 [metadata]
@@ -21,6 +21,7 @@ classifiers =
 install_requires =
     numpy
     flatbuffers == 23.5.26
+    jsonschema
 python_requires = >=3.6
 include_package_data = True
 packages =
@@ -33,6 +34,7 @@ packages =
     xunit
     json2fbbin
     json2numpy
+    schemavalidation
     convert2conformance
     tosa
     serializer
@@ -44,19 +46,25 @@ package_dir =
     convert2conformance = scripts/convert2conformance
     tosa = thirdparty/serialization_lib/python/tosa
     serializer = thirdparty/serialization_lib/python/serializer
+    schemavalidation = scripts/schemavalidation
 
 [options.entry_points]
 console_scripts =
     tosa_verif_run_ref = runner.tosa_verif_run_tests:main
     tosa_verif_run_tests = runner.tosa_verif_run_tests:main
     tosa_verif_build_tests = generator.tosa_verif_build_tests:main
-    json2numpy = json2numpy.json2numpy:main
-    json2fbbin = json2fbbin.json2fbbin:main
+    tosa_json2numpy = json2numpy.json2numpy:main
+    tosa_json2fbbin = json2fbbin.json2fbbin:main
     tosa_verif_result_check = checker.tosa_result_checker:main
-    convert2conformance = convert2conformance.convert2conformance:main
+    tosa_convert2conformance = convert2conformance.convert2conformance:main
     tosa_verif_framework_generator = frameworks.tosa_verif_framework_generator:main
     tosa_verif_framework_compiler_runner = frameworks.tosa_verif_framework_compiler_runner:main
     tosa_verif_conformance_generator = conformance.tosa_verif_conformance_generator:main
+    tosa_schemavalidation = schemavalidation.schemavalidation:main
+
+[options.package_data]
+schemavalidation=
+    *.schema.json
 
 [tool:pytest]
 testpaths=verif/tests
diff --git a/verif/tests/test_schemavalidation.py b/verif/tests/test_schemavalidation.py
new file mode 100644
index 0000000..1ecd3ee
--- /dev/null
+++ b/verif/tests/test_schemavalidation.py
@@ -0,0 +1,102 @@
+"""Tests for schemavalidation.py."""
+# Copyright (c) 2023, ARM Limited.
+# SPDX-License-Identifier: Apache-2.0
+import pytest
+import schemavalidation.schemavalidation as sch
+from jsonschema.exceptions import ValidationError
+
+
+def test_schemavalidation_full_fail():
+    json = {}
+
+    sv = sch.TestDescSchemaValidator()
+    with pytest.raises(ValidationError) as excinfo:
+        sv.validate_config(json)
+    info = str(excinfo.value).split("\n")
+    assert info[0] == "'tosa_file' is a required property"
+
+
+def test_schemavalidation_compliance_fail():
+    json = {"version": "v"}
+
+    sv = sch.TestDescSchemaValidator()
+    with pytest.raises(ValidationError) as excinfo:
+        sv.validate_config(json, sch.TD_SCHEMA_COMPLIANCE)
+    info = str(excinfo.value).split("\n")
+    assert info[0] == "'tensors' is a required property"
+
+
+def test_schemavalidation_data_gen_fail():
+    json = {"version": "v", "tensors": {"input": {}}}
+
+    sv = sch.TestDescSchemaValidator()
+    with pytest.raises(ValidationError) as excinfo:
+        sv.validate_config(json, sch.TD_SCHEMA_DATA_GEN)
+    info = str(excinfo.value).split("\n")
+    assert info[0] == "'generator' is a required property"
+
+
+def test_schemavalidation_full_minimal():
+    json = {
+        "tosa_file": "file",
+        "ifm_name": ["name1", "name2"],
+        "ifm_file": ["file1", "file2"],
+        "ofm_name": ["name1", "name2"],
+        "ofm_file": ["file1", "file2"],
+    }
+
+    sv = sch.TestDescSchemaValidator()
+    sv.validate_config(json)
+
+
+def test_schemavalidation_full_unexpected():
+    json = {
+        "tosa_file": "file",
+        "ifm_name": ["name1", "name2"],
+        "ifm_file": ["file1", "file2"],
+        "ofm_name": ["name1", "name2"],
+        "ofm_file": ["file1", "file2"],
+        "unexpected_property": 1,
+    }
+
+    sv = sch.TestDescSchemaValidator()
+    with pytest.raises(ValidationError) as excinfo:
+        sv.validate_config(json)
+    info = str(excinfo.value).split("\n")
+    assert (
+        info[0]
+        == "Additional properties are not allowed ('unexpected_property' was unexpected)"
+    )
+
+
+def test_schemavalidation_compliance_minimal():
+    json = {
+        "version": "v",
+        "tensors": {
+            "output": {
+                "mode": "mode",
+            }
+        },
+    }
+
+    sv = sch.TestDescSchemaValidator()
+    sv.validate_config(json, sch.TD_SCHEMA_COMPLIANCE)
+
+
+def test_schemavalidation_data_gen_minimal():
+    json = {
+        "version": "v",
+        "tensors": {
+            "input": {
+                "generator": "generator",
+                "data_type": "type",
+                "input_type": "constant",
+                "shape": [],
+                "op": "name",
+                "input_pos": 0,
+            }
+        },
+    }
+
+    sv = sch.TestDescSchemaValidator()
+    sv.validate_config(json, sch.TD_SCHEMA_DATA_GEN)