diff options
-rw-r--r-- | README.md | 34 | ||||
-rw-r--r-- | pyproject.toml | 12 | ||||
-rw-r--r-- | scripts/py-dev-env.bash | 24 | ||||
-rw-r--r-- | scripts/py-dev-env.sh | 22 | ||||
-rw-r--r-- | scripts/xunit/__init__.py | 3 | ||||
-rw-r--r-- | setup.cfg | 40 | ||||
-rw-r--r-- | setup.py | 6 | ||||
m--------- | thirdparty/serialization_lib | 0 | ||||
-rw-r--r-- | verif/generator/__init__.py | 3 | ||||
-rw-r--r-- | verif/generator/tosa_error_if.py (renamed from verif/tosa_error_if.py) | 0 | ||||
-rw-r--r-- | verif/generator/tosa_test_gen.py (renamed from verif/tosa_test_gen.py) | 16 | ||||
-rw-r--r--[-rwxr-xr-x] | verif/generator/tosa_verif_build_tests.py (renamed from verif/tosa_verif_build_tests.py) | 21 | ||||
-rw-r--r-- | verif/runner/__init__.py | 3 | ||||
-rw-r--r-- | verif/runner/tosa_ref_run.py (renamed from verif/tosa_ref_run.py) | 2 | ||||
-rw-r--r-- | verif/runner/tosa_test_runner.py (renamed from verif/tosa_test_runner.py) | 0 | ||||
-rw-r--r--[-rwxr-xr-x] | verif/runner/tosa_verif_run_ref.py (renamed from verif/tosa_verif_run_ref.py) | 22 |
16 files changed, 150 insertions, 58 deletions
@@ -46,7 +46,7 @@ The testing infrastructure requires: Check out the required git submodules with: ``` bash -$ git submodule update --init --recursive +git submodule update --init --recursive ``` # Compilation @@ -54,16 +54,16 @@ $ git submodule update --init --recursive The *TOSA Reference Model* build can be prepared by creating makefiles using CMake: ``` bash -$ mkdir -p build -$ cd build -$ cmake .. +mkdir -p build +cd build +cmake .. ``` Optionally, `-DCMAKE_BUILD_MODE=Debug` can be used on the `cmake` command to create a debug build. Next compile using `make`: ``` bash -$ make +make ``` The resulting executable will be named: @@ -101,7 +101,7 @@ The JSON test descriptor must have the following field: type: int The expected return code of the reference model i (0 = VALID, 1 = ERROR, 2 = UNPREDICTABLE) - + Note by default, all the files specified by "tosa_file", "ifm_file", "ofm_file" are relative to desc.json. This could be overwritten by -Cflatbuffer_dir=, if desired. @@ -109,7 +109,7 @@ Note by default, all the files specified by "tosa_file", "ifm_file", An example command is shown below: ``` bash -$ ./build/reference_model/tosa_reference_model \ +./build/reference_model/tosa_reference_model \ -Ctest_desc=examples/test_add_1x4x4x4_f32/flatbuffer-tflite/desc.json ``` @@ -127,7 +127,7 @@ format into output tensors specified by "ofm_file". For example, you can generate new output .npy by: ``` bash -$ ./build/reference_model/tosa_reference_model \ +./build/reference_model/tosa_reference_model \ -Ctest_desc=examples/test_add_1x4x4x4_f32/flatbuffer-tflite/desc.json -Cofm_file=out.npy ``` @@ -193,10 +193,24 @@ tests on additional tools or frameworks. The reference output NumPy files are generated by this step and can be programatically compared with output of other tools. to validate those tools. +## Installation + +The test infrastructure needs installing before being used. It is recommended +to create a [python virtual environment](https://docs.python.org/3/library/venv.html) +and then install the TOSA Unit Test infrastruture from the root of the +reference model: + +``` bash +pip install . +``` + +When installing without a python virtual environment, use the pip +option `--user` to install it for the current user only. + ## Usage ### Unit Test Builder -The test builder is in ``verif/tosa_verif_build_tests.py``. The +The test builder is invoked by ``tosa_verif_build_tests``. The builder generates test outputs in ``./vtest/<operator_name>/`` by default. To restrict test generation to particular regular expression wildcard, use the ``--filter `` argument. The tool can be run with no @@ -222,7 +236,7 @@ parallel using the ``-j`` switch. For example, to run all of the add operator tests: ``` bash -$ ./verif/tosa_verif_run_ref.py -t vtest/add/add* -j 8 +tosa_verif_run_ref -t vtest/add/add* -j 8 ``` The test runner is quiet by default, so running a large number of diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..49aa0ed --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,12 @@ +# Copyright (c) 2021-2022 Arm Limited. +# SPDX-License-Identifier: Apache-2.0 + +[build-system] +requires = [ + "setuptools>=42", + "wheel", + "setuptools_scm[toml]>=6.0" +] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] diff --git a/scripts/py-dev-env.bash b/scripts/py-dev-env.bash new file mode 100644 index 0000000..82c16e3 --- /dev/null +++ b/scripts/py-dev-env.bash @@ -0,0 +1,24 @@ +# Copyright (c) 2021-2022 Arm Limited. +# SPDX-License-Identifier: Apache-2.0 + +# Source this script to install the python tosa tools as editable: +# . scripts/py-dev-env.bash + +# NOTE: This script is needed to fix up PYTHONPATH due to a bug +# in setuptools that does not support multiple package_dirs +# properly: https://github.com/pypa/setuptools/issues/230 + +if [ -z "$VIRTUAL_ENV" ]; then + echo "Please source this within a python virtual env" + return +fi + +if [ -e "${BASH_SOURCE[0]}" ]; then + SCRIPTS_DIR=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) + REFMODEL_DIR=$(dirname "$SCRIPTS_DIR") + pip install -e "$REFMODEL_DIR" + export PYTHONPATH="$SCRIPTS_DIR:$REFMODEL_DIR/thirdparty/serialization_lib/python" + echo "Set PYTHONPATH=$PYTHONPATH" +else + echo "Please source this using bash" +fi diff --git a/scripts/py-dev-env.sh b/scripts/py-dev-env.sh new file mode 100644 index 0000000..904ff80 --- /dev/null +++ b/scripts/py-dev-env.sh @@ -0,0 +1,22 @@ +# Copyright (c) 2021-2022 Arm Limited. +# SPDX-License-Identifier: Apache-2.0 + +# Source this script to install the python tosa tools as editable: +# . scripts/py-dev-env.sh + +# NOTE: This script is needed to fix up PYTHONPATH due to a bug +# in setuptools that does not support multiple package_dirs +# properly: https://github.com/pypa/setuptools/issues/230 + +if [ -z "$VIRTUAL_ENV" ]; then + echo "Please source this within a python virtual env" + return +fi + +if [ -e "setup.cfg" ]; then + pip install -e . + export PYTHONPATH=$PWD/scripts:$PWD/thirdparty/serialization_lib/python + echo "Set PYTHONPATH=$PYTHONPATH" +else + echo "Please source this from the root of reference_model" +fi diff --git a/scripts/xunit/__init__.py b/scripts/xunit/__init__.py new file mode 100644 index 0000000..39e9ecc --- /dev/null +++ b/scripts/xunit/__init__.py @@ -0,0 +1,3 @@ +"""Namespace.""" +# Copyright (c) 2021-2022 Arm Limited. +# SPDX-License-Identifier: Apache-2.0 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..5b7ffc4 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,40 @@ +# Copyright (c) 2021-2022 Arm Limited. +# SPDX-License-Identifier: Apache-2.0 + +[metadata] +name = tosa-tools +# version = done by setuptools_scm in pyproject.toml +author = Arm Limited +#author_email = +description = TOSA tools +long_description = file: README.md +long_description_content_type = text/markdown +url = https://git.mlplatform.org/tosa/reference_model.git/ +project_urls = + Website = https://developer.mlplatform.org/w/tosa/ +classifiers = + Programming Language :: Python :: 3 + License :: OSI Approved :: Apache Software License + Operating System :: OS Independent + +[options] +install_requires = + numpy + flatbuffers == 2.0 +python_requires = >=3.6 +packages = + runner + generator + xunit + tosa + serializer +package_dir = + = verif + xunit = scripts/xunit + tosa = thirdparty/serialization_lib/python/tosa + serializer = thirdparty/serialization_lib/python/serializer + +[options.entry_points] +console_scripts = + tosa_verif_run_ref = runner.tosa_verif_run_ref:main + tosa_verif_build_tests = generator.tosa_verif_build_tests:main diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..4916510 --- /dev/null +++ b/setup.py @@ -0,0 +1,6 @@ +# Copyright (c) 2021-2022 Arm Limited. +# SPDX-License-Identifier: Apache-2.0 +"""Setup script for backward compatibility.""" +from setuptools import setup + +setup() diff --git a/thirdparty/serialization_lib b/thirdparty/serialization_lib -Subproject 49faa4e42b763dbbce214bb101619bf395cf836 +Subproject 9b22517ba0cd6f767123583ce56e864f50e9d75 diff --git a/verif/generator/__init__.py b/verif/generator/__init__.py new file mode 100644 index 0000000..39e9ecc --- /dev/null +++ b/verif/generator/__init__.py @@ -0,0 +1,3 @@ +"""Namespace.""" +# Copyright (c) 2021-2022 Arm Limited. +# SPDX-License-Identifier: Apache-2.0 diff --git a/verif/tosa_error_if.py b/verif/generator/tosa_error_if.py index 7c162be..7c162be 100644 --- a/verif/tosa_error_if.py +++ b/verif/generator/tosa_error_if.py diff --git a/verif/tosa_test_gen.py b/verif/generator/tosa_test_gen.py index 0d29704..0d5a881 100644 --- a/verif/tosa_test_gen.py +++ b/verif/generator/tosa_test_gen.py @@ -32,17 +32,11 @@ import itertools from copy import deepcopy from enum import IntEnum, Enum, unique -from tosa_ref_run import TosaReturnCode - -# Include the ../thirdparty/serialization_lib/python directory in PYTHONPATH -parent_dir = os.path.dirname(os.path.realpath(__file__)) -sys.path.append( - os.path.join(parent_dir, "..", "thirdparty", "serialization_lib", "python") -) -import tosa_serializer as ts -from tosa_serializer import * + +import serializer.tosa_serializer as ts +from serializer.tosa_serializer import * import tosa -from tosa_error_if import ErrorIf +from generator.tosa_error_if import ErrorIf # Convenience variables to the flatc-generated types that should be enums, but aren't from tosa.DType import DType @@ -1587,7 +1581,7 @@ class TosaErrorValidator: overall_result &= expected_result if expected_result and error_result: - serializer.setExpectedReturnCode(2, error_reason) + serializer.setExpectedReturnCode(2, True, desc=error_reason) elif error_result: # and not expected_result print(f"Unexpected ERROR_IF: Op: {valueToName(Op, kwargs['op']['op'])}" f" Expected: {error_name}, Got: {validator_name}") diff --git a/verif/tosa_verif_build_tests.py b/verif/generator/tosa_verif_build_tests.py index c667e79..09ee238 100755..100644 --- a/verif/tosa_verif_build_tests.py +++ b/verif/generator/tosa_verif_build_tests.py @@ -1,6 +1,4 @@ -#!/usr/bin/env python3 - -# Copyright (c) 2020, ARM Limited. +# Copyright (c) 2020-2021, ARM Limited. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -32,21 +30,8 @@ import traceback from enum import IntEnum, Enum, unique from datetime import datetime -# Include the ../scripts and ../scripts/xunit directory in PYTHONPATH -parent_dir = os.path.dirname(os.path.realpath(__file__)) -sys.path.append(os.path.join(parent_dir, "..", "scripts")) -sys.path.append(os.path.join(parent_dir, "..", "scripts", "xunit")) - -import xunit - -# Include the ../thirdparty/serialization_lib/python directory in PYTHONPATH -parent_dir = os.path.dirname(os.path.realpath(__file__)) -sys.path.append( - os.path.join(parent_dir, "..", "thirdparty", "serialization_lib", "python") -) -from tosa_serializer import * -from tosa_test_gen import TosaTestGen -import tosa +from generator.tosa_test_gen import TosaTestGen +from serializer.tosa_serializer import dtype_str_to_val # Used for parsing a comma-separated list of integers in a string # to an actual list of integers diff --git a/verif/runner/__init__.py b/verif/runner/__init__.py new file mode 100644 index 0000000..39e9ecc --- /dev/null +++ b/verif/runner/__init__.py @@ -0,0 +1,3 @@ +"""Namespace.""" +# Copyright (c) 2021-2022 Arm Limited. +# SPDX-License-Identifier: Apache-2.0 diff --git a/verif/tosa_ref_run.py b/verif/runner/tosa_ref_run.py index de35d93..c1d5e79 100644 --- a/verif/tosa_ref_run.py +++ b/verif/runner/tosa_ref_run.py @@ -17,7 +17,7 @@ import json import shlex import subprocess from enum import Enum, IntEnum, unique -from tosa_test_runner import TosaTestRunner, run_sh_command +from runner.tosa_test_runner import TosaTestRunner, run_sh_command @unique diff --git a/verif/tosa_test_runner.py b/verif/runner/tosa_test_runner.py index e8f921d..e8f921d 100644 --- a/verif/tosa_test_runner.py +++ b/verif/runner/tosa_test_runner.py diff --git a/verif/tosa_verif_run_ref.py b/verif/runner/tosa_verif_run_ref.py index 0bfca0f..626819f 100755..100644 --- a/verif/tosa_verif_run_ref.py +++ b/verif/runner/tosa_verif_run_ref.py @@ -1,6 +1,4 @@ -#!/usr/bin/env python3 - -# Copyright (c) 2020, ARM Limited. +# Copyright (c) 2020-2021, ARM Limited. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -33,21 +31,9 @@ import importlib from enum import IntEnum, Enum, unique from datetime import datetime -# Include the ../scripts and ../scripts/xunit directory in PYTHONPATH -parent_dir = os.path.dirname(os.path.realpath(__file__)) -sys.path.append(os.path.join(parent_dir, "..", "scripts")) -sys.path.append(os.path.join(parent_dir, "..", "scripts", "xunit")) - -import xunit +from xunit import xunit -# Include the ../thirdparty/serialization_lib/python directory in PYTHONPATH -parent_dir = os.path.dirname(os.path.realpath(__file__)) -sys.path.append( - os.path.join(parent_dir, "..", "thirdparty", "serialization_lib", "python") -) -import tosa -from tosa_test_gen import TosaTestGen -from tosa_test_runner import TosaTestRunner +from runner.tosa_test_runner import TosaTestRunner no_color_printing = False # from run_tf_unit_test import LogColors, print_color, run_sh_command @@ -105,7 +91,7 @@ def parseArgs(): dest="sut_module", type=str, nargs="+", - default=["tosa_ref_run"], + default=["runner.tosa_ref_run"], help="System under test module to load (derives from TosaTestRunner). May be repeated", ) parser.add_argument( |