diff options
Diffstat (limited to 'test/scripts')
| -rwxr-xr-x | test/scripts/test_npy_fileio.py | 152 | ||||
| -rwxr-xr-x | test/scripts/test_serialization.py | 197 | ||||
| -rw-r--r-- | test/scripts/testfiles/test.tosa | bin | 0 -> 544 bytes | |||
| -rw-r--r-- | test/scripts/xunit/xunit.py | 109 |
4 files changed, 458 insertions, 0 deletions
diff --git a/test/scripts/test_npy_fileio.py b/test/scripts/test_npy_fileio.py new file mode 100755 index 0000000..e0a6f5d --- /dev/null +++ b/test/scripts/test_npy_fileio.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2021, ARM Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Simple test script which tests numpy file read/write""" + +import argparse +import random +import shlex +import subprocess +from datetime import datetime +from enum import IntEnum, unique +from pathlib import Path +from xunit.xunit import xunit_results, xunit_test + + +@unique +class TestResult(IntEnum): + PASS = 0 + COMMAND_ERROR = 1 + MISMATCH = 2 + SKIPPED = 3 + + +def parseArgs(): + baseDir = (Path(__file__).parent / "../..").resolve() + buildDir = (baseDir / "build").resolve() + parser = argparse.ArgumentParser() + + parser.add_argument( + "-c", + "--cmd", + default=str(buildDir / "serialization_npy_test"), + help="Command to write/read test file", + ) + parser.add_argument("-s", "--seed", default=1, help="Random number seed") + parser.add_argument( + "-v", "--verbose", action="store_true", help="verbose", default=False + ) + parser.add_argument( + "--xunit-file", default="npy-result.xml", help="xunit result output file" + ) + args = parser.parse_args() + + # check that required files exist + if not Path(args.cmd).exists(): + print("command not found at location " + args.cmd) + parser.print_help() + exit(1) + return args + + +def run_sh_command(full_cmd, verbose=False, capture_output=False): + """Utility function to run an external command. Optionally return captured + stdout/stderr""" + + # Quote the command line for printing + full_cmd_esc = [shlex.quote(x) for x in full_cmd] + + if verbose: + print("### Running {}".format(" ".join(full_cmd_esc))) + + if capture_output: + rc = subprocess.run(full_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if rc.returncode != 0: + print(rc.stdout.decode("utf-8")) + print(rc.stderr.decode("utf-8")) + raise Exception( + "Error running command: {}.\n{}".format( + " ".join(full_cmd_esc), rc.stderr.decode("utf-8") + ) + ) + return (rc.stdout, rc.stderr) + else: + rc = subprocess.run(full_cmd) + if rc.returncode != 0: + raise Exception("Error running command: {}".format(" ".join(full_cmd_esc))) + + +def runTest(args, dtype, shape): + start_time = datetime.now() + result = TestResult.PASS + message = "" + + target = Path(f"npytest-{random.randint(0,10000)}.npy") + shape_str = ",".join(shape) + # Remove any previous files + if target.exists(): + target.unlink() + + try: + cmd = [args.cmd, "-d", dtype, "-f", str(target), "-t", shape_str] + run_sh_command(cmd, args.verbose) + target.unlink() + + except Exception as e: + message = str(e) + result = TestResult.COMMAND_ERROR + end_time = datetime.now() + return result, message, end_time - start_time + + +def main(): + args = parseArgs() + + suitename = "basic_serialization" + classname = "npy_test" + + xunit_result = xunit_results() + xunit_suite = xunit_result.create_suite("basic_serialization") + + max_size = 128 + datatypes = ["int32", "int64", "float", "bool"] + random.seed(args.seed) + + failed = 0 + count = 0 + for test in datatypes: + count = count + 1 + shape = [] + for i in range(4): + shape.append(str(random.randint(1, max_size))) + (result, message, time_delta) = runTest(args, test, shape) + xt = xunit_test(str(test), f"{suitename}.{classname}") + xt.time = str( + float(time_delta.seconds) + (float(time_delta.microseconds) * 1e-6) + ) + if result == TestResult.PASS: + pass + else: + xt.failed(message) + failed = failed + 1 + xunit_suite.tests.append(xt) + + xunit_result.write_results(args.xunit_file) + print(f"Total tests run: {count} failures: {failed}") + + +if __name__ == "__main__": + exit(main()) diff --git a/test/scripts/test_serialization.py b/test/scripts/test_serialization.py new file mode 100755 index 0000000..834bc1d --- /dev/null +++ b/test/scripts/test_serialization.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2021, ARM Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Simple test script which uses serialization_read_write to copy tosa files. It +uses flatc to convert to json for comparison since the binary files may +differ. """ + +import argparse +import filecmp +import random +import shlex +import subprocess +from datetime import datetime +from enum import IntEnum, unique +from pathlib import Path +from xunit.xunit import xunit_results, xunit_test + + +@unique +class TestResult(IntEnum): + PASS = 0 + COMMAND_ERROR = 1 + MISMATCH = 2 + SKIPPED = 3 + + +def parseArgs(): + baseDir = (Path(__file__).parent / "../..").resolve() + buildDir = (baseDir / "build").resolve() + parser = argparse.ArgumentParser() + parser.add_argument( + "-t", + "--testdir", + dest="test", + type=str, + required=True, + help="Directory of tosa files to verify", + ) + parser.add_argument( + "--flatc", + default=str(buildDir / "third_party/flatbuffers/flatc"), + help="location of flatc compiler", + ) + parser.add_argument( + "-s", + "--schema", + default=str(baseDir / "schema/tosa.fbs"), + help="location of schema file", + ) + parser.add_argument( + "-c", + "--cmd", + default=str(buildDir / "serialization_read_write"), + help="Command to read/write test file", + ) + parser.add_argument( + "-v", "--verbose", action="store_true", help="verbose", default=False + ) + parser.add_argument( + "--xunit-file", default="result.xml", help="xunit result output file" + ) + args = parser.parse_args() + + # check that required files exist + if not Path(args.flatc).exists(): + print("flatc not found at location " + args.flatc) + parser.print_help() + exit(1) + if not Path(args.cmd).exists(): + print("command not found at location " + args.cmd) + parser.print_help() + exit(1) + if not Path(args.schema).exists(): + print("schema not found at location " + args.schema) + parser.print_help() + exit(1) + return args + + +def run_sh_command(full_cmd, verbose=False, capture_output=False): + """Utility function to run an external command. Optionally return captured + stdout/stderr""" + + # Quote the command line for printing + full_cmd_esc = [shlex.quote(x) for x in full_cmd] + + if verbose: + print("### Running {}".format(" ".join(full_cmd_esc))) + + if capture_output: + rc = subprocess.run(full_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if rc.returncode != 0: + print(rc.stdout.decode("utf-8")) + print(rc.stderr.decode("utf-8")) + raise Exception( + "Error running command: {}.\n{}".format( + " ".join(full_cmd_esc), rc.stderr.decode("utf-8") + ) + ) + return (rc.stdout, rc.stderr) + else: + rc = subprocess.run(full_cmd) + if rc.returncode != 0: + raise Exception("Error running command: {}".format(" ".join(full_cmd_esc))) + + +def runTest(args, testfile): + start_time = datetime.now() + result = TestResult.PASS + message = "" + + target = Path(f"serialization_script_output-{random.randint(0,10000)}.tosa") + source_json = Path(testfile.stem + ".json") + target_json = Path(target.stem + ".json") + + # Remove any previous files + if target.exists(): + target.unlink() + if source_json.exists(): + source_json.unlink() + if target_json.exists(): + target_json.unlink() + + try: + cmd = [args.cmd, str(testfile), str(target)] + run_sh_command(cmd, args.verbose) + # Create result json + cmd = [args.flatc, "--json", "--raw-binary", args.schema, "--", str(target)] + run_sh_command(cmd, args.verbose) + # Create source json + cmd = [args.flatc, "--json", "--raw-binary", args.schema, "--", str(testfile)] + run_sh_command(cmd, args.verbose) + if not filecmp.cmp(str(target_json), str(source_json), False): + print("Failed to compare files on " + str(testfile)) + result = TestResult.MISMATCH + # Cleanup generated files + source_json.unlink() + target_json.unlink() + target.unlink() + + except Exception as e: + message = str(e) + result = TestResult.COMMAND_ERROR + end_time = datetime.now() + return result, message, end_time - start_time + + +def getTestFiles(dir): + files = Path(dir).glob("**/*.tosa") + return files + + +def main(): + args = parseArgs() + testfiles = getTestFiles(args.test) + + suitename = "basic_serialization" + classname = "copy_test" + + xunit_result = xunit_results() + xunit_suite = xunit_result.create_suite("basic_serialization") + + failed = 0 + count = 0 + for test in testfiles: + count = count + 1 + (result, message, time_delta) = runTest(args, test) + xt = xunit_test(str(test), f"{suitename}.{classname}") + xt.time = str( + float(time_delta.seconds) + (float(time_delta.microseconds) * 1e-6) + ) + if result == TestResult.PASS: + pass + else: + xt.failed(message) + failed = failed + 1 + xunit_suite.tests.append(xt) + + xunit_result.write_results(args.xunit_file) + print(f"Total tests run: {count} failures: {failed}") + + +if __name__ == "__main__": + exit(main()) diff --git a/test/scripts/testfiles/test.tosa b/test/scripts/testfiles/test.tosa Binary files differnew file mode 100644 index 0000000..3b4ca56 --- /dev/null +++ b/test/scripts/testfiles/test.tosa diff --git a/test/scripts/xunit/xunit.py b/test/scripts/xunit/xunit.py new file mode 100644 index 0000000..2de0d5c --- /dev/null +++ b/test/scripts/xunit/xunit.py @@ -0,0 +1,109 @@ +# Copyright (c) 2020-2021, ARM Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import xml.etree.ElementTree as ET +from xml.dom import minidom + + +class xunit_results: + def __init__(self): + self.name = "testsuites" + self.suites = [] + + def create_suite(self, name): + s = xunit_suite(name) + self.suites.append(s) + return s + + def write_results(self, filename): + suites = ET.Element(self.name) + tree = ET.ElementTree(suites) + for s in self.suites: + testsuite = ET.SubElement( + suites, "testsuite", {"name": s.name, "errors": "0"} + ) + tests = 0 + failures = 0 + skip = 0 + for t in s.tests: + test = ET.SubElement( + testsuite, + "testcase", + {"name": t.name, "classname": t.classname, "time": t.time}, + ) + tests += 1 + if t.skip: + skip += 1 + ET.SubElement(test, "skipped", {"type": "Skipped test"}) + if t.fail: + failures += 1 + fail = ET.SubElement(test, "failure", {"type": "Test failed"}) + fail.text = t.fail + if t.sysout: + sysout = ET.SubElement(test, "system-out") + sysout.text = t.sysout + if t.syserr: + syserr = ET.SubElement(test, "system-err") + syserr.text = t.syserr + testsuite.attrib["tests"] = str(tests) + testsuite.attrib["failures"] = str(failures) + testsuite.attrib["skip"] = str(skip) + xmlstr = minidom.parseString(ET.tostring(tree.getroot())).toprettyxml( + indent=" " + ) + with open(filename, "w") as f: + f.write(xmlstr) + + +class xunit_suite: + def __init__(self, name): + self.name = name + self.tests = [] + + +# classname should be of the form suite.class/subclass/subclass2/... It appears +# you can have an unlimited number of subclasses in this manner + + +class xunit_test: + def __init__(self, name, classname=None): + self.name = name + if classname: + self.classname = classname + else: + self.classname = name + self.time = "0.000" + self.fail = None + self.skip = False + self.sysout = None + self.syserr = None + + def failed(self, text): + self.fail = text + + def skipped(self): + self.skip = True + + +if __name__ == "__main__": + r = xunit_results() + s = r.create_suite("selftest") + for i in range(0, 10): + t = xunit_test("atest" + str(i), "selftest") + if i == 3: + t.failed("Unknown failure foo") + if i == 7: + t.skipped() + s.tests.append(t) + r.write_results("foo.xml") |