Initial commit of serialization library code

Change-Id: Ie09a7245176aa799e59622e5118b145833b23590
Signed-off-by: Eric Kunze <eric.kunze@arm.com>

6 files changed, 733 insertions, 0 deletions

diff --git a/test/scripts/test_npy_fileio.py b/test/scripts/test_npy_fileio.py
new file mode 100755
index 0000000..e0a6f5d
--- /dev/null
+++ b/test/scripts/test_npy_fileio.py
@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2021, ARM Limited.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+""" Simple test script which tests numpy file read/write"""
+
+import argparse
+import random
+import shlex
+import subprocess
+from datetime import datetime
+from enum import IntEnum, unique
+from pathlib import Path
+from xunit.xunit import xunit_results, xunit_test
+
+
+@unique
+class TestResult(IntEnum):
+    PASS = 0
+    COMMAND_ERROR = 1
+    MISMATCH = 2
+    SKIPPED = 3
+
+
+def parseArgs():
+    baseDir = (Path(__file__).parent / "../..").resolve()
+    buildDir = (baseDir / "build").resolve()
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "-c",
+        "--cmd",
+        default=str(buildDir / "serialization_npy_test"),
+        help="Command to write/read test file",
+    )
+    parser.add_argument("-s", "--seed", default=1, help="Random number seed")
+    parser.add_argument(
+        "-v", "--verbose", action="store_true", help="verbose", default=False
+    )
+    parser.add_argument(
+        "--xunit-file", default="npy-result.xml", help="xunit result output file"
+    )
+    args = parser.parse_args()
+
+    # check that required files exist
+    if not Path(args.cmd).exists():
+        print("command not found at location " + args.cmd)
+        parser.print_help()
+        exit(1)
+    return args
+
+
+def run_sh_command(full_cmd, verbose=False, capture_output=False):
+    """Utility function to run an external command. Optionally return captured
+    stdout/stderr"""
+
+    # Quote the command line for printing
+    full_cmd_esc = [shlex.quote(x) for x in full_cmd]
+
+    if verbose:
+        print("### Running {}".format(" ".join(full_cmd_esc)))
+
+    if capture_output:
+        rc = subprocess.run(full_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        if rc.returncode != 0:
+            print(rc.stdout.decode("utf-8"))
+            print(rc.stderr.decode("utf-8"))
+            raise Exception(
+                "Error running command: {}.\n{}".format(
+                    " ".join(full_cmd_esc), rc.stderr.decode("utf-8")
+                )
+            )
+        return (rc.stdout, rc.stderr)
+    else:
+        rc = subprocess.run(full_cmd)
+    if rc.returncode != 0:
+        raise Exception("Error running command: {}".format(" ".join(full_cmd_esc)))
+
+
+def runTest(args, dtype, shape):
+    start_time = datetime.now()
+    result = TestResult.PASS
+    message = ""
+
+    target = Path(f"npytest-{random.randint(0,10000)}.npy")
+    shape_str = ",".join(shape)
+    # Remove any previous files
+    if target.exists():
+        target.unlink()
+
+    try:
+        cmd = [args.cmd, "-d", dtype, "-f", str(target), "-t", shape_str]
+        run_sh_command(cmd, args.verbose)
+        target.unlink()
+
+    except Exception as e:
+        message = str(e)
+        result = TestResult.COMMAND_ERROR
+    end_time = datetime.now()
+    return result, message, end_time - start_time
+
+
+def main():
+    args = parseArgs()
+
+    suitename = "basic_serialization"
+    classname = "npy_test"
+
+    xunit_result = xunit_results()
+    xunit_suite = xunit_result.create_suite("basic_serialization")
+
+    max_size = 128
+    datatypes = ["int32", "int64", "float", "bool"]
+    random.seed(args.seed)
+
+    failed = 0
+    count = 0
+    for test in datatypes:
+        count = count + 1
+        shape = []
+        for i in range(4):
+            shape.append(str(random.randint(1, max_size)))
+        (result, message, time_delta) = runTest(args, test, shape)
+        xt = xunit_test(str(test), f"{suitename}.{classname}")
+        xt.time = str(
+            float(time_delta.seconds) + (float(time_delta.microseconds) * 1e-6)
+        )
+        if result == TestResult.PASS:
+            pass
+        else:
+            xt.failed(message)
+            failed = failed + 1
+        xunit_suite.tests.append(xt)
+
+    xunit_result.write_results(args.xunit_file)
+    print(f"Total tests run: {count} failures: {failed}")
+
+
+if __name__ == "__main__":
+    exit(main())
diff --git a/test/scripts/test_serialization.py b/test/scripts/test_serialization.py
new file mode 100755
index 0000000..834bc1d
--- /dev/null
+++ b/test/scripts/test_serialization.py
@@ -0,0 +1,197 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2021, ARM Limited.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+""" Simple test script which uses serialization_read_write to copy tosa files. It
+uses flatc to convert to json for comparison since the binary files may
+differ. """
+
+import argparse
+import filecmp
+import random
+import shlex
+import subprocess
+from datetime import datetime
+from enum import IntEnum, unique
+from pathlib import Path
+from xunit.xunit import xunit_results, xunit_test
+
+
+@unique
+class TestResult(IntEnum):
+    PASS = 0
+    COMMAND_ERROR = 1
+    MISMATCH = 2
+    SKIPPED = 3
+
+
+def parseArgs():
+    baseDir = (Path(__file__).parent / "../..").resolve()
+    buildDir = (baseDir / "build").resolve()
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-t",
+        "--testdir",
+        dest="test",
+        type=str,
+        required=True,
+        help="Directory of tosa files to verify",
+    )
+    parser.add_argument(
+        "--flatc",
+        default=str(buildDir / "third_party/flatbuffers/flatc"),
+        help="location of flatc compiler",
+    )
+    parser.add_argument(
+        "-s",
+        "--schema",
+        default=str(baseDir / "schema/tosa.fbs"),
+        help="location of schema file",
+    )
+    parser.add_argument(
+        "-c",
+        "--cmd",
+        default=str(buildDir / "serialization_read_write"),
+        help="Command to read/write test file",
+    )
+    parser.add_argument(
+        "-v", "--verbose", action="store_true", help="verbose", default=False
+    )
+    parser.add_argument(
+        "--xunit-file", default="result.xml", help="xunit result output file"
+    )
+    args = parser.parse_args()
+
+    # check that required files exist
+    if not Path(args.flatc).exists():
+        print("flatc not found at location " + args.flatc)
+        parser.print_help()
+        exit(1)
+    if not Path(args.cmd).exists():
+        print("command not found at location " + args.cmd)
+        parser.print_help()
+        exit(1)
+    if not Path(args.schema).exists():
+        print("schema not found at location " + args.schema)
+        parser.print_help()
+        exit(1)
+    return args
+
+
+def run_sh_command(full_cmd, verbose=False, capture_output=False):
+    """Utility function to run an external command. Optionally return captured
+    stdout/stderr"""
+
+    # Quote the command line for printing
+    full_cmd_esc = [shlex.quote(x) for x in full_cmd]
+
+    if verbose:
+        print("### Running {}".format(" ".join(full_cmd_esc)))
+
+    if capture_output:
+        rc = subprocess.run(full_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        if rc.returncode != 0:
+            print(rc.stdout.decode("utf-8"))
+            print(rc.stderr.decode("utf-8"))
+            raise Exception(
+                "Error running command: {}.\n{}".format(
+                    " ".join(full_cmd_esc), rc.stderr.decode("utf-8")
+                )
+            )
+        return (rc.stdout, rc.stderr)
+    else:
+        rc = subprocess.run(full_cmd)
+    if rc.returncode != 0:
+        raise Exception("Error running command: {}".format(" ".join(full_cmd_esc)))
+
+
+def runTest(args, testfile):
+    start_time = datetime.now()
+    result = TestResult.PASS
+    message = ""
+
+    target = Path(f"serialization_script_output-{random.randint(0,10000)}.tosa")
+    source_json = Path(testfile.stem + ".json")
+    target_json = Path(target.stem + ".json")
+
+    # Remove any previous files
+    if target.exists():
+        target.unlink()
+    if source_json.exists():
+        source_json.unlink()
+    if target_json.exists():
+        target_json.unlink()
+
+    try:
+        cmd = [args.cmd, str(testfile), str(target)]
+        run_sh_command(cmd, args.verbose)
+        # Create result json
+        cmd = [args.flatc, "--json", "--raw-binary", args.schema, "--", str(target)]
+        run_sh_command(cmd, args.verbose)
+        # Create source json
+        cmd = [args.flatc, "--json", "--raw-binary", args.schema, "--", str(testfile)]
+        run_sh_command(cmd, args.verbose)
+        if not filecmp.cmp(str(target_json), str(source_json), False):
+            print("Failed to compare files on " + str(testfile))
+            result = TestResult.MISMATCH
+        # Cleanup generated files
+        source_json.unlink()
+        target_json.unlink()
+        target.unlink()
+
+    except Exception as e:
+        message = str(e)
+        result = TestResult.COMMAND_ERROR
+    end_time = datetime.now()
+    return result, message, end_time - start_time
+
+
+def getTestFiles(dir):
+    files = Path(dir).glob("**/*.tosa")
+    return files
+
+
+def main():
+    args = parseArgs()
+    testfiles = getTestFiles(args.test)
+
+    suitename = "basic_serialization"
+    classname = "copy_test"
+
+    xunit_result = xunit_results()
+    xunit_suite = xunit_result.create_suite("basic_serialization")
+
+    failed = 0
+    count = 0
+    for test in testfiles:
+        count = count + 1
+        (result, message, time_delta) = runTest(args, test)
+        xt = xunit_test(str(test), f"{suitename}.{classname}")
+        xt.time = str(
+            float(time_delta.seconds) + (float(time_delta.microseconds) * 1e-6)
+        )
+        if result == TestResult.PASS:
+            pass
+        else:
+            xt.failed(message)
+            failed = failed + 1
+        xunit_suite.tests.append(xt)
+
+    xunit_result.write_results(args.xunit_file)
+    print(f"Total tests run: {count} failures: {failed}")
+
+
+if __name__ == "__main__":
+    exit(main())
diff --git a/test/scripts/testfiles/test.tosa b/test/scripts/testfiles/test.tosa
new file mode 100644
index 0000000..3b4ca56
--- /dev/null
+++ b/test/scripts/testfiles/test.tosa
diff --git a/test/scripts/xunit/xunit.py b/test/scripts/xunit/xunit.py
new file mode 100644
index 0000000..2de0d5c
--- /dev/null
+++ b/test/scripts/xunit/xunit.py
@@ -0,0 +1,109 @@
+# Copyright (c) 2020-2021, ARM Limited.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+import xml.etree.ElementTree as ET
+from xml.dom import minidom
+
+
+class xunit_results:
+    def __init__(self):
+        self.name = "testsuites"
+        self.suites = []
+
+    def create_suite(self, name):
+        s = xunit_suite(name)
+        self.suites.append(s)
+        return s
+
+    def write_results(self, filename):
+        suites = ET.Element(self.name)
+        tree = ET.ElementTree(suites)
+        for s in self.suites:
+            testsuite = ET.SubElement(
+                suites, "testsuite", {"name": s.name, "errors": "0"}
+            )
+            tests = 0
+            failures = 0
+            skip = 0
+            for t in s.tests:
+                test = ET.SubElement(
+                    testsuite,
+                    "testcase",
+                    {"name": t.name, "classname": t.classname, "time": t.time},
+                )
+                tests += 1
+                if t.skip:
+                    skip += 1
+                    ET.SubElement(test, "skipped", {"type": "Skipped test"})
+                if t.fail:
+                    failures += 1
+                    fail = ET.SubElement(test, "failure", {"type": "Test failed"})
+                    fail.text = t.fail
+                if t.sysout:
+                    sysout = ET.SubElement(test, "system-out")
+                    sysout.text = t.sysout
+                if t.syserr:
+                    syserr = ET.SubElement(test, "system-err")
+                    syserr.text = t.syserr
+            testsuite.attrib["tests"] = str(tests)
+            testsuite.attrib["failures"] = str(failures)
+            testsuite.attrib["skip"] = str(skip)
+        xmlstr = minidom.parseString(ET.tostring(tree.getroot())).toprettyxml(
+            indent="  "
+        )
+        with open(filename, "w") as f:
+            f.write(xmlstr)
+
+
+class xunit_suite:
+    def __init__(self, name):
+        self.name = name
+        self.tests = []
+
+
+# classname should be of the form suite.class/subclass/subclass2/... It appears
+# you can have an unlimited number of subclasses in this manner
+
+
+class xunit_test:
+    def __init__(self, name, classname=None):
+        self.name = name
+        if classname:
+            self.classname = classname
+        else:
+            self.classname = name
+        self.time = "0.000"
+        self.fail = None
+        self.skip = False
+        self.sysout = None
+        self.syserr = None
+
+    def failed(self, text):
+        self.fail = text
+
+    def skipped(self):
+        self.skip = True
+
+
+if __name__ == "__main__":
+    r = xunit_results()
+    s = r.create_suite("selftest")
+    for i in range(0, 10):
+        t = xunit_test("atest" + str(i), "selftest")
+        if i == 3:
+            t.failed("Unknown failure foo")
+        if i == 7:
+            t.skipped()
+        s.tests.append(t)
+    r.write_results("foo.xml")
diff --git a/test/src/serialization_npy_test.cpp b/test/src/serialization_npy_test.cpp
new file mode 100644
index 0000000..27ec464
--- /dev/null
+++ b/test/src/serialization_npy_test.cpp
@@ -0,0 +1,225 @@
+// Copyright (c) 2021, ARM Limited.
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//         http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+
+#include <getopt.h>
+#include <iostream>
+#include <random>
+#include <sstream>
+#include <tosa_serialization_handler.h>
+
+using namespace tosa;
+
+void usage()
+{
+    std::cout << "Usage: serialization_npy_test -f <filename> -t <shape> -d <datatype> -s <seed>" << std::endl;
+}
+
+template <class T>
+int test_int_type(std::vector<int32_t> shape, std::default_random_engine& gen, std::string& filename)
+{
+    size_t total_size = 1;
+    std::uniform_int_distribution<T> gen_data(std::numeric_limits<T>::min(), std::numeric_limits<T>::max());
+
+    for (auto i : shape)
+    {
+        total_size *= i;
+    }
+
+    auto buffer = std::make_unique<T[]>(total_size);
+    for (int i = 0; i < total_size; i++)
+    {
+        buffer[i] = gen_data(gen);
+    }
+
+    NumpyUtilities::NPError err = NumpyUtilities::writeToNpyFile(filename.c_str(), shape, buffer.get());
+    if (err != NumpyUtilities::NO_ERROR)
+    {
+        std::cout << "Error writing file, code " << err << std::endl;
+        return 1;
+    }
+
+    auto read_buffer = std::make_unique<T[]>(total_size);
+    err              = NumpyUtilities::readFromNpyFile(filename.c_str(), total_size, read_buffer.get());
+    if (err != NumpyUtilities::NO_ERROR)
+    {
+        std::cout << "Error reading file, code " << err << std::endl;
+        return 1;
+    }
+    if (memcmp(buffer.get(), read_buffer.get(), total_size * sizeof(T)))
+    {
+        std::cout << "Miscompare" << std::endl;
+        return 1;
+    }
+    return 0;
+}
+
+template <class T>
+int test_float_type(std::vector<int32_t> shape, std::default_random_engine& gen, std::string& filename)
+{
+    size_t total_size = 1;
+    std::uniform_real_distribution<T> gen_data(std::numeric_limits<T>::min(), std::numeric_limits<T>::max());
+
+    for (auto i : shape)
+    {
+        total_size *= i;
+    }
+
+    auto buffer = std::make_unique<T[]>(total_size);
+    for (int i = 0; i < total_size; i++)
+    {
+        buffer[i] = gen_data(gen);
+    }
+
+    NumpyUtilities::NPError err = NumpyUtilities::writeToNpyFile(filename.c_str(), shape, buffer.get());
+    if (err != NumpyUtilities::NO_ERROR)
+    {
+        std::cout << "Error writing file, code " << err << std::endl;
+        return 1;
+    }
+
+    auto read_buffer = std::make_unique<T[]>(total_size);
+    err              = NumpyUtilities::readFromNpyFile(filename.c_str(), total_size, read_buffer.get());
+    if (err != NumpyUtilities::NO_ERROR)
+    {
+        std::cout << "Error reading file, code " << err << std::endl;
+        return 1;
+    }
+    if (memcmp(buffer.get(), read_buffer.get(), total_size * sizeof(T)))
+    {
+        std::cout << "Miscompare" << std::endl;
+        return 1;
+    }
+    return 0;
+}
+
+int test_bool_type(std::vector<int32_t> shape, std::default_random_engine& gen, std::string& filename)
+{
+    size_t total_size = 1;
+    std::uniform_int_distribution<uint32_t> gen_data(0, 1);
+
+    for (auto i : shape)
+    {
+        total_size *= i;
+    }
+
+    auto buffer = std::make_unique<bool[]>(total_size);
+    for (int i = 0; i < total_size; i++)
+    {
+        buffer[i] = (gen_data(gen)) ? true : false;
+    }
+
+    NumpyUtilities::NPError err = NumpyUtilities::writeToNpyFile(filename.c_str(), shape, buffer.get());
+    if (err != NumpyUtilities::NO_ERROR)
+    {
+        std::cout << "Error writing file, code " << err << std::endl;
+        return 1;
+    }
+
+    auto read_buffer = std::make_unique<bool[]>(total_size);
+    err              = NumpyUtilities::readFromNpyFile(filename.c_str(), total_size, read_buffer.get());
+    if (err != NumpyUtilities::NO_ERROR)
+    {
+        std::cout << "Error reading file, code " << err << std::endl;
+        return 1;
+    }
+
+    if (memcmp(buffer.get(), read_buffer.get(), total_size * sizeof(bool)))
+    {
+        std::cout << "Miscompare" << std::endl;
+        return 1;
+    }
+    return 0;
+}
+
+int main(int argc, char** argv)
+{
+    size_t total_size = 1;
+    int32_t seed      = 1;
+    std::string str_type;
+    std::string str_shape;
+    std::string filename = "npytest.npy";
+    std::vector<int32_t> shape;
+    bool verbose = false;
+    int opt;
+    while ((opt = getopt(argc, argv, "d:f:s:t:v")) != -1)
+    {
+        switch (opt)
+        {
+            case 'd':
+                str_type = optarg;
+                break;
+            case 'f':
+                filename = optarg;
+                break;
+            case 's':
+                seed = strtol(optarg, nullptr, 0);
+                break;
+            case 't':
+                str_shape = optarg;
+                break;
+            case 'v':
+                verbose = true;
+                break;
+            default:
+                std::cerr << "Invalid argument" << std::endl;
+                break;
+        }
+    }
+    if (str_shape == "")
+    {
+        usage();
+        return 1;
+    }
+
+    // parse shape from argument
+    std::stringstream ss(str_shape);
+    while (ss.good())
+    {
+        std::string substr;
+        size_t pos;
+        std::getline(ss, substr, ',');
+        if (substr == "")
+            break;
+        int val = stoi(substr, &pos, 0);
+        assert(val);
+        total_size *= val;
+        shape.push_back(val);
+    }
+
+    std::default_random_engine gen(seed);
+
+    // run with type from argument
+    if (str_type == "int32")
+    {
+        return test_int_type<int32_t>(shape, gen, filename);
+    }
+    else if (str_type == "int64")
+    {
+        return test_int_type<int64_t>(shape, gen, filename);
+    }
+    else if (str_type == "float")
+    {
+        return test_float_type<float>(shape, gen, filename);
+    }
+    else if (str_type == "bool")
+    {
+        return test_bool_type(shape, gen, filename);
+    }
+    else
+    {
+        std::cout << "Unknown type " << str_type << std::endl;
+        usage();
+        return 1;
+    }
+}
diff --git a/test/src/serialization_read_write.cpp b/test/src/serialization_read_write.cpp
new file mode 100644
index 0000000..1f29fac
--- /dev/null
+++ b/test/src/serialization_read_write.cpp
@@ -0,0 +1,50 @@
+// Copyright (c) 2021, ARM Limited.
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//         http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+
+#include <iostream>
+#include <tosa_serialization_handler.h>
+
+using namespace tosa;
+
+void usage()
+{
+    std::cerr << "Usage: <src> <dest>" << std::endl;
+    std::cerr << "    <src>: source TOSA serialize filename" << std::endl;
+    std::cerr << "    <dest>: destination TOSA serialized filename" << std::endl;
+}
+
+int main(int argc, char** argv)
+{
+    TosaSerializationHandler handler;
+    if (argc != 3)
+    {
+        usage();
+        return 1;
+    }
+
+    tosa_err_t err = handler.LoadFileTosaFlatbuffer(argv[1]);
+    if (err != TOSA_OK)
+    {
+        std::cout << "error reading file " << argv[1] << " code " << err << std::endl;
+        return 1;
+    }
+
+    err = handler.SaveFileTosaFlatbuffer(argv[2]);
+    if (err != TOSA_OK)
+    {
+        std::cout << "error writing file " << argv[2] << " code " << err << std::endl;
+        return 1;
+    }
+    return 0;
+}