diff options
author | Jeremy Johnson <jeremy.johnson@arm.com> | 2023-08-22 16:55:08 +0100 |
---|---|---|
committer | Eric Kunze <eric.kunze@arm.com> | 2023-08-30 17:03:11 +0000 |
commit | 0633c3a2603ea626ec7cc035dc8f6dc7f9f07795 (patch) | |
tree | d1c883f0defecac32fbf5b78936c4fbbe9b014e1 | |
parent | 0913dba65846e369e1bcabce246da764c3e454b4 (diff) | |
download | reference_model-0633c3a2603ea626ec7cc035dc8f6dc7f9f07795.tar.gz |
Fix conformance regressions due to type and region changes.
Compress flatbuffer JSON files for conformance to allow while_loop
tests to fit in the 30MB file size limit.
Do not include new SHAPE type in usable DTypes so that selection of
ERROR_IF tests are not changed.
Also enable strict json mode for flatbuffers.
Change-Id: Id89a9963244c34769a4662f9951adde0aa3334d9
Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
-rw-r--r-- | scripts/json2fbbin/json2fbbin.py | 70 | ||||
-rw-r--r-- | verif/generator/tosa_utils.py | 2 |
2 files changed, 67 insertions, 5 deletions
diff --git a/scripts/json2fbbin/json2fbbin.py b/scripts/json2fbbin/json2fbbin.py index 8f9f274..08b13be 100644 --- a/scripts/json2fbbin/json2fbbin.py +++ b/scripts/json2fbbin/json2fbbin.py @@ -1,14 +1,69 @@ """Conversion utility from flatbuffer JSON files to binary and the reverse.""" -# Copyright (c) 2021-2022, ARM Limited. +# Copyright (c) 2021-2023, ARM Limited. # SPDX-License-Identifier: Apache-2.0 +import re from pathlib import Path from typing import Optional from runner.run_command import run_sh_command from runner.run_command import RunShCommandError - -def fbbin_to_json(flatc: Path, fbs: Path, t_path: Path, o_path: Optional[Path] = None): +MAX_LINE_LEN = 120 +MAX_INDENT_LEN = 20 + + +def json_squeeze(json_path: Path): + """File compression for JSONs, reducing spaces used for number lists.""" + # Move existing file to a new name + temp_path = json_path.with_suffix(".json_unsqueezed") + json_path.rename(temp_path) + # Now read the original file and write a smaller output with less new lines/spaces + with temp_path.open("r") as tfd: + with json_path.open("w") as jfd: + found = False + + for line in tfd: + # Find lines that are part of number lists + match = re.match(r"(\s+)(-?[0-9]+),?", line) + if match: + # Found a line with just a number on it (and optional comma) + if not found: + # New list of numbers + numbers = [] + # Save indent (upto maximum) + indent = match.group(1)[0:MAX_INDENT_LEN] + found = True + numbers.append(match.group(2)) + else: + # Found a line without just a number + if found: + # Format the list of numbers recorded into a concise output + # with multiple numbers on a single line, rather than one per line + numbers_str = indent + for num in numbers: + nums = f"{num}," + if len(numbers_str) + len(nums) > MAX_LINE_LEN: + print(numbers_str, file=jfd) + numbers_str = indent + numbers_str += nums + # print all but the last comma + print(numbers_str[:-1], file=jfd) + + found = False + # print the line we just read (that wasn't just a number) + print(line, file=jfd, end="") + + # Remove the uncompressed version + temp_path.unlink() + + +def fbbin_to_json( + flatc: Path, + fbs: Path, + t_path: Path, + o_path: Optional[Path] = None, + squeeze: Optional[bool] = True, +): """Convert the binary flatbuffer to JSON. flatc: the Path to the flatc compiler program @@ -23,6 +78,7 @@ def fbbin_to_json(flatc: Path, fbs: Path, t_path: Path, o_path: Optional[Path] = "-o", str(o_path.absolute()), "--json", + "--strict-json", "--defaults-json", "--raw-binary", str(fbs.absolute()), @@ -30,6 +86,9 @@ def fbbin_to_json(flatc: Path, fbs: Path, t_path: Path, o_path: Optional[Path] = str(t_path.absolute()), ] run_sh_command(verbose=False, full_cmd=cmd) + if squeeze: + json_path = (o_path / t_path.name).with_suffix(".json") + json_squeeze(json_path) def json_to_fbbin(flatc: Path, fbs: Path, j_path: Path, o_path: Optional[Path] = None): @@ -62,6 +121,9 @@ def main(argv=None): parser = argparse.ArgumentParser() parser.add_argument( + "--no-squeeze", action="store_true", help="no compression of json output" + ) + parser.add_argument( "--flatc", type=Path, default=( @@ -97,7 +159,7 @@ def main(argv=None): json_to_fbbin(args.flatc, args.fbs, path) else: # Have to assume this is a binary flatbuffer file as could have any suffix - fbbin_to_json(args.flatc, args.fbs, path) + fbbin_to_json(args.flatc, args.fbs, path, squeeze=(not args.no_squeeze)) except RunShCommandError as e: print(e) return 1 diff --git a/verif/generator/tosa_utils.py b/verif/generator/tosa_utils.py index f9df8d5..3cd0370 100644 --- a/verif/generator/tosa_utils.py +++ b/verif/generator/tosa_utils.py @@ -93,7 +93,7 @@ def usableDTypes(*, excludes=None): Returns: A set of DType values """ - omit = {DType.UNKNOWN, DType.UINT8, DType.UINT16} + omit = {DType.UNKNOWN, DType.UINT8, DType.UINT16, DType.SHAPE} omit.update(excludes if excludes else ()) return allDTypes(excludes=omit) |