From a9d539548ab90693682683b4c685a1597b1c8c26 Mon Sep 17 00:00:00 2001 From: Jared Smolens Date: Tue, 24 Aug 2021 23:48:19 +0000 Subject: Restore expectedFailure to test description files - Updated example tests, added missing npy files - Updated README file with fields in desc.json Change-Id: I5651ed5260866a8381adf620fb6dd6405cb2833b --- README.md | 6 +++++- examples/test_add_1x4x4x4_f32/flatbuffer-tf/desc.json | 2 +- .../flatbuffer-tf/ref_model_Result0.npy | Bin 384 -> 0 bytes .../test_add_1x4x4x4_f32/flatbuffer-tflite/desc.json | 2 +- .../flatbuffer-tflite/ref_model_Result0.npy | Bin 384 -> 0 bytes examples/test_add_1x4x4x4_f32/model.pb | 2 +- .../flatbuffer-tf/desc.json | 2 +- .../flatbuffer-tf/ref_model_Result0.npy | Bin 65664 -> 0 bytes .../flatbuffer-tflite/desc.json | 2 +- .../flatbuffer-tflite/ref_model_Result0.npy | Bin 65664 -> 0 bytes .../model.pb | 2 +- .../model.tflite | Bin 1272 -> 1272 bytes .../flatbuffer-tflite/desc.json | 2 +- .../flatbuffer-tflite/ref_model_Result0.npy | Bin 65664 -> 0 bytes .../model.tflite | Bin 1408 -> 1408 bytes verif/tosa_serializer.py | 9 +++++++++ 16 files changed, 21 insertions(+), 8 deletions(-) delete mode 100644 examples/test_add_1x4x4x4_f32/flatbuffer-tf/ref_model_Result0.npy delete mode 100644 examples/test_add_1x4x4x4_f32/flatbuffer-tflite/ref_model_Result0.npy delete mode 100644 examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tf/ref_model_Result0.npy delete mode 100644 examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tflite/ref_model_Result0.npy delete mode 100644 examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/flatbuffer-tflite/ref_model_Result0.npy diff --git a/README.md b/README.md index bfdb199..f2b8bec 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,11 @@ The JSON test descriptor must have the following field: * expected_failure: type: boolean. Is this test expected to fail in runtime. - +* expected_return_code: + type: int + The expected return code of the reference model i + (0 = VALID, 1 = ERROR, 2 = UNPREDICTABLE) + Note by default, all the files specified by "tosa_file", "ifm_file", "ofm_file" are relative to desc.json. This could be overwritten by -Cflatbuffer_dir=, if desired. diff --git a/examples/test_add_1x4x4x4_f32/flatbuffer-tf/desc.json b/examples/test_add_1x4x4x4_f32/flatbuffer-tf/desc.json index 423772a..0749286 100644 --- a/examples/test_add_1x4x4x4_f32/flatbuffer-tf/desc.json +++ b/examples/test_add_1x4x4x4_f32/flatbuffer-tf/desc.json @@ -12,7 +12,7 @@ "TosaOutput_0" ], "ofm_file": [ - "ref_model_Result0.npy" + "ref_model_output_0.npy" ], "expected_failure": false } \ No newline at end of file diff --git a/examples/test_add_1x4x4x4_f32/flatbuffer-tf/ref_model_Result0.npy b/examples/test_add_1x4x4x4_f32/flatbuffer-tf/ref_model_Result0.npy deleted file mode 100644 index 60baeac..0000000 Binary files a/examples/test_add_1x4x4x4_f32/flatbuffer-tf/ref_model_Result0.npy and /dev/null differ diff --git a/examples/test_add_1x4x4x4_f32/flatbuffer-tflite/desc.json b/examples/test_add_1x4x4x4_f32/flatbuffer-tflite/desc.json index 423772a..0749286 100644 --- a/examples/test_add_1x4x4x4_f32/flatbuffer-tflite/desc.json +++ b/examples/test_add_1x4x4x4_f32/flatbuffer-tflite/desc.json @@ -12,7 +12,7 @@ "TosaOutput_0" ], "ofm_file": [ - "ref_model_Result0.npy" + "ref_model_output_0.npy" ], "expected_failure": false } \ No newline at end of file diff --git a/examples/test_add_1x4x4x4_f32/flatbuffer-tflite/ref_model_Result0.npy b/examples/test_add_1x4x4x4_f32/flatbuffer-tflite/ref_model_Result0.npy deleted file mode 100644 index 60baeac..0000000 Binary files a/examples/test_add_1x4x4x4_f32/flatbuffer-tflite/ref_model_Result0.npy and /dev/null differ diff --git a/examples/test_add_1x4x4x4_f32/model.pb b/examples/test_add_1x4x4x4_f32/model.pb index c8b634c..59c6aec 100644 --- a/examples/test_add_1x4x4x4_f32/model.pb +++ b/examples/test_add_1x4x4x4_f32/model.pb @@ -92,5 +92,5 @@ node { } } versions { - producer: 790 + producer: 810 } diff --git a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tf/desc.json b/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tf/desc.json index e59bac3..e9df724 100644 --- a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tf/desc.json +++ b/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tf/desc.json @@ -10,7 +10,7 @@ "TosaOutput_0" ], "ofm_file": [ - "ref_model_Result0.npy" + "ref_model_output_0.npy" ], "expected_failure": false } \ No newline at end of file diff --git a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tf/ref_model_Result0.npy b/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tf/ref_model_Result0.npy deleted file mode 100644 index d57e8da..0000000 Binary files a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tf/ref_model_Result0.npy and /dev/null differ diff --git a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tflite/desc.json b/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tflite/desc.json index e59bac3..e9df724 100644 --- a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tflite/desc.json +++ b/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tflite/desc.json @@ -10,7 +10,7 @@ "TosaOutput_0" ], "ofm_file": [ - "ref_model_Result0.npy" + "ref_model_output_0.npy" ], "expected_failure": false } \ No newline at end of file diff --git a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tflite/ref_model_Result0.npy b/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tflite/ref_model_Result0.npy deleted file mode 100644 index d57e8da..0000000 Binary files a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tflite/ref_model_Result0.npy and /dev/null differ diff --git a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/model.pb b/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/model.pb index e4f47e2..b21dff3 100644 --- a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/model.pb +++ b/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/model.pb @@ -137,5 +137,5 @@ node { } } versions { - producer: 790 + producer: 810 } diff --git a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/model.tflite b/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/model.tflite index 02ad2fb..8b5bdf6 100644 Binary files a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/model.tflite and b/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/model.tflite differ diff --git a/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/flatbuffer-tflite/desc.json b/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/flatbuffer-tflite/desc.json index 360394e..e0889fc 100644 --- a/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/flatbuffer-tflite/desc.json +++ b/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/flatbuffer-tflite/desc.json @@ -10,7 +10,7 @@ "TosaOutput_0" ], "ofm_file": [ - "ref_model_Result0.npy" + "ref_model_output_0.npy" ], "expected_failure": false } \ No newline at end of file diff --git a/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/flatbuffer-tflite/ref_model_Result0.npy b/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/flatbuffer-tflite/ref_model_Result0.npy deleted file mode 100644 index 5c9ff1d..0000000 Binary files a/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/flatbuffer-tflite/ref_model_Result0.npy and /dev/null differ diff --git a/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/model.tflite b/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/model.tflite index 93dd386..840fdb9 100644 Binary files a/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/model.tflite and b/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/model.tflite differ diff --git a/verif/tosa_serializer.py b/verif/tosa_serializer.py index 35dd9a2..b7f5565 100644 --- a/verif/tosa_serializer.py +++ b/verif/tosa_serializer.py @@ -591,6 +591,7 @@ class TosaSerializer: # Is this an illegal test that is expected to fail? self.expectedReturnCode = TosaReturnCode.VALID + self.expectedFailure = False self.expectedFailureDesc = "" def __str__(self): @@ -674,6 +675,13 @@ class TosaSerializer: self.expectedReturnCode = val self.expectedFailureDesc = desc + if val == TosaReturnCode.VALID: + self.expectedFailure = False + else: + # Unpredictable or error results are considered expected failures + # for conformance + self.expectedFailure = True + def serialize(self): builder = self.builder @@ -724,6 +732,7 @@ class TosaSerializer: test_desc["ofm_name"] = ofm_name test_desc["ofm_file"] = ofm_file test_desc["expected_return_code"] = self.expectedReturnCode + test_desc["expected_failure"] = self.expectedFailure if self.expectedFailureDesc: test_desc["expected_failure_desc"] = self.expectedFailureDesc -- cgit v1.2.1