diff options
author | Jared Smolens <jared.smolens@arm.com> | 2021-08-24 23:48:19 +0000 |
---|---|---|
committer | Jared Smolens <jared.smolens@arm.com> | 2021-08-25 08:28:03 -0700 |
commit | a9d539548ab90693682683b4c685a1597b1c8c26 (patch) | |
tree | 140029c3deb9a541c092bef91584bf41c1bf34eb | |
parent | acb550f4410ae861e53cae27a9feb4b11d45769f (diff) | |
download | reference_model-a9d539548ab90693682683b4c685a1597b1c8c26.tar.gz |
Restore expectedFailure to test description files
- Updated example tests, added missing npy files
- Updated README file with fields in desc.json
Change-Id: I5651ed5260866a8381adf620fb6dd6405cb2833b
16 files changed, 21 insertions, 8 deletions
@@ -97,7 +97,11 @@ The JSON test descriptor must have the following field: * expected_failure: type: boolean. Is this test expected to fail in runtime. - +* expected_return_code: + type: int + The expected return code of the reference model i + (0 = VALID, 1 = ERROR, 2 = UNPREDICTABLE) + Note by default, all the files specified by "tosa_file", "ifm_file", "ofm_file" are relative to desc.json. This could be overwritten by -Cflatbuffer_dir=, if desired. diff --git a/examples/test_add_1x4x4x4_f32/flatbuffer-tf/desc.json b/examples/test_add_1x4x4x4_f32/flatbuffer-tf/desc.json index 423772a..0749286 100644 --- a/examples/test_add_1x4x4x4_f32/flatbuffer-tf/desc.json +++ b/examples/test_add_1x4x4x4_f32/flatbuffer-tf/desc.json @@ -12,7 +12,7 @@ "TosaOutput_0" ], "ofm_file": [ - "ref_model_Result0.npy" + "ref_model_output_0.npy" ], "expected_failure": false }
\ No newline at end of file diff --git a/examples/test_add_1x4x4x4_f32/flatbuffer-tf/ref_model_Result0.npy b/examples/test_add_1x4x4x4_f32/flatbuffer-tf/ref_model_Result0.npy Binary files differdeleted file mode 100644 index 60baeac..0000000 --- a/examples/test_add_1x4x4x4_f32/flatbuffer-tf/ref_model_Result0.npy +++ /dev/null diff --git a/examples/test_add_1x4x4x4_f32/flatbuffer-tflite/desc.json b/examples/test_add_1x4x4x4_f32/flatbuffer-tflite/desc.json index 423772a..0749286 100644 --- a/examples/test_add_1x4x4x4_f32/flatbuffer-tflite/desc.json +++ b/examples/test_add_1x4x4x4_f32/flatbuffer-tflite/desc.json @@ -12,7 +12,7 @@ "TosaOutput_0" ], "ofm_file": [ - "ref_model_Result0.npy" + "ref_model_output_0.npy" ], "expected_failure": false }
\ No newline at end of file diff --git a/examples/test_add_1x4x4x4_f32/flatbuffer-tflite/ref_model_Result0.npy b/examples/test_add_1x4x4x4_f32/flatbuffer-tflite/ref_model_Result0.npy Binary files differdeleted file mode 100644 index 60baeac..0000000 --- a/examples/test_add_1x4x4x4_f32/flatbuffer-tflite/ref_model_Result0.npy +++ /dev/null diff --git a/examples/test_add_1x4x4x4_f32/model.pb b/examples/test_add_1x4x4x4_f32/model.pb index c8b634c..59c6aec 100644 --- a/examples/test_add_1x4x4x4_f32/model.pb +++ b/examples/test_add_1x4x4x4_f32/model.pb @@ -92,5 +92,5 @@ node { } } versions { - producer: 790 + producer: 810 } diff --git a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tf/desc.json b/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tf/desc.json index e59bac3..e9df724 100644 --- a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tf/desc.json +++ b/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tf/desc.json @@ -10,7 +10,7 @@ "TosaOutput_0" ], "ofm_file": [ - "ref_model_Result0.npy" + "ref_model_output_0.npy" ], "expected_failure": false }
\ No newline at end of file diff --git a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tf/ref_model_Result0.npy b/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tf/ref_model_Result0.npy Binary files differdeleted file mode 100644 index d57e8da..0000000 --- a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tf/ref_model_Result0.npy +++ /dev/null diff --git a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tflite/desc.json b/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tflite/desc.json index e59bac3..e9df724 100644 --- a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tflite/desc.json +++ b/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tflite/desc.json @@ -10,7 +10,7 @@ "TosaOutput_0" ], "ofm_file": [ - "ref_model_Result0.npy" + "ref_model_output_0.npy" ], "expected_failure": false }
\ No newline at end of file diff --git a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tflite/ref_model_Result0.npy b/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tflite/ref_model_Result0.npy Binary files differdeleted file mode 100644 index d57e8da..0000000 --- a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tflite/ref_model_Result0.npy +++ /dev/null diff --git a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/model.pb b/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/model.pb index e4f47e2..b21dff3 100644 --- a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/model.pb +++ b/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/model.pb @@ -137,5 +137,5 @@ node { } } versions { - producer: 790 + producer: 810 } diff --git a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/model.tflite b/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/model.tflite Binary files differindex 02ad2fb..8b5bdf6 100644 --- a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/model.tflite +++ b/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/model.tflite diff --git a/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/flatbuffer-tflite/desc.json b/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/flatbuffer-tflite/desc.json index 360394e..e0889fc 100644 --- a/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/flatbuffer-tflite/desc.json +++ b/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/flatbuffer-tflite/desc.json @@ -10,7 +10,7 @@ "TosaOutput_0" ], "ofm_file": [ - "ref_model_Result0.npy" + "ref_model_output_0.npy" ], "expected_failure": false }
\ No newline at end of file diff --git a/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/flatbuffer-tflite/ref_model_Result0.npy b/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/flatbuffer-tflite/ref_model_Result0.npy Binary files differdeleted file mode 100644 index 5c9ff1d..0000000 --- a/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/flatbuffer-tflite/ref_model_Result0.npy +++ /dev/null diff --git a/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/model.tflite b/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/model.tflite Binary files differindex 93dd386..840fdb9 100644 --- a/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/model.tflite +++ b/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/model.tflite diff --git a/verif/tosa_serializer.py b/verif/tosa_serializer.py index 35dd9a2..b7f5565 100644 --- a/verif/tosa_serializer.py +++ b/verif/tosa_serializer.py @@ -591,6 +591,7 @@ class TosaSerializer: # Is this an illegal test that is expected to fail? self.expectedReturnCode = TosaReturnCode.VALID + self.expectedFailure = False self.expectedFailureDesc = "" def __str__(self): @@ -674,6 +675,13 @@ class TosaSerializer: self.expectedReturnCode = val self.expectedFailureDesc = desc + if val == TosaReturnCode.VALID: + self.expectedFailure = False + else: + # Unpredictable or error results are considered expected failures + # for conformance + self.expectedFailure = True + def serialize(self): builder = self.builder @@ -724,6 +732,7 @@ class TosaSerializer: test_desc["ofm_name"] = ofm_name test_desc["ofm_file"] = ofm_file test_desc["expected_return_code"] = self.expectedReturnCode + test_desc["expected_failure"] = self.expectedFailure if self.expectedFailureDesc: test_desc["expected_failure_desc"] = self.expectedFailureDesc |