From 93d4390f9aa5c4369f889e1cd336aa4e809ff6a7 Mon Sep 17 00:00:00 2001 From: Jeremy Johnson Date: Tue, 27 Sep 2022 12:26:14 +0100 Subject: Minor fixes & add FP16 support to refmodel testing and conformance gen Improve base inference conformance generation: * Change to target specific dtypes required for conformance tests. * Reduce dimension sizes of ERROR_IF tests. NOTE: Will impact tensor, clamp and all ERROR_IF tests. Add option to change seed on conformance generation for extra testing. Stop creation of convolution tests with negative output dimensions. Improve reporting on failing to allocate tensor due to above issue. Fix runner to correctly pass ref model debug flags. Update reference_model examples. Signed-off-by: Jeremy Johnson Change-Id: I992180dcfe265a7d50edfb151c9f38eeaef5c369 --- .../flatbuffer-tf/test_add_1x4x4x4_f32.tosa | Bin 488 -> 492 bytes .../flatbuffer-tflite/test_add_1x4x4x4_f32.tosa | Bin 488 -> 492 bytes examples/test_add_1x4x4x4_f32/model.pb | 2 +- examples/test_add_1x4x4x4_f32/model.tflite | Bin 596 -> 956 bytes examples/test_add_1x4x4x4_f32/test.json | 2 +- ...v2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11.tosa | Bin 1284 -> 1584 bytes ...v2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11.tosa | Bin 1284 -> 1296 bytes .../model.pb | 2 +- .../model.tflite | Bin 1276 -> 1588 bytes .../test.json | 2 +- ...v2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11.tosa | Bin 1236 -> 1232 bytes .../model.tflite | Bin 1408 -> 1720 bytes .../test.json | 2 +- reference_model/src/subgraph_traverser.cc | 1 + verif/conformance/test_select.py | 21 +- verif/conformance/tosa_base_profile_ops_info.json | 434 +++++++++++++-------- .../tosa_verif_conformance_generator.py | 32 +- verif/generator/tosa_arg_gen.py | 22 +- verif/runner/tosa_refmodel_sut_run.py | 4 +- verif/tests/test_tosa_refmodel.py | 4 + 20 files changed, 345 insertions(+), 183 deletions(-) diff --git a/examples/test_add_1x4x4x4_f32/flatbuffer-tf/test_add_1x4x4x4_f32.tosa b/examples/test_add_1x4x4x4_f32/flatbuffer-tf/test_add_1x4x4x4_f32.tosa index 861665f..c240a91 100644 Binary files a/examples/test_add_1x4x4x4_f32/flatbuffer-tf/test_add_1x4x4x4_f32.tosa and b/examples/test_add_1x4x4x4_f32/flatbuffer-tf/test_add_1x4x4x4_f32.tosa differ diff --git a/examples/test_add_1x4x4x4_f32/flatbuffer-tflite/test_add_1x4x4x4_f32.tosa b/examples/test_add_1x4x4x4_f32/flatbuffer-tflite/test_add_1x4x4x4_f32.tosa index 861665f..c240a91 100644 Binary files a/examples/test_add_1x4x4x4_f32/flatbuffer-tflite/test_add_1x4x4x4_f32.tosa and b/examples/test_add_1x4x4x4_f32/flatbuffer-tflite/test_add_1x4x4x4_f32.tosa differ diff --git a/examples/test_add_1x4x4x4_f32/model.pb b/examples/test_add_1x4x4x4_f32/model.pb index a3c5cae..479979c 100644 --- a/examples/test_add_1x4x4x4_f32/model.pb +++ b/examples/test_add_1x4x4x4_f32/model.pb @@ -92,5 +92,5 @@ node { } } versions { - producer: 925 + producer: 1247 } diff --git a/examples/test_add_1x4x4x4_f32/model.tflite b/examples/test_add_1x4x4x4_f32/model.tflite index 5a88e98..c8fdae2 100644 Binary files a/examples/test_add_1x4x4x4_f32/model.tflite and b/examples/test_add_1x4x4x4_f32/model.tflite differ diff --git a/examples/test_add_1x4x4x4_f32/test.json b/examples/test_add_1x4x4x4_f32/test.json index 48df6a1..c75c169 100644 --- a/examples/test_add_1x4x4x4_f32/test.json +++ b/examples/test_add_1x4x4x4_f32/test.json @@ -4,7 +4,7 @@ "tf_result_name": "result", "tflite_model_filename": "model.tflite", "tflite_result_npy_filename": "tflite_result.npy", - "tflite_result_name": "Identity", + "tflite_result_name": "PartitionedCall:0", "ifm_file": [ "placeholder_0.npy", "placeholder_1.npy" diff --git a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tf/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11.tosa b/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tf/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11.tosa index a0a268d..c4224d3 100644 Binary files a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tf/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11.tosa and b/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tf/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11.tosa differ diff --git a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tflite/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11.tosa b/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tflite/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11.tosa index 497c01b..98480be 100644 Binary files a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tflite/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11.tosa and b/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/flatbuffer-tflite/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11.tosa differ diff --git a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/model.pb b/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/model.pb index 9d199e4..e95ba45 100644 --- a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/model.pb +++ b/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/model.pb @@ -137,5 +137,5 @@ node { } } versions { - producer: 925 + producer: 1247 } diff --git a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/model.tflite b/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/model.tflite index b2a60f8..17a4e6a 100644 Binary files a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/model.tflite and b/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/model.tflite differ diff --git a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/test.json b/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/test.json index 58766b7..2bd0aa8 100644 --- a/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/test.json +++ b/examples/test_conv2d_1x1_1x32x32x8_f32_st11_padSAME_dilat11/test.json @@ -4,7 +4,7 @@ "tf_result_name": "result", "tflite_model_filename": "model.tflite", "tflite_result_npy_filename": "tflite_result.npy", - "tflite_result_name": "Identity", + "tflite_result_name": "PartitionedCall:0", "ifm_file": [ "placeholder_0.npy" ], diff --git a/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/flatbuffer-tflite/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11.tosa b/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/flatbuffer-tflite/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11.tosa index a18e8c4..5570543 100644 Binary files a/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/flatbuffer-tflite/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11.tosa and b/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/flatbuffer-tflite/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11.tosa differ diff --git a/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/model.tflite b/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/model.tflite index 840fdb9..a503e7b 100644 Binary files a/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/model.tflite and b/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/model.tflite differ diff --git a/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/test.json b/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/test.json index 5124ff9..9fc6bad 100644 --- a/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/test.json +++ b/examples/test_conv2d_1x1_1x32x32x8_qi8_st11_padSAME_dilat11/test.json @@ -1,7 +1,7 @@ { "tflite_model_filename": "model.tflite", "tflite_result_npy_filename": "tflite_result.npy", - "tflite_result_name": "Identity", + "tflite_result_name": "PartitionedCall:0", "ifm_file": [ "placeholder_0.npy" ], diff --git a/reference_model/src/subgraph_traverser.cc b/reference_model/src/subgraph_traverser.cc index 386f0e5..cbb7001 100644 --- a/reference_model/src/subgraph_traverser.cc +++ b/reference_model/src/subgraph_traverser.cc @@ -327,6 +327,7 @@ int SubgraphTraverser::allocateTensor() { if (dim <= 0) { + DEBUG_INFO(GT, "Failed to allocate tensor %s with invalid dimension of %d", ts->GetName().c_str(), dim); this->setGraphStatus(GraphStatus::TOSA_UNPREDICTABLE); return 1; } diff --git a/verif/conformance/test_select.py b/verif/conformance/test_select.py index 8b60fbb..9e73b0d 100644 --- a/verif/conformance/test_select.py +++ b/verif/conformance/test_select.py @@ -336,7 +336,7 @@ class AvgPool2dOperator(Operator): """Test selector for the AVG_POOL2D operator.""" name = "avg_pool2d" - param_names = ["shape", "type", "stride", "kernel", "pad"] + param_names = ["shape", "type", "accum_type", "stride", "kernel", "pad"] class BitwiseAndOperator(Operator): @@ -380,7 +380,6 @@ class CLZOperator(Operator): """Test selector for the CLZ operator.""" name = "clz" - param_names = ["shape", "type"] class ConcatOperator(Operator): @@ -407,21 +406,21 @@ class Conv2dOperator(Operator): """Test selector for the CONV2D operator.""" name = "conv2d" - param_names = ["kernel", "shape", "type", "stride", "pad", "dilation"] + param_names = ["kernel", "shape", "type", "accum_type", "stride", "pad", "dilation"] class Conv3dOperator(Operator): """Test selector for the CONV3D operator.""" name = "conv3d" - param_names = ["kernel", "shape", "type", "stride", "pad", "dilation"] + param_names = ["kernel", "shape", "type", "accum_type", "stride", "pad", "dilation"] class DepthwiseConv2dOperator(Operator): """Test selector for the DEPTHWISE_CONV2D operator.""" name = "depthwise_conv2d" - param_names = ["kernel", "shape", "type", "stride", "pad", "dilation"] + param_names = ["kernel", "shape", "type", "accum_type", "stride", "pad", "dilation"] class EqualOperator(Operator): @@ -434,6 +433,7 @@ class FullyConnectedOperator(Operator): """Test selector for the FULLY_CONNECTED operator.""" name = "fully_connected" + param_names = ["shape", "type", "accum_type"] class GatherOperator(Operator): @@ -511,6 +511,7 @@ class MatmulOperator(Operator): """Test selector for the MATMUL operator.""" name = "matmul" + param_names = ["shape", "type", "accum_type"] class MaximumOperator(Operator): @@ -684,7 +685,15 @@ class TransposeConv2dOperator(Operator): """Test selector for the TRANSPOSE_CONV2D operator.""" name = "transpose_conv2d" - param_names = ["kernel", "shape", "type", "stride", "pad", "out_shape"] + param_names = [ + "kernel", + "shape", + "type", + "accum_type", + "stride", + "pad", + "out_shape", + ] def path_params(self, path): """Return a dictionary of params from the test path.""" diff --git a/verif/conformance/tosa_base_profile_ops_info.json b/verif/conformance/tosa_base_profile_ops_info.json index 2af4c47..9dd5592 100644 --- a/verif/conformance/tosa_base_profile_ops_info.json +++ b/verif/conformance/tosa_base_profile_ops_info.json @@ -25,12 +25,12 @@ "5" ], [ + "--target-dtype", + "int32", "--target-shape", "1,2,65533,1", "--target-shape", - "1,65539,1,1,1", - "--target-dtype", - "int32" + "1,65539,1,1,1" ] ], "params": {}, @@ -69,12 +69,12 @@ "5" ], [ + "--target-dtype", + "int32", "--target-shape", "1,2,65533,1", "--target-shape", - "1,65539,1,1,1", - "--target-dtype", - "int32" + "1,65539,1,1,1" ] ], "params": {}, @@ -119,12 +119,12 @@ "2" ], [ + "--target-dtype", + "int8", "--target-shape", "1,3,1,65535", "--target-shape", "1,65538,1,2", - "--target-dtype", - "int8", "--num-rand-permutations", "2" ] @@ -143,6 +143,12 @@ "group": "ew_binary", "generator_args": [ [ + "--target-dtype", + "int8", + "--target-dtype", + "int16", + "--target-dtype", + "int32", "--tensor-dim-range", "16,64", "--target-rank", @@ -153,6 +159,12 @@ "3" ], [ + "--target-dtype", + "int8", + "--target-dtype", + "int16", + "--target-dtype", + "int32", "--tensor-dim-range", "1,16", "--target-rank", @@ -161,12 +173,12 @@ "5" ], [ + "--target-dtype", + "int8", "--target-shape", "1,2,65533,1", "--target-shape", - "1,65539,1,1,1", - "--target-dtype", - "int8" + "1,65539,1,1,1" ] ], "params": {}, @@ -184,6 +196,10 @@ "group": "tensor", "generator_args": [ [ + "--target-dtype", + "int8", + "--target-dtype", + "int16", "--target-shape", "1,49,42,28", "--target-shape", @@ -191,12 +207,12 @@ "--allow-pooling-and-conv-oversizes" ], [ + "--target-dtype", + "int8", "--target-shape", "1,65535,5,1", "--target-shape", "1,3,65537,1", - "--target-dtype", - "int8", "--allow-pooling-and-conv-oversizes" ] ], @@ -224,6 +240,12 @@ "group": "ew_binary", "generator_args": [ [ + "--target-dtype", + "int8", + "--target-dtype", + "int16", + "--target-dtype", + "int32", "--tensor-dim-range", "1,64", "--target-rank", @@ -244,12 +266,12 @@ "5" ], [ + "--target-dtype", + "int8", "--target-shape", "1,2,65536,1", "--target-shape", - "1,65539,1,1,1", - "--target-dtype", - "int8" + "1,65539,1,1,1" ] ], "params": {}, @@ -266,6 +288,12 @@ "group": "ew_unary", "generator_args": [ [ + "--target-dtype", + "int8", + "--target-dtype", + "int16", + "--target-dtype", + "int32", "--tensor-dim-range", "2,64", "--target-rank", @@ -286,12 +314,12 @@ "5" ], [ + "--target-dtype", + "int8", "--target-shape", "1,2,65537,1", "--target-shape", - "1,65538,1,1,1", - "--target-dtype", - "int8" + "1,65538,1,1,1" ] ], "params": {}, @@ -308,6 +336,12 @@ "group": "ew_binary", "generator_args": [ [ + "--target-dtype", + "int8", + "--target-dtype", + "int16", + "--target-dtype", + "int32", "--tensor-dim-range", "3,64", "--target-rank", @@ -328,12 +362,12 @@ "5" ], [ + "--target-dtype", + "int8", "--target-shape", "1,2,65538,1", "--target-shape", - "1,65537,1,1,1", - "--target-dtype", - "int8" + "1,65537,1,1,1" ] ], "params": {}, @@ -350,6 +384,12 @@ "group": "ew_binary", "generator_args": [ [ + "--target-dtype", + "int8", + "--target-dtype", + "int16", + "--target-dtype", + "int32", "--tensor-dim-range", "4,64", "--target-rank", @@ -370,12 +410,12 @@ "5" ], [ + "--target-dtype", + "int8", "--target-shape", "1,2,65536,1", "--target-shape", - "1,65536,1,1,1", - "--target-dtype", - "int8" + "1,65536,1,1,1" ] ], "params": {}, @@ -426,12 +466,12 @@ "5" ], [ + "--target-dtype", + "int8", "--target-shape", "1,1,65535,1,1", "--target-shape", - "1,65538,1,2", - "--target-dtype", - "int8" + "1,65538,1,2" ] ], "params": { @@ -458,6 +498,10 @@ "group": "activation", "generator_args": [ [ + "--target-dtype", + "int8", + "--target-dtype", + "int16", "--tensor-dim-range", "20,64", "--target-rank", @@ -468,6 +512,10 @@ "3" ], [ + "--target-dtype", + "int8", + "--target-dtype", + "int16", "--tensor-dim-range", "1,20", "--target-rank", @@ -476,12 +524,12 @@ "5" ], [ + "--target-dtype", + "int8", "--target-shape", "1,2,1,65536", "--target-shape", - "1,3,65537,1,1", - "--target-dtype", - "int8" + "1,3,65537,1,1" ] ], "params": {}, @@ -520,12 +568,12 @@ "5" ], [ + "--target-dtype", + "int32", "--target-shape", "1,2,65533,1", "--target-shape", - "1,65539,1,1,1", - "--target-dtype", - "int32" + "1,65539,1,1,1" ] ], "params": {}, @@ -580,12 +628,12 @@ "1" ], [ + "--target-dtype", + "int8", "--target-shape", "1,1,65535,1,1", "--target-shape", "1,65538,1,2", - "--target-dtype", - "int8", "--num-const-inputs-concat", "2" ] @@ -614,10 +662,10 @@ "int32" ], [ - "--target-shape", - "3,22,17,1,2", "--target-dtype", - "int32" + "int32", + "--target-shape", + "3,22,17,1,2" ] ], "params": {}, @@ -668,12 +716,12 @@ "5" ], [ + "--target-dtype", + "int8", "--target-shape", "1,1,65537,1,1", "--target-shape", - "1,65530,1,2", - "--target-dtype", - "int8" + "1,65530,1,2" ] ], "params": {}, @@ -690,6 +738,10 @@ "group": "tensor", "generator_args": [ [ + "--target-dtype", + "int8", + "--target-dtype", + "int16", "--target-shape", "1,49,42,28", "--target-shape", @@ -697,12 +749,12 @@ "--allow-pooling-and-conv-oversizes" ], [ + "--target-dtype", + "int8", "--target-shape", "1,65535,4,1", "--target-shape", "1,5,65536,1", - "--target-dtype", - "int8", "--max-conv-dilation", "1", "--allow-pooling-and-conv-oversizes" @@ -735,23 +787,23 @@ "group": "tensor", "generator_args": [ [ - "--target-shape", - "1,9,21,14,1", - "--target-shape", - "1,9,11,12,3", "--target-dtype", "int8", "--target-dtype", "int16", + "--target-shape", + "1,9,21,14,1", + "--target-shape", + "1,9,11,12,3", "--allow-pooling-and-conv-oversizes" ], [ + "--target-dtype", + "int8", "--target-shape", "1,1,65535,3,1", "--target-shape", "1,1,3,65536,1", - "--target-dtype", - "int8", "--max-conv-dilation", "1", "--allow-pooling-and-conv-oversizes" @@ -785,6 +837,10 @@ "group": "tensor", "generator_args": [ [ + "--target-dtype", + "int8", + "--target-dtype", + "int16", "--target-shape", "1,9,44,3", "--target-shape", @@ -792,12 +848,12 @@ "--allow-pooling-and-conv-oversizes" ], [ + "--target-dtype", + "int8", "--target-shape", "1,65532,2,1", "--target-shape", "1,4,65537,1", - "--target-dtype", - "int8", "--max-conv-dilation", "1", "--allow-pooling-and-conv-oversizes" @@ -852,12 +908,12 @@ "5" ], [ + "--target-dtype", + "int32", "--target-shape", "1,2,65501,1", "--target-shape", - "1,65541,1,1,1", - "--target-dtype", - "int32" + "1,65541,1,1,1" ] ], "params": {}, @@ -880,24 +936,24 @@ "int16" ], [ + "--target-dtype", + "int8", "--target-shape", "444,1", "--target-shape", "3,65538", "--tensor-dim-range", - "1,2", - "--target-dtype", - "int8" + "1,2" ], [ + "--target-dtype", + "int8", "--target-shape", "3,16", "--target-shape", "1,23", "--tensor-dim-range", - "100,200", - "--target-dtype", - "int8" + "100,200" ] ], "params": {}, @@ -922,14 +978,14 @@ "4,64" ], [ + "--target-dtype", + "int8", "--target-shape", "2,65533,1", "--target-shape", "1,2,65533", "--target-shape", - "5000,1,1", - "--target-dtype", - "int8" + "5000,1,1" ] ], "params": {}, @@ -974,12 +1030,12 @@ [ "--filter", "^greater$", + "--target-dtype", + "int32", "--target-shape", "1,2,65530,1", "--target-shape", - "1,65539,1,2,1", - "--target-dtype", - "int32" + "1,65539,1,2,1" ] ], "params": {}, @@ -1018,12 +1074,12 @@ "5" ], [ + "--target-dtype", + "int32", "--target-shape", "1,2,65533,1", "--target-shape", - "1,65539,1,3,1", - "--target-dtype", - "int32" + "1,65539,1,3,1" ] ], "params": {}, @@ -1075,12 +1131,12 @@ "5" ], [ + "--target-dtype", + "int8", "--target-shape", "1,1,65536,1,1", "--target-shape", - "1,65531,1,2", - "--target-dtype", - "int8" + "1,65531,1,2" ] ], "params": {}, @@ -1119,12 +1175,12 @@ "5" ], [ + "--target-dtype", + "int32", "--target-shape", "1,65533,1", "--target-shape", - "1,1,65539,1,1", - "--target-dtype", - "int32" + "1,1,65539,1,1" ] ], "params": {}, @@ -1141,6 +1197,8 @@ "group": "ew_binary", "generator_args": [ [ + "--target-dtype", + "bool", "--tensor-dim-range", "1,64", "--target-rank", @@ -1151,6 +1209,8 @@ "3" ], [ + "--target-dtype", + "bool", "--tensor-dim-range", "1,16", "--target-rank", @@ -1159,6 +1219,8 @@ "5" ], [ + "--target-dtype", + "bool", "--target-shape", "1,65538,1", "--target-shape", @@ -1209,12 +1271,12 @@ "5" ], [ + "--target-dtype", + "int8", "--target-shape", "1,65532,1,1", "--target-shape", - "1,2,1,65538", - "--target-dtype", - "int8" + "1,2,1,65538" ] ], "params": {}, @@ -1231,6 +1293,8 @@ "group": "ew_unary", "generator_args": [ [ + "--target-dtype", + "bool", "--tensor-dim-range", "2,64", "--target-rank", @@ -1241,6 +1305,8 @@ "3" ], [ + "--target-dtype", + "bool", "--tensor-dim-range", "1,15", "--target-rank", @@ -1249,6 +1315,8 @@ "5" ], [ + "--target-dtype", + "bool", "--target-shape", "1,65537,1", "--target-shape", @@ -1269,6 +1337,8 @@ "group": "ew_binary", "generator_args": [ [ + "--target-dtype", + "bool", "--tensor-dim-range", "3,64", "--target-rank", @@ -1279,6 +1349,8 @@ "3" ], [ + "--target-dtype", + "bool", "--tensor-dim-range", "1,14", "--target-rank", @@ -1287,6 +1359,8 @@ "5" ], [ + "--target-dtype", + "bool", "--target-shape", "1,65536,1", "--target-shape", @@ -1337,12 +1411,12 @@ "5" ], [ + "--target-dtype", + "int8", "--target-shape", "1,1,65540,3", "--target-shape", - "2,1,65530,1,1", - "--target-dtype", - "int8" + "2,1,65530,1,1" ] ], "params": {}, @@ -1359,6 +1433,8 @@ "group": "ew_binary", "generator_args": [ [ + "--target-dtype", + "bool", "--tensor-dim-range", "4,64", "--target-rank", @@ -1369,6 +1445,8 @@ "3" ], [ + "--target-dtype", + "bool", "--tensor-dim-range", "3,16", "--target-rank", @@ -1377,6 +1455,8 @@ "5" ], [ + "--target-dtype", + "bool", "--target-shape", "1,65539,1", "--target-shape", @@ -1413,14 +1493,14 @@ "1,16" ], [ + "--target-dtype", + "int8", "--target-shape", "1,65532,1", "--target-shape", "1,2,65538", "--target-shape", - "2000,1,1", - "--target-dtype", - "int8" + "2000,1,1" ] ], "params": {}, @@ -1459,12 +1539,12 @@ "5" ], [ + "--target-dtype", + "int32", "--target-shape", "1,2,65533,1", "--target-shape", - "1,65539,1,1,1", - "--target-dtype", - "int32" + "1,65539,1,1,1" ] ], "params": {}, @@ -1481,6 +1561,10 @@ "group": "tensor", "generator_args": [ [ + "--target-dtype", + "int8", + "--target-dtype", + "int16", "--target-shape", "1,4,75,3", "--target-shape", @@ -1488,14 +1572,14 @@ "--allow-pooling-and-conv-oversizes" ], [ + "--target-dtype", + "int8", "--target-shape", "1,65535,5,1", "--target-shape", "1,3,65537,1", "--target-shape", "33333,3,2,1", - "--target-dtype", - "int8", "--allow-pooling-and-conv-oversizes" ] ], @@ -1545,12 +1629,12 @@ "5" ], [ + "--target-dtype", + "int32", "--target-shape", "1,2,65533,1", "--target-shape", - "1,65539,1,1,1", - "--target-dtype", - "int32" + "1,65539,1,1,1" ] ], "params": {}, @@ -1597,12 +1681,12 @@ "5" ], [ + "--target-dtype", + "int8", "--target-shape", "1,2,65500,1", "--target-shape", - "1,68539,1,1,1", - "--target-dtype", - "int8" + "1,68539,1,1,1" ] ], "params": {}, @@ -1651,12 +1735,12 @@ "5" ], [ + "--target-dtype", + "int8", "--target-shape", "3,1,65540,1", "--target-shape", - "1,1,2,1,65531", - "--target-dtype", - "int8" + "1,1,2,1,65531" ] ], "params": {}, @@ -1673,8 +1757,6 @@ "group": "data_layout", "generator_args": [ [ - "--tensor-dim-range", - "1,64", "--target-dtype", "int8", "--target-dtype", @@ -1683,6 +1765,8 @@ "int32", "--target-dtype", "bool", + "--tensor-dim-range", + "1,64", "--target-rank", "1", "--target-rank", @@ -1691,8 +1775,6 @@ "3" ], [ - "--tensor-dim-range", - "1,16", "--target-dtype", "int8", "--target-dtype", @@ -1701,22 +1783,24 @@ "int32", "--target-dtype", "bool", + "--tensor-dim-range", + "1,16", "--target-rank", "4" ], [ + "--target-dtype", + "int16", "--tensor-dim-range", "1,16", "--target-rank", - "5", - "--target-dtype", - "int16" + "5" ], [ - "--target-shape", - "1,1,65537,1", "--target-dtype", - "int8" + "int8", + "--target-shape", + "1,1,65537,1" ] ], "params": {}, @@ -1743,10 +1827,14 @@ "group": "reduction", "generator_args": [ [ + "--target-dtype", + "bool", "--tensor-dim-range", "1,32" ], [ + "--target-dtype", + "bool", "--target-shape", "65530,1,1,1", "--target-shape", @@ -1768,10 +1856,14 @@ "group": "reduction", "generator_args": [ [ + "--target-dtype", + "bool", "--tensor-dim-range", "1,32" ], [ + "--target-dtype", + "bool", "--target-shape", "65530,1,1,1", "--target-shape", @@ -1803,12 +1895,12 @@ "1,32" ], [ + "--target-dtype", + "int8", "--target-shape", "65530,1,1,1", "--target-shape", - "2,65538,1,1", - "--target-dtype", - "int8" + "2,65538,1,1" ] ], "params": {}, @@ -1836,12 +1928,12 @@ "1,24" ], [ + "--target-dtype", + "int8", "--target-shape", "1,1,65538,1", "--target-shape", - "2,65535,1,1", - "--target-dtype", - "int8" + "2,65535,1,1" ] ], "params": {}, @@ -1865,12 +1957,12 @@ "1,32" ], [ + "--target-dtype", + "int32", "--target-shape", "3,65532,1,1", "--target-shape", - "65536,1,1,2", - "--target-dtype", - "int32" + "65536,1,1,2" ] ], "params": {}, @@ -1888,6 +1980,18 @@ "group": "type_conversion", "generator_args": [ [ + "--target-dtype", + "int8", + "--target-dtype", + "int16", + "--target-dtype", + "int32", + "--target-dtype", + "int48", + "--target-dtype", + "uint8", + "--target-dtype", + "uint16", "--tensor-dim-range", "16,32", "--target-rank", @@ -1898,6 +2002,18 @@ "3" ], [ + "--target-dtype", + "int8", + "--target-dtype", + "int16", + "--target-dtype", + "int32", + "--target-dtype", + "int48", + "--target-dtype", + "uint8", + "--target-dtype", + "uint16", "--tensor-dim-range", "1,16", "--target-rank", @@ -1906,12 +2022,12 @@ "5" ], [ + "--target-dtype", + "int8", "--target-shape", "1,2,65533,1", "--target-shape", - "1,65539,1,1", - "--target-dtype", - "int8" + "1,65539,1,1" ] ], "params": { @@ -1967,12 +2083,12 @@ "2" ], [ + "--target-dtype", + "int8", "--target-shape", "1,1,65535,1", "--target-shape", "1,65538,1,2", - "--target-dtype", - "int8", "--num-rand-permutations", "2" ] @@ -1999,10 +2115,10 @@ "int16" ], [ - "--target-shape", - "1,3,16383,1", "--target-dtype", "int8", + "--target-shape", + "1,3,16383,1", "--max-resize-output-dim", "9500" ] @@ -2033,12 +2149,12 @@ "1,32" ], [ + "--target-dtype", + "int8", "--target-shape", "1,1,65535", "--target-shape", - "1,65540,1,2", - "--target-dtype", - "int8" + "1,65540,1,2" ] ], "params": {}, @@ -2063,14 +2179,14 @@ "4,64" ], [ + "--target-dtype", + "int8", "--target-shape", "1,65540,1", "--target-shape", "1,1,65533", "--target-shape", - "3000,7,1", - "--target-dtype", - "int8" + "3000,7,1" ] ], "params": {}, @@ -2121,12 +2237,12 @@ "5" ], [ + "--target-dtype", + "int8", "--target-shape", "1,1,65535,1", "--target-shape", "1,65538,1,2", - "--target-dtype", - "int8", "--num-rand-permutations", "2" ] @@ -2181,12 +2297,12 @@ "2" ], [ + "--target-dtype", + "int8", "--target-shape", "1,1,65535,1", "--target-shape", "1,65538,1,2", - "--target-dtype", - "int8", "--num-rand-permutations", "2" ] @@ -2228,12 +2344,12 @@ "5" ], [ + "--target-dtype", + "int32", "--target-shape", "1,2,65533,1", "--target-shape", - "1,65539,1,1,1", - "--target-dtype", - "int32" + "1,65539,1,1,1" ] ], "params": {}, @@ -2250,6 +2366,10 @@ "group": "ew_binary", "generator_args": [ [ + "--target-dtype", + "int8", + "--target-dtype", + "int16", "--tensor-dim-range", "16,64", "--target-rank", @@ -2260,6 +2380,10 @@ "3" ], [ + "--target-dtype", + "int8", + "--target-dtype", + "int16", "--tensor-dim-range", "1,16", "--target-rank", @@ -2268,12 +2392,12 @@ "5" ], [ + "--target-dtype", + "int8", "--target-shape", "1,2,65536,1", "--target-shape", - "1,65537,1,1,3", - "--target-dtype", - "int8" + "1,65537,1,1,3" ] ], "params": {}, @@ -2298,12 +2422,12 @@ "4,32" ], [ + "--target-dtype", + "int8", "--target-shape", "65533", "--target-shape", - "1,2,65540,1", - "--target-dtype", - "int8" + "1,2,65540,1" ] ], "params": {}, @@ -2352,12 +2476,12 @@ [ "--filter", "^transpose$", + "--target-dtype", + "int8", "--target-shape", "1,65537,1,2", "--target-shape", "65535,1,1,1", - "--target-dtype", - "int8", "--num-rand-permutations", "1" ] @@ -2377,21 +2501,21 @@ "group": "tensor", "generator_args": [ [ + "--target-dtype", + "int8", + "--target-dtype", + "int16", "--target-shape", "1,49,33,1", "--target-shape", "2,11,33,3", - "--target-dtype", - "int16", - "--target-dtype", - "int8", "--allow-pooling-and-conv-oversizes" ], [ - "--target-shape", - "1,65536,1,1", "--target-dtype", "int8", + "--target-shape", + "1,65536,1,1", "--max-conv-dilation", "1" ] @@ -2427,12 +2551,12 @@ "int32" ], [ + "--target-dtype", + "int32", "--target-shape", "3,22,17,1,2", "--target-shape", - "1,65537,2,1", - "--target-dtype", - "int32" + "1,65537,2,1" ] ], "params": {}, diff --git a/verif/conformance/tosa_verif_conformance_generator.py b/verif/conformance/tosa_verif_conformance_generator.py index c057b73..7032ad4 100644 --- a/verif/conformance/tosa_verif_conformance_generator.py +++ b/verif/conformance/tosa_verif_conformance_generator.py @@ -34,12 +34,14 @@ PROFILE_OPS_INFO = { "tosa-bi": { "operator_test_params": "tosa_base_profile_ops_info.json", "framework_tests": "tosa_base_profile_framework_ops_info.json", - "exclude_types": ["float"], + "exclude_types": [], } } LOCATION_REF_MODEL_BINARY = Path("build/reference_model/tosa_reference_model") +DEFAULT_SEED = 42 + class GenConformanceError(Exception): """Generation error reporting exception.""" @@ -90,7 +92,7 @@ def build_op_tests(args, operator, test_params): "-o", str(op_build_dir), "--seed", - "42", + str(args.random_seed), ] ref_cmds = [] @@ -99,12 +101,26 @@ def build_op_tests(args, operator, test_params): # Append extra parameters and run test generator for each set of parameters. for arglist in test_params[operator]["generator_args"]: ref_cmd_pos_test = ref_cmd_base.copy() + ref_cmd_pos_test.extend(["--test-type", "positive"]) ref_cmd_pos_test.extend(arglist) ref_cmds.append(ref_cmd_pos_test) if args.test_type in ["negative", "both"]: + # Get target-dtypes options only to limit tests to those needed + target_dtypes_args = [] + for arglist in test_params[operator]["generator_args"]: + idx = 0 + while idx < len(arglist): + if arglist[idx] == "--target-dtype": + if arglist[idx + 1] not in target_dtypes_args: + target_dtypes_args.extend(arglist[idx : idx + 2]) + idx += 1 # skip over option (and then argument below) + idx += 1 ref_cmd_neg_test = ref_cmd_base.copy() ref_cmd_neg_test.extend(["--test-type", "negative"]) + # Limit sizes of negative tests + ref_cmd_neg_test.extend(["--tensor-dim-range", "1,16"]) + ref_cmd_neg_test.extend(target_dtypes_args) ref_cmds.append(ref_cmd_neg_test) logger.debug(f"Creating {operator} tests with {len(ref_cmds)} parameter(s)") @@ -429,6 +445,13 @@ def parse_args(argv=None): required=True, help="Reference Model directory (must be pre-built)", ) + parser.add_argument( + "--seed", + dest="random_seed", + default=DEFAULT_SEED, + type=int, + help="Random test seed", + ) parser.add_argument( "--framework-tests-directory", dest="framework_tests_dir", @@ -535,6 +558,11 @@ def main(): print(f"Creating conformance tests for TOSA {args.profile} profile") print(f"Output directory: {args.output_dir}") + if args.random_seed != DEFAULT_SEED: + logger.warning( + "Random test seed changed from default, tests will not match official conformance" + ) + args.build_dir = args.build_dir.resolve() logger.debug(f"Creating build directory: {args.build_dir}") args.build_dir.mkdir(parents=True, exist_ok=True) diff --git a/verif/generator/tosa_arg_gen.py b/verif/generator/tosa_arg_gen.py index 69968d3..e0c6cf0 100644 --- a/verif/generator/tosa_arg_gen.py +++ b/verif/generator/tosa_arg_gen.py @@ -1017,7 +1017,7 @@ class TosaArgGen: s_vals = [testGen.rng.choice(range(-5, 0))] else: # Stride must be greater than 1 to force non-integer error - startStride = 1 if error_name != ErrorIf.PoolingOutputShapeNonInteger else 2 + startStride = 1 if error_name != ErrorIf.ConvOutputShapeNonInteger else 2 s_vals = [x for x in range(startStride, testGen.args.max_conv_stride + 1)] strides = {x for x in itertools.product(*([s_vals] * k_rank))} if error_name == ErrorIf.DilationSmallerOne: @@ -1058,18 +1058,14 @@ class TosaArgGen: for d in sorted(list(dilations)): if ( n % sparsity == 0 - # padding must not exceed the kernel size ? - # and p[0] < k[0] and p[1] < k[0] - # and p[2] < k[1] and p[3] < k[1] - # and (k_rank < 3 or (p[4] < k[2] and p[5] < k[2])) - # the padded shape must exceed the kernel size - and (ifm_shape[1] + p[0] + p[1]) > k[0] - and (ifm_shape[2] + p[2] + p[3]) > k[1] - and (k_rank < 3 or ((ifm_shape[3] + p[4] + p[5]) > k[2])) - # the padded shape must exceed the dilation - and (ifm_shape[1] + p[0] + p[1]) > d[0] - and (ifm_shape[2] + p[2] + p[3]) > d[1] - and (k_rank < 3 or ((ifm_shape[3] + p[4] + p[5]) > d[2])) + # the padded shape must exceed the dilation * kernel to get a positive + # sized output shape + and (ifm_shape[1] - 1 + p[0] + p[1]) > d[0] * (k[0] - 1) + and (ifm_shape[2] - 1 + p[2] + p[3]) > d[1] * (k[1] - 1) + and ( + k_rank < 3 + or ((ifm_shape[3] - 1 + p[4] + p[5]) > d[2] * (k[2] - 1)) + ) ): remainders = [] for index in range(k_rank): diff --git a/verif/runner/tosa_refmodel_sut_run.py b/verif/runner/tosa_refmodel_sut_run.py index 2ae7a69..6acaaf4 100644 --- a/verif/runner/tosa_refmodel_sut_run.py +++ b/verif/runner/tosa_refmodel_sut_run.py @@ -40,10 +40,10 @@ class TosaSUTRunner(TosaTestRunner): # Specific debug options for reference model if args.ref_debug: - cmd.extend(["-d ALL", "-l {}".format(args.ref_debug)]) + cmd.extend(["-d", "ALL", "-l", args.ref_debug]) if args.ref_intermediates: - cmd.extend(["-D dump_intermediates=1"]) + cmd.extend(["--dump_intermediates", str(args.ref_intermediates)]) # Run command and interpret tosa graph result via process return codes graphMessage = None diff --git a/verif/tests/test_tosa_refmodel.py b/verif/tests/test_tosa_refmodel.py index b2f33dd..fbe3a7d 100644 --- a/verif/tests/test_tosa_refmodel.py +++ b/verif/tests/test_tosa_refmodel.py @@ -46,6 +46,7 @@ REF_MODEL_TYPE_TO_OUT = { "int16": "i16", "int32": "i32", "float": "float", + "fp16": "f16", } @@ -125,16 +126,19 @@ TEST_PARAMS = [ ("add", "float", 1), ("abs", "int32", 1), ("abs", "float", 1), + ("abs", "fp16", 1), ("negate", "int8", 1), ("negate", "int16", 1), ("negate", "int32", 1), ("negate", "float", 1), + ("negate", "fp16", 1), # One test per axis (shape dimensions) ("concat", "bool", SHAPE_DIMS), ("concat", "int8", SHAPE_DIMS), ("concat", "int16", SHAPE_DIMS), ("concat", "int32", SHAPE_DIMS), ("concat", "float", SHAPE_DIMS), + ("concat", "fp16", SHAPE_DIMS), ] -- cgit v1.2.1