MLIA-136 Add flaky dependency for unstable tests

- https://github.com/box/flaky
- Is set to re-run tests marked with @flaky
- Provides a report on failures
- Add flaky guard to tests/test_nn_tensorflow_optimizations_clustering.py

Signed-off-by: Annie Tallund <annie.tallund@arm.com>
Change-Id: I6795fd8bb2c38be6513f3689c3eeb805e7976add

3 files changed, 11 insertions, 3 deletions

diff --git a/setup.cfg b/setup.cfg
index 5a68b6b..a139bb4 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -32,6 +32,7 @@ install_requires =
     tensorflow~=2.12.1
     tensorflow-model-optimization~=0.7.5
     ethos-u-vela~=3.9.0
+    flaky~=3.7.0
     requests~=2.31.0
     rich~=13.5.2
     tomli~=2.0.1 ; python_version<"3.11"
diff --git a/tests/test_nn_tensorflow_optimizations_clustering.py b/tests/test_nn_tensorflow_optimizations_clustering.py
index ba7aea3..d3c0da6 100644
--- a/tests/test_nn_tensorflow_optimizations_clustering.py
+++ b/tests/test_nn_tensorflow_optimizations_clustering.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
 # SPDX-License-Identifier: Apache-2.0
 """Test for module optimizations/clustering."""
 from __future__ import annotations
@@ -8,6 +8,7 @@ from pathlib import Path
 
 import pytest
 import tensorflow as tf
+from flaky import flaky
 
 from mlia.nn.tensorflow.optimizations.clustering import Clusterer
 from mlia.nn.tensorflow.optimizations.clustering import ClusteringConfiguration
@@ -81,6 +82,12 @@ def _test_sparsity(
     assert num_sparse_layers == expected_num_sparse_layers
 
 
+# This test fails sporadically for stochastic reasons, due to a threshold not being met.
+# Re-running the test will help. We are yet to find a more deterministic approach
+# to run the test, and in the meantime we classify it as a known issue.
+# Additionally, flaky is (as of 2023) untyped and thus we need to silence the
+# warning from mypy.
+@flaky  # type: ignore
 @pytest.mark.parametrize("target_num_clusters", (32, 4))
 @pytest.mark.parametrize("sparsity_aware", (False, True))
 @pytest.mark.parametrize("layers_to_cluster", (["conv1"], ["conv1", "conv2"], None))
diff --git a/tox.ini b/tox.ini
index 94c3b69..6e51df0 100644
--- a/tox.ini
+++ b/tox.ini
@@ -9,7 +9,7 @@ description = Run the unit tests.
 deps =
     pytest==7.4.0
 commands =
-    pytest {posargs:tests/}
+    pytest --no-success-flaky-report {posargs:tests/}
 
 [testenv:{e2e,e2e_setup}]
 description = Run the end-to-end tests.
@@ -37,7 +37,7 @@ deps =
     {[testenv:test]deps}
     pytest-cov==4.1.0
 commands =
-    pytest --cov=mlia --cov-report term-missing --cov-fail-under=95 tests/
+    pytest --cov=mlia --cov-report term-missing --cov-fail-under=95 --no-success-flaky-report tests/
 
 [testenv:{lint,lint_setup}]
 description = Run and setup the pre-commit hooks.