From 60e97eb8711eee9a682bc58820ca3e218d0baa56 Mon Sep 17 00:00:00 2001 From: Annie Tallund Date: Wed, 22 Nov 2023 16:08:12 +0100 Subject: MLIA-136 Add flaky dependency for unstable tests - https://github.com/box/flaky - Is set to re-run tests marked with @flaky - Provides a report on failures - Add flaky guard to tests/test_nn_tensorflow_optimizations_clustering.py Signed-off-by: Annie Tallund Change-Id: I6795fd8bb2c38be6513f3689c3eeb805e7976add --- setup.cfg | 1 + tests/test_nn_tensorflow_optimizations_clustering.py | 9 ++++++++- tox.ini | 4 ++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 5a68b6b..a139bb4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,6 +32,7 @@ install_requires = tensorflow~=2.12.1 tensorflow-model-optimization~=0.7.5 ethos-u-vela~=3.9.0 + flaky~=3.7.0 requests~=2.31.0 rich~=13.5.2 tomli~=2.0.1 ; python_version<"3.11" diff --git a/tests/test_nn_tensorflow_optimizations_clustering.py b/tests/test_nn_tensorflow_optimizations_clustering.py index ba7aea3..d3c0da6 100644 --- a/tests/test_nn_tensorflow_optimizations_clustering.py +++ b/tests/test_nn_tensorflow_optimizations_clustering.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Test for module optimizations/clustering.""" from __future__ import annotations @@ -8,6 +8,7 @@ from pathlib import Path import pytest import tensorflow as tf +from flaky import flaky from mlia.nn.tensorflow.optimizations.clustering import Clusterer from mlia.nn.tensorflow.optimizations.clustering import ClusteringConfiguration @@ -81,6 +82,12 @@ def _test_sparsity( assert num_sparse_layers == expected_num_sparse_layers +# This test fails sporadically for stochastic reasons, due to a threshold not being met. +# Re-running the test will help. We are yet to find a more deterministic approach +# to run the test, and in the meantime we classify it as a known issue. +# Additionally, flaky is (as of 2023) untyped and thus we need to silence the +# warning from mypy. +@flaky # type: ignore @pytest.mark.parametrize("target_num_clusters", (32, 4)) @pytest.mark.parametrize("sparsity_aware", (False, True)) @pytest.mark.parametrize("layers_to_cluster", (["conv1"], ["conv1", "conv2"], None)) diff --git a/tox.ini b/tox.ini index 94c3b69..6e51df0 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,7 @@ description = Run the unit tests. deps = pytest==7.4.0 commands = - pytest {posargs:tests/} + pytest --no-success-flaky-report {posargs:tests/} [testenv:{e2e,e2e_setup}] description = Run the end-to-end tests. @@ -37,7 +37,7 @@ deps = {[testenv:test]deps} pytest-cov==4.1.0 commands = - pytest --cov=mlia --cov-report term-missing --cov-fail-under=95 tests/ + pytest --cov=mlia --cov-report term-missing --cov-fail-under=95 --no-success-flaky-report tests/ [testenv:{lint,lint_setup}] description = Run and setup the pre-commit hooks. -- cgit v1.2.1