1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
|
# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
# SPDX-License-Identifier: Apache-2.0
"""Tests for module backend/manager."""
from __future__ import annotations
import base64
from pathlib import Path
from typing import Generator
from unittest.mock import MagicMock
import pytest
from mlia.backend.corstone.performance import build_corstone_command
from mlia.backend.corstone.performance import estimate_performance
from mlia.backend.corstone.performance import GenericInferenceOutputParser
from mlia.backend.corstone.performance import get_metrics
from mlia.backend.corstone.performance import PerformanceMetrics
from mlia.backend.errors import BackendExecutionFailed
from mlia.utils.proc import Command
def encode_b64(data: str) -> str:
"""Encode data in base64 format."""
return base64.b64encode(data.encode()).decode()
def valid_fvp_output() -> list[str]:
"""Return valid FVP output that could be succesfully parsed."""
json_data = """[
{
"profiling_group": "Inference",
"count": 1,
"samples": [
{"name": "NPU IDLE", "value": [2]},
{"name": "NPU AXI0_RD_DATA_BEAT_RECEIVED", "value": [4]},
{"name": "NPU AXI0_WR_DATA_BEAT_WRITTEN", "value": [5]},
{"name": "NPU AXI1_RD_DATA_BEAT_RECEIVED", "value": [6]},
{"name": "NPU ACTIVE", "value": [1]},
{"name": "NPU TOTAL", "value": [3]}
]
}
]"""
return [
"some output",
f"<metrics>{encode_b64(json_data)}</metrics>",
"some_output",
]
def test_generic_inference_output_parser_success() -> None:
"""Test successful generic inference output parsing."""
output_parser = GenericInferenceOutputParser()
for line in valid_fvp_output():
output_parser(line)
assert output_parser.get_metrics() == PerformanceMetrics(1, 2, 3, 4, 5, 6)
@pytest.mark.parametrize(
"wrong_fvp_output",
[
[],
["NPU IDLE: 123"],
["<metrics>123</metrics>"],
],
)
def test_generic_inference_output_parser_failure(wrong_fvp_output: list[str]) -> None:
"""Test unsuccessful generic inference output parsing."""
output_parser = GenericInferenceOutputParser()
for line in wrong_fvp_output:
output_parser(line)
with pytest.raises(ValueError, match="Unable to parse output and get metrics"):
output_parser.get_metrics()
@pytest.mark.parametrize(
"backend_path, fvp, target, mac, model, profile, expected_command",
[
[
Path("backend_path"),
"corstone-300",
"ethos-u55",
256,
Path("model.tflite"),
"default",
Command(
[
"backend_path/FVP_Corstone_SSE-300_Ethos-U55",
"-a",
"apps/backends/applications/"
"inference_runner-sse-300-22.08.02-ethos-U55-Default-noTA/"
"ethos-u-inference_runner.axf",
"--data",
"model.tflite@0x90000000",
"-C",
"ethosu.num_macs=256",
"-C",
"mps3_board.telnetterminal0.start_telnet=0",
"-C",
"mps3_board.uart0.out_file='-'",
"-C",
"mps3_board.uart0.shutdown_on_eot=1",
"-C",
"mps3_board.visualisation.disable-visualisation=1",
"--stat",
]
),
],
],
)
def test_build_corsone_command(
monkeypatch: pytest.MonkeyPatch,
backend_path: Path,
fvp: str,
target: str,
mac: int,
model: Path,
profile: str,
expected_command: Command,
) -> None:
"""Test function build_corstone_command."""
monkeypatch.setattr(
"mlia.backend.corstone.performance.get_mlia_resources", lambda: Path("apps")
)
command = build_corstone_command(backend_path, fvp, target, mac, model, profile)
assert command == expected_command
def test_get_metrics_wrong_fvp() -> None:
"""Test that command construction should fail for wrong FVP."""
with pytest.raises(
BackendExecutionFailed, match=r"Unable to construct a command line for some_fvp"
):
get_metrics(
Path("backend_path"),
"some_fvp",
"ethos-u55",
256,
Path("model.tflite"),
)
def test_estimate_performance(monkeypatch: pytest.MonkeyPatch) -> None:
"""Test function estimate_performance."""
mock_repository = MagicMock()
mock_repository.get_backend_settings.return_value = Path("backend_path"), {
"profile": "default"
}
monkeypatch.setattr(
"mlia.backend.corstone.performance.get_backend_repository",
lambda: mock_repository,
)
def command_output_mock(_command: Command) -> Generator[str, None, None]:
"""Mock FVP output."""
yield from valid_fvp_output()
monkeypatch.setattr("mlia.utils.proc.command_output", command_output_mock)
result = estimate_performance(
"ethos-u55", 256, Path("model.tflite"), "corstone-300"
)
assert result == PerformanceMetrics(1, 2, 3, 4, 5, 6)
mock_repository.get_backend_settings.assert_called_once()
|
