1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
|
# SPDX-FileCopyrightText: Copyright 2022-2024, Arm Limited and/or its affiliates.
# SPDX-License-Identifier: Apache-2.0
"""Vela performance module."""
from __future__ import annotations
import csv
import logging
import os
from collections import Counter
from dataclasses import dataclass
from dataclasses import fields
from pathlib import Path
import numpy as np
from mlia.backend.vela.compiler import VelaCompiler
from mlia.backend.vela.compiler import VelaCompilerOptions
from mlia.backend.vela.compiler import VelaSummary
logger = logging.getLogger(__name__)
@dataclass
class PerformanceMetrics: # pylint: disable=too-many-instance-attributes
"""Contains all the performance metrics Vela generates in a run."""
npu_cycles: int
sram_access_cycles: int
dram_access_cycles: int
on_chip_flash_access_cycles: int
off_chip_flash_access_cycles: int
total_cycles: int
batch_inference_time: float
inferences_per_second: float
batch_size: int
sram_memory_area_size: float
dram_memory_area_size: float
on_chip_flash_memory_area_size: float
off_chip_flash_memory_area_size: float
layerwise_performance_info: LayerwisePerfInfo
@dataclass
class LayerPerfInfo: # pylint: disable=too-many-instance-attributes
"""Contains metrics from a row from the per-layer csv file from Vela."""
name: str
tflite_operator: str
sram_usage: int
op_cycles: int
npu_cycles: int
sram_access_cycles: int
dram_access_cycles: int
on_chip_flash_access_cycles: int
off_chip_flash_access_cycles: int
mac_count: int
util_mac_percentage: float
def __repr__(self) -> str:
"""Return String Representation of LayerPerfInfo object."""
header_values = {key: value for key, value, _ in layer_metrics}
string_to_check = ""
for field in fields(self):
string_to_check += (
f"{header_values[field.name]}: {getattr(self, field.name)}, "
)
return string_to_check
@dataclass
class LayerwisePerfInfo:
"""Contains all the per-layer metrics from the per-layer csv file from Vela."""
layerwise_info: list[LayerPerfInfo]
def __repr__(self) -> str:
"""Return String Representation of LayerwisePerfInfo object."""
strings_to_check_layerwise_object = ""
for layer in self.layerwise_info:
string_to_check = repr(layer)
strings_to_check_layerwise_object += string_to_check
return strings_to_check_layerwise_object
complete_layer_metrics = [
("tflite_operator", "TFLite_operator", "TFLite Operator"),
("nng_operator", "NNG Operator", "NNG Operator"),
("sram_usage", "SRAM Usage", "SRAM Usage"),
("peak_percentage", "Peak%", "Peak SRAM Usage (%)"),
("op_cycles", "Op Cycles", "OP Cycles"),
("network_percentage_1", "Network%", "OP Cycles in Network (%)"),
("npu_cycles", "NPU", "NPU Cycles"),
("sram_access_cycles", "SRAM AC", "SRAM AC"),
("dram_access_cycles", "DRAM AC", "DRAM AC"),
("on_chip_flash_access_cycles", "OnFlash AC", "OnFlash AC"),
("off_chip_flash_access_cycles", "OffFlash AC", "OffFlash AC"),
("mac_count", "MAC Count", "MAC Count"),
("network_percentage_2", "Network% (1)", "MAC Count in Network (%)"),
("util_mac_percentage", "Util%", "MAC Util (%)"),
("name", "Name", "Layer Name"),
]
OUTPUT_METRICS = [field.name for field in fields(LayerPerfInfo)]
layer_metrics = [
layer_metric
for layer_metric in complete_layer_metrics
if layer_metric[0] in OUTPUT_METRICS
]
layer_metrics.sort(key=lambda e: OUTPUT_METRICS.index(e[0]))
def parse_layerwise_perf_csv( # pylint: disable=too-many-locals
vela_csv_file: Path, metrics: list
) -> LayerwisePerfInfo:
"""Parse the per-layer csv file from backend vela."""
if not vela_csv_file.is_file():
raise FileNotFoundError(f"CSV File not found at {vela_csv_file}\n")
layerwise_info = [] # type: list[LayerPerfInfo]
with open(vela_csv_file, encoding="UTF-8") as csv_file:
layerwise_reader = csv.reader(csv_file, delimiter=",")
try:
headers = list(next(layerwise_reader))
except StopIteration:
return LayerwisePerfInfo(layerwise_info=layerwise_info)
headers_to_check_cpu_ops = headers.copy()
multiple_header_count = Counter(headers)
# Deal with multiple of the same values in CSV header.
for idx, header in enumerate(reversed(headers)):
if multiple_header_count[header] > 1:
headers[len(headers) - idx - 1] = (
headers[len(headers) - idx - 1]
+ " ("
+ str(multiple_header_count[header] - 1)
+ ")"
)
multiple_header_count[header] -= 1
for row in layerwise_reader:
row_as_dict = dict(zip(headers, row))
if row == headers_to_check_cpu_ops:
continue
try:
# pylint: disable=eval-used
key_types = {
field.name: eval(field.type) # type: ignore # nosec
for field in fields(LayerPerfInfo)
}
# pylint: enable=eval-used
ids_to_metrics = {}
for key, title, _ in metrics:
try:
ids_to_metrics[key] = key_types[key](row_as_dict[title])
except ValueError as err:
if "invalid literal for int() with base 10" in str(err):
ids_to_metrics[key] = key_types[key](
float(row_as_dict[title])
)
else:
raise
layerwise_info.append(LayerPerfInfo(**ids_to_metrics))
except KeyError as err:
raise KeyError("Generated CSV missing expected headers") from err
return LayerwisePerfInfo(layerwise_info=layerwise_info)
def estimate_performance(
model_path: Path, compiler_options: VelaCompilerOptions
) -> PerformanceMetrics:
"""Return performance estimations for the model/target.
Logic for this function comes from Vela module stats_writer.py
"""
logger.debug(
"Estimate performance for the model %s on %s",
model_path,
compiler_options.accelerator_config,
)
vela_compiler = VelaCompiler(compiler_options)
if Path(
Path(compiler_options.output_dir).as_posix()
+ "/"
+ model_path.stem
+ "_summary_"
+ compiler_options.system_config
+ ".csv"
).is_file():
summary_data, _ = vela_compiler.compile_model(model_path, True)
else:
summary_data, _ = vela_compiler.compile_model(model_path)
output_dir = compiler_options.output_dir
csv_paths = [entry for entry in os.listdir(output_dir) if "per-layer.csv" in entry]
model_name = str(model_path.stem)
csv_file_found = None
for path in csv_paths:
if model_name in path:
csv_file_found = path
if csv_file_found is None:
raise FileNotFoundError("Vela per-layer CSV file not found")
csv_path = Path(output_dir) / csv_file_found
layerwise_performance_info = parse_layerwise_perf_csv(
vela_csv_file=csv_path, metrics=layer_metrics
)
return _performance_metrics(layerwise_performance_info, summary_data)
def _performance_metrics(
layerwise_performance_info: LayerwisePerfInfo, summary_data: VelaSummary
) -> PerformanceMetrics:
"""Return performance metrics for optimized model."""
midpoint_fps = np.nan
midpoint_inference_time = summary_data.cycles_total / summary_data.core_clock
if midpoint_inference_time > 0:
midpoint_fps = 1 / midpoint_inference_time
return PerformanceMetrics(
npu_cycles=int(summary_data.cycles_npu),
sram_access_cycles=int(summary_data.cycles_sram_access),
dram_access_cycles=int(summary_data.cycles_dram_access),
on_chip_flash_access_cycles=int(summary_data.cycles_on_chip_flash_access),
off_chip_flash_access_cycles=int(summary_data.cycles_off_chip_flash_access),
total_cycles=int(summary_data.cycles_total),
batch_inference_time=midpoint_inference_time * 1000,
inferences_per_second=midpoint_fps,
batch_size=summary_data.batch_size,
sram_memory_area_size=float(summary_data.sram_memory_used),
dram_memory_area_size=float(summary_data.dram_memory_used),
on_chip_flash_memory_area_size=float(summary_data.on_chip_flash_memory_used),
off_chip_flash_memory_area_size=float(summary_data.off_chip_flash_memory_used),
layerwise_performance_info=layerwise_performance_info,
)
|
