aboutsummaryrefslogtreecommitdiff
path: root/src/mlia
diff options
context:
space:
mode:
Diffstat (limited to 'src/mlia')
-rw-r--r--src/mlia/backend/vela/compiler.py9
-rw-r--r--src/mlia/backend/vela/performance.py151
-rw-r--r--src/mlia/core/workflow.py3
-rw-r--r--src/mlia/target/ethos_u/advisor.py3
-rw-r--r--src/mlia/target/ethos_u/performance.py46
-rw-r--r--src/mlia/target/ethos_u/reporters.py109
6 files changed, 285 insertions, 36 deletions
diff --git a/src/mlia/backend/vela/compiler.py b/src/mlia/backend/vela/compiler.py
index b591056..fe9e365 100644
--- a/src/mlia/backend/vela/compiler.py
+++ b/src/mlia/backend/vela/compiler.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2024, Arm Limited and/or its affiliates.
# SPDX-License-Identifier: Apache-2.0
"""Vela compiler wrapper module."""
from __future__ import annotations
@@ -90,7 +90,7 @@ class VelaCompilerOptions: # pylint: disable=too-many-instance-attributes
tensor_allocator: TensorAllocatorType = "HillClimb"
cpu_tensor_alignment: int = Tensor.AllocationQuantum
optimization_strategy: OptimizationStrategyType = "Performance"
- output_dir: str = "output"
+ output_dir: Path = Path("output")
recursion_limit: int = 1000
@@ -251,6 +251,7 @@ class VelaCompiler: # pylint: disable=too-many-instance-attributes
verbose_register_command_stream=False,
verbose_operators=False,
verbose_weights=False,
+ verbose_performance=True,
show_cpu_operations=False,
tensor_allocator=self.tensor_allocator,
timing=False,
@@ -258,6 +259,10 @@ class VelaCompiler: # pylint: disable=too-many-instance-attributes
cpu_tensor_alignment=self.cpu_tensor_alignment,
)
+ def return_compiler_options(self) -> CompilerOptions:
+ """Return CompilerOptions instance for test purposes."""
+ return self._compiler_options()
+
def resolve_compiler_config(
vela_compiler_options: VelaCompilerOptions,
diff --git a/src/mlia/backend/vela/performance.py b/src/mlia/backend/vela/performance.py
index a548b26..72a8ceb 100644
--- a/src/mlia/backend/vela/performance.py
+++ b/src/mlia/backend/vela/performance.py
@@ -1,11 +1,16 @@
-# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2024, Arm Limited and/or its affiliates.
# SPDX-License-Identifier: Apache-2.0
"""Vela performance module."""
from __future__ import annotations
+import csv
import logging
+import os
+from collections import Counter
from dataclasses import dataclass
+from dataclasses import fields
from pathlib import Path
+from pydoc import locate
import numpy as np
from ethosu.vela.npu_performance import PassCycles
@@ -37,6 +42,130 @@ class PerformanceMetrics: # pylint: disable=too-many-instance-attributes
dram_memory_area_size: int
on_chip_flash_memory_area_size: int
off_chip_flash_memory_area_size: int
+ layerwise_performance_info: LayerwisePerfInfo
+
+
+@dataclass
+class LayerPerfInfo: # pylint: disable=too-many-instance-attributes
+ """Contains metrics from a row from the per-layer csv file from Vela."""
+
+ name: str
+ tflite_operator: str
+ sram_usage: int
+ op_cycles: int
+ npu_cycles: int
+ sram_access_cycles: int
+ dram_access_cycles: int
+ on_chip_flash_access_cycles: int
+ off_chip_flash_access_cycles: int
+ mac_count: int
+ util_mac_percentage: float
+
+ def __repr__(self) -> str:
+ """Return String Representation of LayerPerfInfo object."""
+ header_values = {key: value for key, value, _ in layer_metrics}
+ string_to_check = ""
+ for field in fields(self):
+ string_to_check += (
+ f"{header_values[field.name]}: {getattr(self, field.name)}, "
+ )
+ return string_to_check
+
+
+@dataclass
+class LayerwisePerfInfo:
+ """Contains all the per-layer metrics from the per-layer csv file from Vela."""
+
+ layerwise_info: list[LayerPerfInfo]
+
+ def __repr__(self) -> str:
+ """Return String Representation of LayerwisePerfInfo object."""
+ strings_to_check_layerwise_object = ""
+ for layer in self.layerwise_info:
+ string_to_check = repr(layer)
+ strings_to_check_layerwise_object += string_to_check
+ return strings_to_check_layerwise_object
+
+
+complete_layer_metrics = [
+ ("tflite_operator", "TFLite_operator", "TFLite Operator"),
+ ("nng_operator", "NNG Operator", "NNG Operator"),
+ ("sram_usage", "SRAM Usage", "SRAM Usage"),
+ ("peak_percentage", "Peak%", "Peak SRAM Usage (%)"),
+ ("op_cycles", "Op Cycles", "OP Cycles"),
+ ("network_percentage_1", "Network%", "OP Cycles in Network (%)"),
+ ("npu_cycles", "NPU", "NPU Cycles"),
+ ("sram_access_cycles", "SRAM AC", "SRAM AC"),
+ ("dram_access_cycles", "DRAM AC", "DRAM AC"),
+ ("on_chip_flash_access_cycles", "OnFlash AC", "OnFlash AC"),
+ ("off_chip_flash_access_cycles", "OffFlash AC", "OffFlash AC"),
+ ("mac_count", "MAC Count", "MAC Count"),
+ ("network_percentage_2", "Network% (1)", "MAC Count in Network (%)"),
+ ("util_mac_percentage", "Util%", "MAC Util (%)"),
+ ("name", "Name", "Layer Name"),
+]
+
+OUTPUT_METRICS = [field.name for field in fields(LayerPerfInfo)]
+
+layer_metrics = [
+ layer_metric
+ for layer_metric in complete_layer_metrics
+ if layer_metric[0] in OUTPUT_METRICS
+]
+layer_metrics.sort(key=lambda e: OUTPUT_METRICS.index(e[0]))
+
+
+def parse_layerwise_perf_csv( # pylint: disable=too-many-locals
+ vela_csv_file: Path, metrics: list
+) -> LayerwisePerfInfo:
+ """Parse the per-layer csv file from backend vela."""
+ if not vela_csv_file.is_file():
+ raise FileNotFoundError(f"CSV File not found at {vela_csv_file}\n")
+ layerwise_info = [] # type: list[LayerPerfInfo]
+ with open(vela_csv_file, encoding="UTF-8") as csv_file:
+ layerwise_reader = csv.reader(csv_file, delimiter=",")
+ try:
+ headers = list(next(layerwise_reader))
+ except StopIteration:
+ return LayerwisePerfInfo(layerwise_info=layerwise_info)
+ headers_to_check_cpu_ops = headers.copy()
+ multiple_header_count = Counter(headers)
+ # Deal with multiple of the same values in CSV header.
+ for idx, header in enumerate(reversed(headers)):
+ if multiple_header_count[header] > 1:
+ headers[len(headers) - idx - 1] = (
+ headers[len(headers) - idx - 1]
+ + " ("
+ + str(multiple_header_count[header] - 1)
+ + ")"
+ )
+ multiple_header_count[header] -= 1
+ for row in layerwise_reader:
+ row_as_dict = dict(zip(headers, row))
+ if row == headers_to_check_cpu_ops:
+ continue
+ try:
+ key_types = {
+ field.name: locate(str(field.type))
+ for field in fields(LayerPerfInfo)
+ }
+ ids_to_metrics = {}
+ for key, title, _ in metrics:
+ try:
+ ids_to_metrics[key] = key_types[key]( # type: ignore
+ row_as_dict[title]
+ )
+ except ValueError as err:
+ if "invalid literal for int() with base 10" in str(err):
+ ids_to_metrics[key] = key_types[key]( # type: ignore
+ float(row_as_dict[title])
+ )
+ else:
+ raise
+ layerwise_info.append(LayerPerfInfo(**ids_to_metrics))
+ except KeyError as err:
+ raise KeyError("Generated CSV missing expected headers") from err
+ return LayerwisePerfInfo(layerwise_info=layerwise_info)
def estimate_performance(
@@ -61,11 +190,26 @@ def estimate_performance(
)
optimized_model = vela_compiler.compile_model(initial_model)
+ output_dir = optimized_model.compiler_options.output_dir
+ csv_paths = [entry for entry in os.listdir(output_dir) if "per-layer.csv" in entry]
+ model_name = str(model_path.stem)
+ csv_file_found = None
+ for path in csv_paths:
+ if model_name in path:
+ csv_file_found = path
+ if csv_file_found is None:
+ raise FileNotFoundError("Vela per-layer CSV file not found")
+ csv_path = Path(output_dir) / csv_file_found
+ layerwise_performance_info = parse_layerwise_perf_csv(
+ vela_csv_file=csv_path, metrics=layer_metrics
+ )
- return _performance_metrics(optimized_model)
+ return _performance_metrics(layerwise_performance_info, optimized_model)
-def _performance_metrics(optimized_model: OptimizedModel) -> PerformanceMetrics:
+def _performance_metrics(
+ layerwise_performance_info: LayerwisePerfInfo, optimized_model: OptimizedModel
+) -> PerformanceMetrics:
"""Return performance metrics for optimized model."""
cycles = optimized_model.nng.cycles
@@ -96,4 +240,5 @@ def _performance_metrics(optimized_model: OptimizedModel) -> PerformanceMetrics:
dram_memory_area_size=memory_usage(MemArea.Dram),
on_chip_flash_memory_area_size=memory_usage(MemArea.OnChipFlash),
off_chip_flash_memory_area_size=memory_usage(MemArea.OffChipFlash),
+ layerwise_performance_info=layerwise_performance_info,
)
diff --git a/src/mlia/core/workflow.py b/src/mlia/core/workflow.py
index 9f8ac83..c645857 100644
--- a/src/mlia/core/workflow.py
+++ b/src/mlia/core/workflow.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2024, Arm Limited and/or its affiliates.
# SPDX-License-Identifier: Apache-2.0
"""Module for executors.
@@ -114,6 +114,7 @@ class DefaultWorkflowExecutor(WorkflowExecutor):
self.before_start()
collected_data = self.collect_data()
+
analyzed_data = self.analyze_data(collected_data)
self.produce_advice(analyzed_data)
diff --git a/src/mlia/target/ethos_u/advisor.py b/src/mlia/target/ethos_u/advisor.py
index 9f5b3a6..b5932d0 100644
--- a/src/mlia/target/ethos_u/advisor.py
+++ b/src/mlia/target/ethos_u/advisor.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2024, Arm Limited and/or its affiliates.
# SPDX-License-Identifier: Apache-2.0
"""Ethos-U MLIA module."""
from __future__ import annotations
@@ -44,6 +44,7 @@ class EthosUInferenceAdvisor(DefaultInferenceAdvisor):
"""Return list of the data collectors."""
model = self.get_model(context)
target_config = self._get_target_config(context)
+ target_config.compiler_options.output_dir = context.output_dir # type: ignore
backends = self._get_backends(context)
collectors: list[DataCollector] = []
diff --git a/src/mlia/target/ethos_u/performance.py b/src/mlia/target/ethos_u/performance.py
index a0526e4..8decb75 100644
--- a/src/mlia/target/ethos_u/performance.py
+++ b/src/mlia/target/ethos_u/performance.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2024, Arm Limited and/or its affiliates.
# SPDX-License-Identifier: Apache-2.0
"""Performance estimation."""
from __future__ import annotations
@@ -13,6 +13,7 @@ import mlia.backend.vela.compiler as vela_comp
import mlia.backend.vela.performance as vela_perf
from mlia.backend.corstone import is_corstone_backend
from mlia.backend.corstone.performance import estimate_performance
+from mlia.backend.vela.performance import LayerwisePerfInfo
from mlia.core.context import Context
from mlia.core.performance import PerformanceEstimator
from mlia.nn.select import OptimizationSettings
@@ -95,16 +96,23 @@ class PerformanceMetrics:
target_config: EthosUConfiguration
npu_cycles: NPUCycles | None
memory_usage: MemoryUsage | None
+ layerwise_perf_info: LayerwisePerfInfo | None
def in_kilobytes(self) -> PerformanceMetrics:
"""Return metrics with memory usage in KiB."""
if self.memory_usage is None:
return PerformanceMetrics(
- self.target_config, self.npu_cycles, self.memory_usage
+ self.target_config,
+ self.npu_cycles,
+ self.memory_usage,
+ self.layerwise_perf_info,
)
return PerformanceMetrics(
- self.target_config, self.npu_cycles, self.memory_usage.in_kilobytes()
+ self.target_config,
+ self.npu_cycles,
+ self.memory_usage.in_kilobytes(),
+ self.layerwise_perf_info,
)
@@ -119,7 +127,9 @@ class OptimizationPerformanceMetrics:
class VelaPerformanceEstimator(
- PerformanceEstimator[Union[Path, ModelConfiguration], MemoryUsage]
+ PerformanceEstimator[
+ Union[Path, ModelConfiguration], tuple[MemoryUsage, LayerwisePerfInfo]
+ ]
):
"""Vela based performance estimator."""
@@ -128,7 +138,9 @@ class VelaPerformanceEstimator(
self.context = context
self.target = target_config
- def estimate(self, model: Path | ModelConfiguration) -> MemoryUsage:
+ def estimate(
+ self, model: Path | ModelConfiguration
+ ) -> tuple[MemoryUsage, LayerwisePerfInfo]:
"""Estimate performance."""
with log_action("Getting the memory usage metrics ..."):
model_path = (
@@ -141,12 +153,15 @@ class VelaPerformanceEstimator(
model_path, self.target.compiler_options
)
- return MemoryUsage(
- vela_perf_metrics.sram_memory_area_size,
- vela_perf_metrics.dram_memory_area_size,
- vela_perf_metrics.unknown_memory_area_size,
- vela_perf_metrics.on_chip_flash_memory_area_size,
- vela_perf_metrics.off_chip_flash_memory_area_size,
+ return (
+ MemoryUsage(
+ vela_perf_metrics.sram_memory_area_size,
+ vela_perf_metrics.dram_memory_area_size,
+ vela_perf_metrics.unknown_memory_area_size,
+ vela_perf_metrics.on_chip_flash_memory_area_size,
+ vela_perf_metrics.off_chip_flash_memory_area_size,
+ ),
+ vela_perf_metrics.layerwise_performance_info,
)
@@ -238,12 +253,15 @@ class EthosUPerformanceEstimator(
memory_usage = None
npu_cycles = None
+ layerwise_perf_info = None
for backend in self.backends:
if backend == "vela":
vela_estimator = VelaPerformanceEstimator(
self.context, self.target_config
)
- memory_usage = vela_estimator.estimate(tflite_model)
+ memory_usage, layerwise_perf_info = vela_estimator.estimate(
+ tflite_model
+ )
elif is_corstone_backend(backend):
corstone_estimator = CorstonePerformanceEstimator(
self.context, self.target_config, backend
@@ -256,4 +274,6 @@ class EthosUPerformanceEstimator(
backend,
)
- return PerformanceMetrics(self.target_config, npu_cycles, memory_usage)
+ return PerformanceMetrics(
+ self.target_config, npu_cycles, memory_usage, layerwise_perf_info
+ )
diff --git a/src/mlia/target/ethos_u/reporters.py b/src/mlia/target/ethos_u/reporters.py
index 711f036..b747ce5 100644
--- a/src/mlia/target/ethos_u/reporters.py
+++ b/src/mlia/target/ethos_u/reporters.py
@@ -1,14 +1,16 @@
-# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2024, Arm Limited and/or its affiliates.
# SPDX-License-Identifier: Apache-2.0
"""Reports module."""
from __future__ import annotations
from collections import defaultdict
+from dataclasses import fields
from typing import Any
from typing import Callable
from mlia.backend.vela.compat import Operator
from mlia.backend.vela.compat import Operators
+from mlia.backend.vela.performance import layer_metrics
from mlia.core.advice_generation import Advice
from mlia.core.reporters import report_advice
from mlia.core.reporting import BytesCell
@@ -16,6 +18,7 @@ from mlia.core.reporting import Cell
from mlia.core.reporting import ClockCell
from mlia.core.reporting import Column
from mlia.core.reporting import CompoundFormatter
+from mlia.core.reporting import CompoundReport
from mlia.core.reporting import CyclesCell
from mlia.core.reporting import Format
from mlia.core.reporting import NestedReport
@@ -237,10 +240,59 @@ def report_target_details(target_config: EthosUConfiguration) -> Report:
)
-def metrics_as_records(perf_metrics: list[PerformanceMetrics]) -> list[tuple]:
+def metrics_as_records(
+ perf_metrics: list[PerformanceMetrics],
+) -> tuple[list[tuple], list[tuple]]:
"""Convert perf metrics object into list of records."""
perf_metrics = [item.in_kilobytes() for item in perf_metrics]
+ def _layerwise_as_metrics(
+ perf_metrics: list[PerformanceMetrics],
+ ) -> list[tuple]:
+ metric_map = defaultdict(list) # type: dict[str, list]
+ format_types = {int: "12,d", str: "", float: "12.2f"}
+ rows = []
+ for perf_metric in perf_metrics:
+ if perf_metric.layerwise_perf_info:
+ for layerwise_metric in perf_metric.layerwise_perf_info.layerwise_info:
+ field_names = [
+ field.name
+ for field in fields(layerwise_metric)
+ if field.name != "name"
+ ]
+ duplicate_idx = 1
+ dict_key = getattr(layerwise_metric, "name")
+ while dict_key in metric_map:
+ dict_key = (
+ getattr(layerwise_metric, "name")
+ + " ("
+ + str(duplicate_idx)
+ + ")"
+ )
+ duplicate_idx += 1
+ for field_name in field_names:
+ metric_map[dict_key].append(
+ getattr(layerwise_metric, field_name)
+ )
+ rows = [
+ (
+ name,
+ *(
+ Cell(
+ value,
+ Format(
+ str_fmt=format_types[type(value)]
+ if type(value) in format_types
+ else ""
+ ),
+ )
+ for value in values
+ ),
+ )
+ for name, values in metric_map.items()
+ ]
+ return rows
+
def _cycles_as_records(perf_metrics: list[PerformanceMetrics]) -> list[tuple]:
metric_map = defaultdict(list)
for metrics in perf_metrics:
@@ -306,7 +358,7 @@ def metrics_as_records(perf_metrics: list[PerformanceMetrics]) -> list[tuple]:
_data_beats_as_records,
)
for metrics in metrics_func(perf_metrics)
- ]
+ ], _layerwise_as_metrics(perf_metrics)
def report_perf_metrics(
@@ -315,9 +367,9 @@ def report_perf_metrics(
"""Return comparison table for the performance metrics."""
if isinstance(perf_metrics, PerformanceMetrics):
perf_metrics = [perf_metrics]
+ rows, layerwise_rows = metrics_as_records(perf_metrics)
- rows = metrics_as_records(perf_metrics)
-
+ # Create a seperate table for layerwise data
if len(perf_metrics) == 2:
return Table(
columns=[
@@ -349,17 +401,42 @@ def report_perf_metrics(
alias="performance_metrics",
notes="IMPORTANT: The performance figures above refer to NPU only",
)
-
- return Table(
- columns=[
- Column("Metric", alias="metric", fmt=Format(wrap_width=30)),
- Column("Value", alias="value", fmt=Format(wrap_width=15)),
- Column("Unit", alias="unit", fmt=Format(wrap_width=15)),
- ],
- rows=rows,
- name="Performance metrics",
- alias="performance_metrics",
- notes="IMPORTANT: The performance figures above refer to NPU only",
+ if layerwise_rows == []:
+ return Table(
+ columns=[
+ Column("Metric", alias="metric", fmt=Format(wrap_width=30)),
+ Column("Value", alias="value", fmt=Format(wrap_width=15)),
+ Column("Unit", alias="unit", fmt=Format(wrap_width=15)),
+ ],
+ rows=rows,
+ name="Performance metrics",
+ alias="performance_metrics",
+ notes="IMPORTANT: The performance figures above refer to NPU only",
+ )
+ return CompoundReport(
+ [
+ Table(
+ columns=[
+ Column("Metric", alias="metric", fmt=Format(wrap_width=30)),
+ Column("Value", alias="value", fmt=Format(wrap_width=15)),
+ Column("Unit", alias="unit", fmt=Format(wrap_width=15)),
+ ],
+ rows=rows,
+ name="Performance metrics",
+ alias="performance_metrics",
+ notes="IMPORTANT: The performance figures above refer to NPU only",
+ ),
+ Table(
+ columns=[
+ Column(name, alias=alias, fmt=Format(wrap_width=30))
+ for alias, _, name in layer_metrics
+ ],
+ rows=layerwise_rows,
+ name="Layer-Wise Metrics",
+ alias="layerwise_metrics",
+ notes="",
+ ),
+ ]
)
generated by cgit v1.2.1 (git 2.23.0) at 2024-05-28 01:43:33 +0000