path: root/tests/test_target_ethos_u_reporters.py

# SPDX-FileCopyrightText: Copyright 2022-2024, Arm Limited and/or its affiliates.
# SPDX-License-Identifier: Apache-2.0
"""Tests for reports module."""
from __future__ import annotations

from typing import Any
from typing import cast

import pytest

from mlia.backend.vela.compat import NpuSupported
from mlia.backend.vela.compat import Operator
from mlia.backend.vela.performance import LayerPerfInfo
from mlia.backend.vela.performance import LayerwisePerfInfo
from mlia.core.reporting import CompoundReport
from mlia.core.reporting import Report
from mlia.core.reporting import Table
from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityInfo
from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityStatus
from mlia.target.ethos_u.config import EthosUConfiguration
from mlia.target.ethos_u.performance import MemorySizeType
from mlia.target.ethos_u.performance import MemoryUsage
from mlia.target.ethos_u.performance import PerformanceMetrics
from mlia.target.ethos_u.reporters import ethos_u_formatters
from mlia.target.ethos_u.reporters import report_operators
from mlia.target.ethos_u.reporters import report_perf_metrics
from mlia.target.ethos_u.reporters import report_target_details
from mlia.target.registry import profile
from mlia.utils.console import remove_ascii_codes


# pylint: disable=line-too-long
@pytest.mark.parametrize(
    "perf_metrics, expected_plain_text, expected_json_dict",
    [
        (
            [
                PerformanceMetrics(
                    target_config=EthosUConfiguration.load_profile("ethos-u55-256"),
                    npu_cycles=None,
                    memory_usage=MemoryUsage(
                        sram_memory_area_size=10,
                        dram_memory_area_size=0,
                        on_chip_flash_memory_area_size=0,
                        off_chip_flash_memory_area_size=20,
                        memory_size_type=MemorySizeType.KILOBYTES,
                    ),
                    layerwise_perf_info=LayerwisePerfInfo(
                        layerwise_info=[
                            LayerPerfInfo(
                                name="Test Layer",
                                tflite_operator="test_operator",
                                sram_usage=0,
                                op_cycles=0.0,
                                npu_cycles=0.0,
                                sram_access_cycles=0.0,
                                dram_access_cycles=0.0,
                                on_chip_flash_access_cycles=0.0,
                                off_chip_flash_access_cycles=0.0,
                                mac_count=0,
                                util_mac_percentage=0.0,
                            ),
                            LayerPerfInfo(
                                name="Test Layer 1",
                                tflite_operator="test_operator",
                                sram_usage=0,
                                op_cycles=0.0,
                                npu_cycles=0.0,
                                sram_access_cycles=0.0,
                                dram_access_cycles=0.0,
                                on_chip_flash_access_cycles=0.0,
                                off_chip_flash_access_cycles=0.0,
                                mac_count=0,
                                util_mac_percentage=0.0,
                            ),
                        ]
                    ),
                )
            ],
            """
Performance metrics:
┌─────────────────────┬──────────────┬──────┐
│ Metric              │ Value        │ Unit │
╞═════════════════════╪══════════════╪══════╡
│ SRAM used           │        10.00 │ KiB  │
├─────────────────────┼──────────────┼──────┤
│ Off-chip flash used │        20.00 │ KiB  │
└─────────────────────┴──────────────┴──────┘
IMPORTANT: The performance figures above refer to NPU only
Layer-Wise Metrics:
┌──────────────┬─────────────────┬──────────────┬──────────────┬──────────────┬──────────────┬──────────────┬──────────────┬──────────────┬──────────────┬──────────────┐
│ Layer Name   │ TFLite Operator │ SRAM Usage   │ OP Cycles    │ NPU Cycles   │ SRAM AC      │ DRAM AC      │ OnFlash AC   │ OffFlash AC  │ MAC Count    │ MAC Util (%) │
╞══════════════╪═════════════════╪══════════════╪══════════════╪══════════════╪══════════════╪══════════════╪══════════════╪══════════════╪══════════════╪══════════════╡
│ Test Layer   │ test_operator   │            0 │         0.00 │         0.00 │         0.00 │         0.00 │         0.00 │         0.00 │            0 │         0.00 │
├──────────────┼─────────────────┼──────────────┼──────────────┼──────────────┼──────────────┼──────────────┼──────────────┼──────────────┼──────────────┼──────────────┤
│ Test Layer 1 │ test_operator   │            0 │         0.00 │         0.00 │         0.00 │         0.00 │         0.00 │         0.00 │            0 │         0.00 │
└──────────────┴─────────────────┴──────────────┴──────────────┴──────────────┴──────────────┴──────────────┴──────────────┴──────────────┴──────────────┴──────────────┘
""".strip(),
            {
                "performance_metrics": [
                    {"metric": "SRAM used", "value": 10, "unit": "KiB"},
                    {"metric": "Off-chip flash used", "value": 20, "unit": "KiB"},
                ],
                "layerwise_metrics": [
                    {
                        "name": "Test Layer",
                        "tflite_operator": "test_operator",
                        "sram_usage": 0,
                        "op_cycles": 0.0,
                        "npu_cycles": 0.0,
                        "sram_access_cycles": 0.0,
                        "dram_access_cycles": 0.0,
                        "on_chip_flash_access_cycles": 0.0,
                        "off_chip_flash_access_cycles": 0.0,
                        "mac_count": 0,
                        "util_mac_percentage": 0.0,
                    },
                    {
                        "name": "Test Layer 1",
                        "tflite_operator": "test_operator",
                        "sram_usage": 0,
                        "op_cycles": 0.0,
                        "npu_cycles": 0.0,
                        "sram_access_cycles": 0.0,
                        "dram_access_cycles": 0.0,
                        "on_chip_flash_access_cycles": 0.0,
                        "off_chip_flash_access_cycles": 0.0,
                        "mac_count": 0,
                        "util_mac_percentage": 0.0,
                    },
                ],
            },
        ),
        (
            [
                PerformanceMetrics(
                    target_config=EthosUConfiguration.load_profile("ethos-u55-256"),
                    npu_cycles=None,
                    memory_usage=MemoryUsage(
                        sram_memory_area_size=10,
                        dram_memory_area_size=0,
                        on_chip_flash_memory_area_size=0,
                        off_chip_flash_memory_area_size=20,
                        memory_size_type=MemorySizeType.KILOBYTES,
                    ),
                    layerwise_perf_info=LayerwisePerfInfo(
                        layerwise_info=[
                            LayerPerfInfo(
                                name="Test Layer",
                                tflite_operator="test_operator",
                                sram_usage=0,
                                op_cycles=0.0,
                                npu_cycles=0.0,
                                sram_access_cycles=0.0,
                                dram_access_cycles=0.0,
                                on_chip_flash_access_cycles=0.0,
                                off_chip_flash_access_cycles=0.0,
                                mac_count=0,
                                util_mac_percentage=0.0,
                            ),
                            LayerPerfInfo(
                                name="Test Layer",
                                tflite_operator="test_operator",
                                sram_usage=0,
                                op_cycles=0.0,
                                npu_cycles=0.0,
                                sram_access_cycles=0.0,
                                dram_access_cycles=0.0,
                                on_chip_flash_access_cycles=0.0,
                                off_chip_flash_access_cycles=0.0,
                                mac_count=0,
                                util_mac_percentage=0.0,
                            ),
                        ]
                    ),
                )
            ],
            """
Performance metrics:
┌─────────────────────┬──────────────┬──────┐
│ Metric              │ Value        │ Unit │
╞═════════════════════╪══════════════╪══════╡
│ SRAM used           │        10.00 │ KiB  │
├─────────────────────┼──────────────┼──────┤
│ Off-chip flash used │        20.00 │ KiB  │
└─────────────────────┴──────────────┴──────┘
IMPORTANT: The performance figures above refer to NPU only
Layer-Wise Metrics:
┌────────────────┬─────────────────┬──────────────┬──────────────┬──────────────┬──────────────┬──────────────┬──────────────┬──────────────┬──────────────┬──────────────┐
│ Layer Name     │ TFLite Operator │ SRAM Usage   │ OP Cycles    │ NPU Cycles   │ SRAM AC      │ DRAM AC      │ OnFlash AC   │ OffFlash AC  │ MAC Count    │ MAC Util (%) │
╞════════════════╪═════════════════╪══════════════╪══════════════╪══════════════╪══════════════╪══════════════╪══════════════╪══════════════╪══════════════╪══════════════╡
│ Test Layer     │ test_operator   │            0 │         0.00 │         0.00 │         0.00 │         0.00 │         0.00 │         0.00 │            0 │         0.00 │
├────────────────┼─────────────────┼──────────────┼──────────────┼──────────────┼──────────────┼──────────────┼──────────────┼──────────────┼──────────────┼──────────────┤
│ Test Layer (1) │ test_operator   │            0 │         0.00 │         0.00 │         0.00 │         0.00 │         0.00 │         0.00 │            0 │         0.00 │
└────────────────┴─────────────────┴──────────────┴──────────────┴──────────────┴──────────────┴──────────────┴──────────────┴──────────────┴──────────────┴──────────────┘
""".strip(),
            {
                "performance_metrics": [
                    {"metric": "SRAM used", "value": 10, "unit": "KiB"},
                    {"metric": "Off-chip flash used", "value": 20, "unit": "KiB"},
                ],
                "layerwise_metrics": [
                    {
                        "name": "Test Layer",
                        "tflite_operator": "test_operator",
                        "sram_usage": 0,
                        "op_cycles": 0.0,
                        "npu_cycles": 0.0,
                        "sram_access_cycles": 0.0,
                        "dram_access_cycles": 0.0,
                        "on_chip_flash_access_cycles": 0.0,
                        "off_chip_flash_access_cycles": 0.0,
                        "mac_count": 0,
                        "util_mac_percentage": 0.0,
                    },
                    {
                        "name": "Test Layer (1)",
                        "tflite_operator": "test_operator",
                        "sram_usage": 0,
                        "op_cycles": 0.0,
                        "npu_cycles": 0.0,
                        "sram_access_cycles": 0.0,
                        "dram_access_cycles": 0.0,
                        "on_chip_flash_access_cycles": 0.0,
                        "off_chip_flash_access_cycles": 0.0,
                        "mac_count": 0,
                        "util_mac_percentage": 0.0,
                    },
                ],
            },
        ),
    ],
)
# pylint: enable=line-too-long
def test_report_perf_metrics(
    perf_metrics: PerformanceMetrics,
    expected_plain_text: str,
    expected_json_dict: dict,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Test report_perf_metrics formatter."""
    monkeypatch.setenv("COLUMNS", "5000")
    report = report_perf_metrics(perf_metrics)
    assert isinstance(report, CompoundReport)
    plain_text = remove_ascii_codes(report.to_plain_text())
    assert plain_text == expected_plain_text
    json_dict = report.to_json()
    assert json_dict == expected_json_dict


@pytest.mark.parametrize(
    "ops, expected_plain_text, expected_json_dict",
    [
        (
            [
                Operator(
                    "npu_supported",
                    "test_type",
                    NpuSupported(True, []),
                ),
                Operator(
                    "cpu_only",
                    "test_type",
                    NpuSupported(
                        False,
                        [
                            (
                                "CPU only operator",
                                "",
                            ),
                        ],
                    ),
                ),
                Operator(
                    "npu_unsupported",
                    "test_type",
                    NpuSupported(
                        False,
                        [
                            (
                                "Not supported operator",
                                "Reason why operator is not supported",
                            )
                        ],
                    ),
                ),
            ],
            """
Operators:
┌───┬─────────────────┬───────────────┬───────────┬───────────────────────────────┐
│ # │ Operator name   │ Operator type │ Placement │ Notes                         │
╞═══╪═════════════════╪═══════════════╪═══════════╪═══════════════════════════════╡
│ 1 │ npu_supported   │ test_type     │ NPU       │                               │
├───┼─────────────────┼───────────────┼───────────┼───────────────────────────────┤
│ 2 │ cpu_only        │ test_type     │ CPU       │ * CPU only operator           │
├───┼─────────────────┼───────────────┼───────────┼───────────────────────────────┤
│ 3 │ npu_unsupported │ test_type     │ CPU       │ * Not supported operator      │
│   │                 │               │           │ * Reason why operator is not  │
│   │                 │               │           │ supported                     │
└───┴─────────────────┴───────────────┴───────────┴───────────────────────────────┘
""".strip(),
            {
                "operators": [
                    {
                        "operator_name": "npu_supported",
                        "operator_type": "test_type",
                        "placement": "NPU",
                        "notes": [],
                    },
                    {
                        "operator_name": "cpu_only",
                        "operator_type": "test_type",
                        "placement": "CPU",
                        "notes": [{"note": "CPU only operator"}],
                    },
                    {
                        "operator_name": "npu_unsupported",
                        "operator_type": "test_type",
                        "placement": "CPU",
                        "notes": [
                            {"note": "Not supported operator"},
                            {"note": "Reason why operator is not supported"},
                        ],
                    },
                ]
            },
        ),
    ],
)
def test_report_operators(
    ops: list[Operator],
    expected_plain_text: str,
    expected_json_dict: dict,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Test report_operatos formatter."""
    # make terminal wide enough to print whole table
    monkeypatch.setenv("COLUMNS", "100")

    report = report_operators(ops)
    assert isinstance(report, Table)

    plain_text = remove_ascii_codes(report.to_plain_text())
    assert plain_text == expected_plain_text

    json_dict = report.to_json()
    assert json_dict == expected_json_dict


@pytest.mark.parametrize(
    "target, expected_plain_text, expected_json_dict",
    [
        [
            cast(EthosUConfiguration, profile("ethos-u55-256")),
            """Target information:
  Target                                                     ethos-u55
  MAC                                                              256

  Memory mode                                              Shared_Sram
    Const mem area                                                Axi1
    Arena mem area                                                Axi0
    Cache mem area                                                Axi0

  System config                            Ethos_U55_High_End_Embedded
    Accelerator clock                                   500,000,000 Hz
    AXI0 port                                                     Sram
    AXI1 port                                             OffChipFlash

    Memory area settings:
      Sram:
        Clock scales                                               1.0
        Burst length                                          32 bytes
        Read latency                                         32 cycles
        Write latency                                        32 cycles

      OffChipFlash:
        Clock scales                                             0.125
        Burst length                                         128 bytes
        Read latency                                         64 cycles
        Write latency                                        64 cycles""",
            {
                "target": {
                    "target": "ethos-u55",
                    "mac": 256,
                    "memory_mode": {
                        "const_mem_area": "Axi1",
                        "arena_mem_area": "Axi0",
                        "cache_mem_area": "Axi0",
                    },
                    "system_config": {
                        "accelerator_clock": {"value": 500000000.0, "unit": "Hz"},
                        "axi0_port": "Sram",
                        "axi1_port": "OffChipFlash",
                        "memory_area": {
                            "Sram": {
                                "clock_scales": 1.0,
                                "burst_length": {"value": 32, "unit": "bytes"},
                                "read_latency": {"value": 32, "unit": "cycles"},
                                "write_latency": {"value": 32, "unit": "cycles"},
                            },
                            "OffChipFlash": {
                                "clock_scales": 0.125,
                                "burst_length": {"value": 128, "unit": "bytes"},
                                "read_latency": {"value": 64, "unit": "cycles"},
                                "write_latency": {"value": 64, "unit": "cycles"},
                            },
                        },
                    },
                }
            },
        ],
    ],
)
def test_report_target_details(
    target: EthosUConfiguration,
    expected_plain_text: str,
    expected_json_dict: dict,
) -> None:
    """Test report_operatos formatter."""
    report = report_target_details(target)
    assert isinstance(report, Report)

    plain_text = report.to_plain_text()
    assert plain_text == expected_plain_text

    json_dict = report.to_json()
    assert json_dict == expected_json_dict


@pytest.mark.parametrize(
    "data",
    (TFLiteCompatibilityInfo(status=TFLiteCompatibilityStatus.COMPATIBLE),),
)
def test_ethos_u_formatters(data: Any) -> None:
    """Test function ethos_u_formatters() with valid input."""
    formatter = ethos_u_formatters(data)
    report = formatter(data)
    assert isinstance(report, Report)


def test_ethos_u_formatters_invalid_data() -> None:
    """Test function ethos_u_formatters() with invalid input."""
    with pytest.raises(
        Exception,
        match=r"^Unable to find appropriate formatter for .*",
    ):
        ethos_u_formatters(200)