Skip to content

Commit 95cfdb9

Browse files
rascaniclaude
rascani
and
claude
committed
Cortex-M: Add MLPerf Tiny model tests
Add Cortex-M dialect and implementation tests for DeepAutoEncoder, DS-CNN, MobileNet V1 0.25, and ResNet-8 from the MLPerf Tiny benchmark suite. All 4D model inputs use explicit channels_last memory format as required by CMSIS-NN. Add atol parameter to CortexMTester.test_dialect() and test_implementation() for models that need relaxed tolerance. Fix McuTestCase.example_inputs type annotation to accept both tuples and callables, matching actual usage. Co-authored-by: Claude <noreply@anthropic.com>
1 parent 6fccd5a commit 95cfdb9

5 files changed

Lines changed: 220 additions & 4 deletions

File tree

‎backends/cortex_m/test/models/test_deep_autoencoder.py‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
import torch
8+
from executorch.backends.arm.test.common import parametrize
9+
from executorch.backends.cortex_m.test.tester import CortexMTester, McuTestCase
10+
from executorch.examples.models.mlperf_tiny.deep_autoencoder import DeepAutoEncoder
11+
12+
ops_before_transforms: dict[str, int] = {
13+
"executorch_exir_dialects_edge__ops_aten_linear_default": 10,
14+
"executorch_exir_dialects_edge__ops_aten_relu_default": 9,
15+
"executorch_exir_dialects_edge__ops_quantized_decomposed_dequantize_per_tensor_default": 31,
16+
"executorch_exir_dialects_edge__ops_quantized_decomposed_quantize_per_tensor_default": 11,
17+
}
18+
19+
ops_after_transforms: dict[str, int] = {
20+
"executorch_exir_dialects_edge__ops_cortex_m_dequantize_per_tensor_default": 1,
21+
"executorch_exir_dialects_edge__ops_cortex_m_quantize_per_tensor_default": 1,
22+
"executorch_exir_dialects_edge__ops_cortex_m_quantized_linear_default": 10,
23+
}
24+
25+
test_cases = {
26+
"deep_autoencoder": McuTestCase(
27+
model=DeepAutoEncoder().eval(),
28+
example_inputs=lambda: ((torch.rand(1, 640) * 2 - 1,)),
29+
),
30+
}
31+
32+
33+
@parametrize("test_case", test_cases)
34+
def test_dialect_deep_autoencoder(test_case):
35+
inputs = test_case.get_example_inputs()
36+
tester = CortexMTester(test_case.model, inputs)
37+
tester.test_dialect(ops_before_transforms, ops_after_transforms, qtol=1)
38+
39+
40+
@parametrize("test_case", test_cases)
41+
def test_implementation_deep_autoencoder(test_case):
42+
inputs = test_case.get_example_inputs()
43+
tester = CortexMTester(test_case.model, inputs)
44+
tester.test_implementation(qtol=1)

‎backends/cortex_m/test/models/test_ds_cnn.py‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
import torch
8+
from executorch.backends.arm.test.common import parametrize
9+
from executorch.backends.cortex_m.test.tester import CortexMTester, McuTestCase
10+
from executorch.examples.models.mlperf_tiny.ds_cnn import DSCNNKWS
11+
12+
ops_before_transforms: dict[str, int] = {
13+
"executorch_exir_dialects_edge__ops_aten_avg_pool2d_default": 1,
14+
"executorch_exir_dialects_edge__ops_aten_convolution_default": 9,
15+
"executorch_exir_dialects_edge__ops_aten_linear_default": 1,
16+
"executorch_exir_dialects_edge__ops_aten_relu_default": 9,
17+
"executorch_exir_dialects_edge__ops_aten_view_copy_default": 1,
18+
"executorch_exir_dialects_edge__ops_dim_order_ops__clone_dim_order_default": 2,
19+
"executorch_exir_dialects_edge__ops_quantized_decomposed_dequantize_per_channel_default": 18,
20+
"executorch_exir_dialects_edge__ops_quantized_decomposed_dequantize_per_tensor_default": 17,
21+
"executorch_exir_dialects_edge__ops_quantized_decomposed_quantize_per_tensor_default": 15,
22+
}
23+
24+
ops_after_transforms: dict[str, int] = {
25+
"executorch_exir_dialects_edge__ops_aten_view_copy_default": 1,
26+
"executorch_exir_dialects_edge__ops_cortex_m_dequantize_per_tensor_default": 1,
27+
"executorch_exir_dialects_edge__ops_cortex_m_pad_default": 1,
28+
"executorch_exir_dialects_edge__ops_cortex_m_quantize_per_tensor_default": 1,
29+
"executorch_exir_dialects_edge__ops_cortex_m_quantized_avg_pool2d_default": 1,
30+
"executorch_exir_dialects_edge__ops_cortex_m_quantized_conv2d_default": 4,
31+
"executorch_exir_dialects_edge__ops_cortex_m_quantized_depthwise_conv2d_default": 5,
32+
"executorch_exir_dialects_edge__ops_cortex_m_quantized_linear_default": 1,
33+
"executorch_exir_dialects_edge__ops_dim_order_ops__clone_dim_order_default": 2,
34+
}
35+
36+
test_cases = {
37+
"ds_cnn": McuTestCase(
38+
model=DSCNNKWS().eval(),
39+
example_inputs=lambda: (
40+
(torch.rand(1, 1, 49, 10) * 2 - 1).to(memory_format=torch.channels_last),
41+
),
42+
),
43+
}
44+
45+
46+
@parametrize("test_case", test_cases)
47+
def test_dialect_ds_cnn(test_case):
48+
inputs = test_case.get_example_inputs()
49+
tester = CortexMTester(test_case.model, inputs)
50+
tester.test_dialect(ops_before_transforms, ops_after_transforms, qtol=1)
51+
52+
53+
@parametrize("test_case", test_cases)
54+
def test_implementation_ds_cnn(test_case):
55+
inputs = test_case.get_example_inputs()
56+
tester = CortexMTester(test_case.model, inputs)
57+
tester.test_implementation(qtol=1)

‎backends/cortex_m/test/models/test_mobilenet_v1_025.py‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
import torch
8+
from executorch.backends.arm.test.common import parametrize
9+
from executorch.backends.cortex_m.test.tester import CortexMTester, McuTestCase
10+
from executorch.examples.models.mlperf_tiny.mobilenet_v1_025 import MobileNetV1025
11+
12+
ops_before_transforms: dict[str, int] = {
13+
"executorch_exir_dialects_edge__ops_aten_avg_pool2d_default": 1,
14+
"executorch_exir_dialects_edge__ops_aten_convolution_default": 27,
15+
"executorch_exir_dialects_edge__ops_aten_linear_default": 1,
16+
"executorch_exir_dialects_edge__ops_aten_relu_default": 27,
17+
"executorch_exir_dialects_edge__ops_aten_view_copy_default": 1,
18+
"executorch_exir_dialects_edge__ops_quantized_decomposed_dequantize_per_channel_default": 54,
19+
"executorch_exir_dialects_edge__ops_quantized_decomposed_dequantize_per_tensor_default": 33,
20+
"executorch_exir_dialects_edge__ops_quantized_decomposed_quantize_per_tensor_default": 31,
21+
}
22+
23+
ops_after_transforms: dict[str, int] = {
24+
"executorch_exir_dialects_edge__ops_aten_view_copy_default": 1,
25+
"executorch_exir_dialects_edge__ops_cortex_m_dequantize_per_tensor_default": 1,
26+
"executorch_exir_dialects_edge__ops_cortex_m_quantize_per_tensor_default": 1,
27+
"executorch_exir_dialects_edge__ops_cortex_m_quantized_avg_pool2d_default": 1,
28+
"executorch_exir_dialects_edge__ops_cortex_m_quantized_conv2d_default": 14,
29+
"executorch_exir_dialects_edge__ops_cortex_m_quantized_depthwise_conv2d_default": 13,
30+
"executorch_exir_dialects_edge__ops_cortex_m_quantized_linear_default": 1,
31+
}
32+
33+
test_cases = {
34+
"mobilenet_v1_025": McuTestCase(
35+
model=MobileNetV1025().eval(),
36+
example_inputs=lambda: (
37+
(torch.rand(1, 3, 96, 96) * 2 - 1).to(memory_format=torch.channels_last),
38+
),
39+
),
40+
}
41+
42+
43+
@parametrize("test_case", test_cases)
44+
def test_dialect_mobilenet_v1_025(test_case):
45+
inputs = test_case.get_example_inputs()
46+
tester = CortexMTester(test_case.model, inputs)
47+
tester.test_dialect(ops_before_transforms, ops_after_transforms, qtol=1)
48+
49+
50+
@parametrize("test_case", test_cases)
51+
def test_implementation_mobilenet_v1_025(test_case):
52+
inputs = test_case.get_example_inputs()
53+
tester = CortexMTester(test_case.model, inputs)
54+
tester.test_implementation(qtol=1)

‎backends/cortex_m/test/models/test_resnet8.py‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
import torch
8+
from executorch.backends.arm.test.common import parametrize
9+
from executorch.backends.cortex_m.test.tester import CortexMTester, McuTestCase
10+
from executorch.examples.models.mlperf_tiny.resnet8 import ResNet8
11+
12+
ops_before_transforms: dict[str, int] = {
13+
"executorch_exir_dialects_edge__ops_aten_add_Tensor": 3,
14+
"executorch_exir_dialects_edge__ops_aten_avg_pool2d_default": 1,
15+
"executorch_exir_dialects_edge__ops_aten_convolution_default": 9,
16+
"executorch_exir_dialects_edge__ops_aten_linear_default": 1,
17+
"executorch_exir_dialects_edge__ops_aten_relu_default": 7,
18+
"executorch_exir_dialects_edge__ops_aten_view_copy_default": 1,
19+
"executorch_exir_dialects_edge__ops_quantized_decomposed_dequantize_per_channel_default": 16,
20+
"executorch_exir_dialects_edge__ops_quantized_decomposed_dequantize_per_tensor_default": 21,
21+
"executorch_exir_dialects_edge__ops_quantized_decomposed_quantize_per_tensor_default": 16,
22+
}
23+
24+
ops_after_transforms: dict[str, int] = {
25+
"executorch_exir_dialects_edge__ops_aten_view_copy_default": 1,
26+
"executorch_exir_dialects_edge__ops_cortex_m_dequantize_per_tensor_default": 1,
27+
"executorch_exir_dialects_edge__ops_cortex_m_quantize_per_tensor_default": 1,
28+
"executorch_exir_dialects_edge__ops_cortex_m_quantized_add_default": 3,
29+
"executorch_exir_dialects_edge__ops_cortex_m_quantized_avg_pool2d_default": 1,
30+
"executorch_exir_dialects_edge__ops_cortex_m_quantized_conv2d_default": 9,
31+
"executorch_exir_dialects_edge__ops_cortex_m_quantized_linear_default": 1,
32+
}
33+
34+
test_cases = {
35+
"resnet8": McuTestCase(
36+
model=ResNet8().eval(),
37+
example_inputs=lambda: (
38+
(torch.rand(1, 3, 32, 32) * 2 - 1).to(memory_format=torch.channels_last),
39+
),
40+
),
41+
}
42+
43+
44+
@parametrize("test_case", test_cases)
45+
def test_dialect_resnet8(test_case):
46+
inputs = test_case.get_example_inputs()
47+
tester = CortexMTester(test_case.model, inputs)
48+
tester.test_dialect(ops_before_transforms, ops_after_transforms, qtol=1, atol=0.1)
49+
50+
51+
@parametrize("test_case", test_cases)
52+
def test_implementation_resnet8(test_case):
53+
inputs = test_case.get_example_inputs()
54+
tester = CortexMTester(test_case.model, inputs)
55+
tester.test_implementation(qtol=1)

‎backends/cortex_m/test/tester.py‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,9 @@
44
# LICENSE file in the root directory of this source tree.
55

66

7+
from collections.abc import Callable
78
from dataclasses import dataclass
8-
from typing import Any, Callable
9+
from typing import Any
910

1011
import torch
1112
from executorch.backends.arm.test.common import get_u55_compile_spec
@@ -84,6 +85,7 @@ def test_dialect(
8485
ops_before_transforms,
8586
ops_after_transforms,
8687
qtol=0,
88+
atol=1e-03,
8789
calibration_samples=None,
8890
):
8991
"""
@@ -102,9 +104,11 @@ def test_dialect(
102104
self.check_count(ops_before_transforms)
103105
self.run_passes()
104106
self.check_count(ops_after_transforms)
105-
self.run_method_and_compare_outputs(inputs=self.example_inputs, qtol=qtol)
107+
self.run_method_and_compare_outputs(
108+
inputs=self.example_inputs, qtol=qtol, atol=atol
109+
)
106110

107-
def test_implementation(self, qtol=0, calibration_samples=None):
111+
def test_implementation(self, qtol=0, atol=1e-03, calibration_samples=None):
108112
"""
109113
Test the optimized op implementation in simulation
110114
"""
@@ -122,7 +126,9 @@ def test_implementation(self, qtol=0, calibration_samples=None):
122126
self.run_passes()
123127
self.to_executorch()
124128
self.serialize()
125-
self.run_method_and_compare_outputs(inputs=self.example_inputs, qtol=qtol)
129+
self.run_method_and_compare_outputs(
130+
inputs=self.example_inputs, qtol=qtol, atol=atol
131+
)
126132

127133

128134
@dataclass

0 commit comments

Comments
 (0)