1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
|
/*
* Copyright (c) 2017-2020 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to
* deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#pragma once
#ifdef __aarch64__
#include <arm_neon.h>
#include "../asmlib.hpp"
#include "../utils.hpp"
template<>
template<typename T>
void TransformImpl<4, 16, false, 1, 1, false>::Transform(T *out, const T *in, int ldin, int y0, int ymax, int k0, int kmax) {
uint8_t *outptr = (uint8_t *)out;
const uint8_t *inptr = (uint8_t *)in;
uint8_t zerobuff[16] = { 0 };
for (int y=y0; y<ymax; y+=4) {
const uint8_t *inptr0 = inptr + static_cast<intptr_t>(y) * ldin + k0;
const uint8_t *inptr1 = inptr0 + ldin;
const uint8_t *inptr2 = inptr1 + ldin;
const uint8_t *inptr3 = inptr2 + ldin;
prefetch_2x(inptr0);
prefetch_2x(inptr1);
prefetch_2x(inptr2);
prefetch_2x(inptr3);
int x=(kmax-k0);
for (;x>15;x-=16) {
/* Cope with ragged cases by copying from a buffer of zeroes instead */
if ((y + 3) >= ymax) {
switch ((y + 3) - ymax) {
case 2:
inptr1 = zerobuff;
// fall through
case 1:
inptr2 = zerobuff;
// fall through
case 0:
inptr3 = zerobuff;
break;
default:
UNREACHABLE("Impossible.");
}
}
__asm __volatile (
"LDR q0, [%[inptr0]], #16\n"
ASM_PREFETCH("[%[inptr0], #176]")
"LDR q1, [%[inptr1]], #16\n"
ASM_PREFETCH("[%[inptr1], #176]")
"STP q0, q1, [%[outptr]], #32\n"
"LDR q0, [%[inptr2]], #16\n"
ASM_PREFETCH("[%[inptr2], #176]")
"LDR q1, [%[inptr3]], #16\n"
ASM_PREFETCH("[%[inptr3], #176]")
"STP q0, q1, [%[outptr]], #32\n"
: [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3),
[outptr] "+r" (outptr)
:
: "v0", "v1"
);
}
if (x>0) {
/* Need to duplicate this here, in case we didn't run the main loop. */
if ((y + 3) >= ymax) {
switch ((y + 3) - ymax) {
case 2:
inptr1 = zerobuff;
// fall through
case 1:
inptr2 = zerobuff;
// fall through
case 0:
inptr3 = zerobuff;
break;
default:
UNREACHABLE("Impossible.");
}
}
/* We have to write out 16 values, copy as many legal values as there are and pad with 0 */
auto f = [&outptr, x](const uint8_t *&p) {
for (int i=0; i<16; i++) {
if (i < x) {
*outptr++ = *p++;
} else {
*outptr++ = 0;
}
}
};
f(inptr0);
f(inptr1);
f(inptr2);
f(inptr3);
}
}
}
#endif // __aarch64__
|
