Add memory operation helper class for OpenCL buffer type

CLMemoryOpBufferHelper will be used by load/store  facility in the CLKernelwriter.

Partially Resolves: COMPMID-5791

Signed-off-by: Gunes Bayir <gunes.bayir@arm.com>
Change-Id: I3e49b1cdd90f0bf6a43dcd6d88863d4a4496c04b
Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/10119
Tested-by: Arm Jenkins <bsgcomp@arm.com>
Reviewed-by: Gian Marco Iodice <gianmarco.iodice@arm.com>
Benchmark: Arm Jenkins <bsgcomp@arm.com>

4 files changed, 452 insertions, 16 deletions

diff --git a/compute_kernel_writer/CMakeLists.txt b/compute_kernel_writer/CMakeLists.txt
index ec985ca427..a8b933acc2 100644
--- a/compute_kernel_writer/CMakeLists.txt
+++ b/compute_kernel_writer/CMakeLists.txt
@@ -139,6 +139,7 @@ if(CKW_ENABLE_OPENCL)
         src/cl/CLHelpers.cpp
         src/cl/CLTile.cpp
         src/cl/CLKernelWriter.cpp
+        src/cl/helpers/CLMemoryOpBufferHelper.cpp
     )
 endif()
 
diff --git a/compute_kernel_writer/src/cl/helpers/CLMemoryOpBufferHelper.cpp b/compute_kernel_writer/src/cl/helpers/CLMemoryOpBufferHelper.cpp
new file mode 100644
index 0000000000..e50418711e
--- /dev/null
+++ b/compute_kernel_writer/src/cl/helpers/CLMemoryOpBufferHelper.cpp
@@ -0,0 +1,331 @@
+/*
+ * Copyright (c) 2023 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "src/cl/helpers/CLMemoryOpBufferHelper.h"
+
+#include "ckw/Error.h"
+#include "ckw/TensorSampler.h"
+#include "ckw/types/MemoryOperation.h"
+#include "ckw/types/TensorStorageType.h"
+
+#include "src/ITensor.h"
+#include "src/Tensor3dMapper.h"
+#include "src/cl/CLHelpers.h"
+#include "src/cl/CLKernelWriter.h"
+#include "src/cl/CLTensorArgument.h"
+#include "src/cl/CLTile.h"
+
+namespace ckw
+{
+bool CLMemoryOpBufferHelper::validate(const CLKernelWriter *writer, const ITensor *tensor, const TensorSampler *sampler, const Tensor3dMapper *mapper, MemoryOperation op, const CLTile *dst)
+{
+    CKW_UNUSED(writer, tensor, mapper, op, dst);
+
+    if(sampler->storage() != TensorStorageType::BufferUint8Ptr)
+    {
+        return false;
+    }
+    return true;
+}
+
+/** Initialization and Finalizing Logic
+ *
+ *   The meanings of if/elses in different dimensions and how they're constructed:
+ *   - x: partial load/store
+ *   - y: no load/store operation
+ *   - z: no load/store operation
+ *   if(x)
+ *   {
+ *       if(z)
+ *       {
+ *           if(y)
+ *           {
+ *               // full load/store width
+ *           }
+ *           else
+ *           {
+ *               // no load/store
+ *           }
+ *       }
+ *       else
+ *       {
+ *           // no load/store
+ *       }
+ *   }
+ *   else
+ *   {
+ *       if(z)
+ *       {
+ *           if(y)
+ *           {
+ *               // partial load/store width
+ *           }
+ *           else
+ *           {
+ *               // no load/store
+ *           }
+ *       }
+ *       else
+ *       {
+ *           // no load/store
+ *       }
+ *   }
+ *
+ *  In general, initialize() writes if conditions, and finalize() writes else conditions.
+ *  The outermost block is x, then z and then y. This is why, if/else's covering for y are initialized
+ *  at each row write. In some addressing modes, such as None, no if/else conditions are written.
+ */
+void CLMemoryOpBufferHelper::initialize(const CLTile *dst, const CLTile *x, const CLTile *z, const CLTile *b)
+{
+    _dst           = dst;
+
+    CKW_ASSERT(validate(_writer, _tensor, _sampler, _mapper.get(), _op, _dst));
+
+    _ls_width_full = dst->info().width();
+    _coord_x      = x->scalar(0, 0).str;
+    _coord_z      = z->scalar(0, 0).str;
+    _coord_b      = b->scalar(0, 0).str;
+    _coord_orig_z = _coord_z;
+
+    out_of_bound_initialize_x(_coord_x);
+    out_of_bound_initialize_z(_coord_z);
+}
+
+void CLMemoryOpBufferHelper::write_row(int32_t row_id, const std::string &coord_y)
+{
+    // The only check required is on Y.
+    out_of_bound_initialize_y(coord_y);
+
+    const std::string dst     = _dst->vector(row_id).str;
+    const std::string address = to_buffer_address(_coord_x, coord_y, _coord_z, _coord_b);
+    const std::string ls_buf  = to_statement(_op, _ls_width_full, dst, address);
+
+    _writer->op_write_raw_code(ls_buf);
+    _writer->op_write_raw_code(";\n");
+
+    out_of_bound_finalize_y(dst);
+
+    // The left over load/store will be written in the finalize stage
+    if(_ls_width_part.size() != 0)
+    {
+        int32_t col_start = 0;
+        for(int32_t partial_width : _ls_width_part)
+        {
+            const std::string dst       = _dst->vector(row_id, col_start, partial_width).str;
+            const std::string coord_x   = _coord_x + " + " + std::to_string(col_start);
+            const std::string address   = to_buffer_address(coord_x, coord_y, _coord_z, _coord_b);
+            const std::string statement = to_statement(_op, partial_width, dst, address);
+            _leftovers_x.emplace_back(dst, coord_y, statement);
+
+            col_start += partial_width;
+        }
+    }
+}
+
+void CLMemoryOpBufferHelper::finalize()
+{
+    out_of_bound_finalize_z();
+    out_of_bound_finalize_x();
+}
+
+void CLMemoryOpBufferHelper::out_of_bound_initialize_x(const std::string &coord)
+{
+    if(_sampler->address_mode_x() == TensorSamplerAddressModeX::OverlappingMin)
+    {
+        TensorInfo tensor_info = _tensor->info();
+        TensorShape shape      = tensor_info.shape();
+
+        _ls_width_part = cl_decompose_vector_width(shape[0] % _ls_width_full);
+        if(_ls_width_part.size() != 0)
+        {
+            _writer->op_write_raw_code("if(" + coord + " > 0)\n{\n");
+        }
+    }
+}
+
+void CLMemoryOpBufferHelper::out_of_bound_finalize_x()
+{
+    if(_sampler->address_mode_x() == TensorSamplerAddressModeX::OverlappingMin)
+    {
+        if(_ls_width_part.size() != 0)
+        {
+            _writer->op_write_raw_code("}\nelse\n{\n");
+
+            out_of_bound_initialize_z(_coord_orig_z);
+            for(LeftoverDescriptor leftover_desc : _leftovers_x)
+            {
+                out_of_bound_initialize_y(leftover_desc.coord);
+                _writer->op_write_raw_code(leftover_desc.statement);
+                _writer->op_write_raw_code(";\n");
+                out_of_bound_finalize_y(leftover_desc.dst);
+            }
+            out_of_bound_finalize_z();
+            _writer->op_write_raw_code("}\n");
+        }
+    }
+}
+
+void CLMemoryOpBufferHelper::out_of_bound_initialize_y(const std::string &coord)
+{
+    std::string max = "";
+
+    const TensorSamplerAddressModeY address_mode_y = _sampler->address_mode_y();
+
+    switch(address_mode_y)
+    {
+        case TensorSamplerAddressModeY::ClampToBorderMaxOnly:
+            // Not to be moved outside the case because it marks the relevant tensor component as used even if we dont't use the variable
+            max = _mapper->dim_y().str;
+            _writer->op_write_raw_code("if(" + coord + " < " + max + ")\n{\n");
+            break;
+        case TensorSamplerAddressModeY::None:
+            break;
+        default:
+            CKW_THROW_MSG("Unsupported address mode for Y dimension");
+    }
+}
+
+void CLMemoryOpBufferHelper::out_of_bound_finalize_y(const std::string &dst)
+{
+    const TensorSamplerAddressModeY address_mode_y = _sampler->address_mode_y();
+
+    switch(address_mode_y)
+    {
+        case TensorSamplerAddressModeY::ClampToBorderMaxOnly:
+            _writer->op_write_raw_code("}\nelse\n{\n");
+            _writer->op_write_raw_code(dst);
+            _writer->op_write_raw_code(" = 0.0f;\n}\n");
+            break;
+        case TensorSamplerAddressModeY::None:
+            break;
+        default:
+            CKW_THROW_MSG("Unsupported address mode for Y dimension");
+    }
+}
+
+void CLMemoryOpBufferHelper::out_of_bound_initialize_z(const std::string &coord)
+{
+    CKW_UNUSED(coord);
+
+    const TensorSamplerAddressModeZ address_mode_z = _sampler->address_mode_z();
+    switch(address_mode_z)
+    {
+        case TensorSamplerAddressModeZ::None:
+            break;
+        default:
+            CKW_THROW_MSG("Unsupported address mode for Z dimension");
+    }
+}
+
+void CLMemoryOpBufferHelper::out_of_bound_finalize_z()
+{
+    const TensorSamplerAddressModeZ address_mode_z = _sampler->address_mode_z();
+
+    switch(address_mode_z)
+    {
+        case TensorSamplerAddressModeZ::None:
+            break;
+        default:
+            CKW_THROW_MSG("Unsupported address mode for Z dimension");
+    }
+}
+
+std::string CLMemoryOpBufferHelper::to_statement(MemoryOperation op, int32_t vector_width, const std::string &data,
+                             const std::string &address) const
+{
+    switch(op)
+    {
+        case MemoryOperation::Load:
+            if(vector_width != 1)
+            {
+                return data + " = vload" + std::to_string(vector_width) + "(0, " + address + ")";
+            }
+            else
+            {
+                return data + " = *(" + address + ")";
+            }
+            break;
+        case MemoryOperation::Store:
+            if(vector_width != 1)
+            {
+                return "vstore" + std::to_string(vector_width) + "(" + data + ", 0, " + address + ")";
+            }
+            else
+            {
+                return "*(" + address + ") = " + data;
+            }
+            break;
+        default:
+            CKW_THROW_MSG("Unsupported MemoryOperation");
+    }
+
+    return "";
+}
+
+std::string CLMemoryOpBufferHelper::to_buffer_address(const std::string &x, const std::string &y, const std::string &z,
+                                     const std::string &b) const
+{
+    TensorStorageType tensor_storage = _sampler->storage();
+    CKW_ASSERT(tensor_storage == TensorStorageType::BufferUint8Ptr);
+
+    const std::string ptr_buf      = _tensor->storage(tensor_storage).val;
+    const std::string dst_type     = cl_data_type_rounded_up_to_valid_vector_width(_dst->info().data_type(), 1);
+
+    std::string address;
+    address += "(__global ";
+    address += dst_type;
+    address += "*)(";
+    address += ptr_buf;
+    if(x != "0" && (_mapper->dim_x().str != "1"))
+    {
+        address += " + (";
+        address += x + ") * sizeof(" + dst_type + ")";
+    }
+    if(y != "0")
+    {
+        const std::string stride_y = _mapper->stride_y().str;
+        address += " + (";
+        address += y + ")";
+        address += " * ";
+        address += stride_y;
+    }
+    if(z != "0" && (_mapper->dim_z().str != "1"))
+    {
+        const std::string stride_z = _mapper->stride_z().str;
+        address += " + (";
+        address += z + ")";
+        address += " * ";
+        address += stride_z;
+    }
+    if(b != "0" && (_mapper->dim_batch().str != "1"))
+    {
+        const std::string stride_b = _mapper->stride_batch().str;
+        address += " + (";
+        address += b + ")";
+        address += " * ";
+        address += stride_b;
+    }
+    address += ")";
+    return address;
+}
+} // namespace ckw
diff --git a/compute_kernel_writer/src/cl/helpers/CLMemoryOpBufferHelper.h b/compute_kernel_writer/src/cl/helpers/CLMemoryOpBufferHelper.h
new file mode 100644
index 0000000000..9bcd571a81
--- /dev/null
+++ b/compute_kernel_writer/src/cl/helpers/CLMemoryOpBufferHelper.h
@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2023 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef CKW_SRC_CL_CLMEMORYOPBUFFERHELPER_H
+#define CKW_SRC_CL_CLMEMORYOPBUFFERHELPER_H
+
+#include "src/cl/helpers/ICLMemoryOpHelper.h"
+
+#include <string>
+#include <vector>
+#include <cstdint>
+
+namespace ckw
+{
+
+// Forward Declarations
+class CLKernelWriter;
+class CLTile;
+enum class MemoryOperation;
+
+/** Helper class to write memory operations (like load/store) in OpenCL
+ */
+class CLMemoryOpBufferHelper : public ICLMemoryOpHelper
+{
+public:
+    /** Constructor similar to @ref ICLMemoryOpHelper() */
+    CLMemoryOpBufferHelper(CLKernelWriter *writer, ITensor *tensor, TensorSampler *sampler, MemoryOperation op)
+        : ICLMemoryOpHelper(writer, tensor, sampler, op)
+    {
+    }
+
+    /** Copy constructor */
+    CLMemoryOpBufferHelper(const CLMemoryOpBufferHelper &) = default;
+
+    /** Assignment operator overload */
+    CLMemoryOpBufferHelper &operator=(const CLMemoryOpBufferHelper &) = default;
+
+    // Methods overridden
+    void initialize(const CLTile *dst, const CLTile *x, const CLTile *z, const CLTile *b) override;
+    void write_row(int32_t row_id, const std::string &coord_y) override;
+    void finalize() override;
+
+private:
+    struct LeftoverDescriptor
+    {
+        LeftoverDescriptor(const std::string &dst, const std::string &coord, const std::string &statement)
+        : dst(dst), coord(coord), statement(statement)
+        {
+        }
+
+        std::string dst{};        // Describes the destination tile or part of it
+        std::string coord{};      // Describes the coordinate to be used in boundary checks
+        std::string statement{};  // Describes the memory operation statement
+    };
+
+    std::vector<int32_t>            _ls_width_part{};
+    std::vector<LeftoverDescriptor> _leftovers_x{};
+    std::string                     _coord_orig_z{};
+
+    static bool validate(const CLKernelWriter *writer, const ITensor *tensor, const TensorSampler *sampler, const Tensor3dMapper *mapper, MemoryOperation op, const CLTile *dst);
+
+    void out_of_bound_initialize_x(const std::string &coord);
+    void out_of_bound_finalize_x();
+    void out_of_bound_initialize_y(const std::string &coord);
+    void out_of_bound_finalize_y(const std::string &dst);
+    void out_of_bound_initialize_z(const std::string &coord);
+    void out_of_bound_finalize_z();
+
+    std::string to_statement(MemoryOperation op, int32_t vector_width, const std::string &data, const std::string &address) const;
+    std::string to_buffer_address(const std::string &x, const std::string &y, const std::string &z, const std::string &b) const;
+};
+} // namespace ckw
+
+#endif /* CKW_SRC_CL_CLMEMORYOPBUFFERHELPER_H */
diff --git a/compute_kernel_writer/src/cl/helpers/ICLMemoryOpHelper.h b/compute_kernel_writer/src/cl/helpers/ICLMemoryOpHelper.h
index 008d147fa2..7f363431e8 100644
--- a/compute_kernel_writer/src/cl/helpers/ICLMemoryOpHelper.h
+++ b/compute_kernel_writer/src/cl/helpers/ICLMemoryOpHelper.h
@@ -25,7 +25,11 @@
 #ifndef CKW_SRC_CL_HELPERS_ICLMEMORYOPHELPER_H
 #define CKW_SRC_CL_HELPERS_ICLMEMORYOPHELPER_H
 
+#include "ckw/TensorSampler.h"
+#include "src/Tensor3dMapper.h"
+
 #include <cstdint>
+#include <memory>
 #include <string>
 
 namespace ckw
@@ -34,7 +38,8 @@ namespace ckw
 // Forward Declarations
 class CLTile;
 class CLKernelWriter;
-class Tensor3dMapper;
+class ITensor;
+class TensorSampler;
 enum class MemoryOperation;
 
 /** Base class OpenCL memory operation helper classes
@@ -45,13 +50,15 @@ class ICLMemoryOpHelper
 public:
     /** Constructor
      *
-     * @param[in] x      @ref ckw::CLKernelWriter object to write the code
-     * @param[in] mapper @ref ckw::Tensor3dMapper object that tells how to map the Nd tensor to 3d
-     * @param[in] op     The memory operation to be done (e.g. Load/Store)
+     * @param[in] writer  @ref ckw::CLKernelWriter object to write the code
+     * @param[in] tensor  @ref ckw::ITensor object to perform the memory operation on
+     * @param[in] sampler @ref ckw::TensorSampler object that tells how to sample a tensor
+     * @param[in] op      The memory operation to be done (e.g. Load/Store)
      */
-    ICLMemoryOpHelper(CLKernelWriter *x, Tensor3dMapper *mapper, MemoryOperation op)
-        : _writer(x), _mapper(mapper), _op(op)
+    ICLMemoryOpHelper(CLKernelWriter *writer, ITensor *tensor, TensorSampler *sampler, MemoryOperation op)
+        : _writer(writer), _tensor(tensor), _sampler(sampler), _op(op)
     {
+        _mapper = std::make_unique<Tensor3dMapper>(tensor, sampler->format());
     }
 
     /** Copy constructor */
@@ -72,7 +79,7 @@ public:
      * @param[in] z    tile object that describes the z-coordinate of the tensor involved
      * @param[in] b    tile object that describes the batch offset of the tensor involved
      */
-    virtual void initialize(CLTile *dst, CLTile *x, CLTile *z, CLTile *b) = 0;
+    virtual void initialize(const CLTile *dst, const CLTile *x, const CLTile *z, const CLTile *b) = 0;
 
     /** Method that writes the actual code to the writer that performs the mentioned memory
      *  operation on the tile initialized. It writes the code for a specific row given in the
@@ -81,7 +88,7 @@ public:
      * @param[in] row_id   row id
      * @param[in] coord_y  y-coordinate as string
      */
-    virtual void write(int32_t row_id, const std::string &coord_y) = 0;
+    virtual void write_row(int32_t row_id, const std::string &coord_y) = 0;
 
     /** Method that finalizes the code in the writer object. This part is usually for taking
      *  care of finalizing anything that's been initialized inside @ref IMemoryHelper::initialize()
@@ -91,14 +98,16 @@ public:
     virtual void finalize() = 0;
 
 protected:
-    CLKernelWriter  *_writer;
-    Tensor3dMapper  *_mapper;
-    MemoryOperation _op;
-    CLTile          *_dst{ nullptr };
-    int32_t         _ls_width_full{ 0 };
-    std::string     _coord_x{};
-    std::string     _coord_z{};
-    std::string     _coord_b{};
+    CLKernelWriter                  *_writer{ nullptr };
+    ITensor                         *_tensor{ nullptr };
+    TensorSampler                   *_sampler{ nullptr };
+    MemoryOperation                  _op;
+    std::unique_ptr<Tensor3dMapper>  _mapper{ nullptr };
+    const CLTile                    *_dst{ nullptr };
+    int32_t                          _ls_width_full{ 0 };
+    std::string                      _coord_x{};
+    std::string                      _coord_z{};
+    std::string                      _coord_b{};
 };
 } // namespace ckw