// Copyright (c) 2020-2022, ARM Limited. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "data_layout.h" #include "quant_util.h" using namespace TosaReference; using namespace Eigen; using namespace tosa; template OpConcat::OpConcat(SubgraphTraverser* sgt_, TosaAttributeBase* attribute_, uint64_t id_) : GraphNode(sgt_, Op_CONCAT, id_) { setRequiredOperands(-1, 1); setRequiredRank(1, 6); INIT_ATTRIBUTE(Axis); } template OpConcat::~OpConcat() { if (attribute) delete attribute; } template int OpConcat::checkTensorAttributes() { if (validateRequiredOperands()) return 1; if (inputs.empty()) { printNodeValidationError("Concat operator must have at least one input tensor"); return 1; } int32_t num_inputs = inputs.size(); // output and input must be the same types and rank for (int32_t i = 0; i < num_inputs; i++) { if (inputs[i]->matchRankType(*outputs[0])) { printNodeValidationError("OpConcat: input ranks and types must match"); return 1; } ins.push_back(dynamic_cast*>(inputs[i])); } if (attribute->axis() < 0 || (size_t)attribute->axis() >= Rank) { printNodeValidationError("OpConcat: axis is beyond output tensor rank"); return 1; } int32_t output_dim_on_axis = 0; for (int32_t j = 0; j < num_inputs; j++) { for (int32_t i = 0; i < Rank; i++) { int32_t input_dim = inputs[j]->getShape()[i]; if (i == attribute->axis()) { output_dim_on_axis += input_dim; } else if (input_dim != outputs[0]->getShape()[i]) { printNodeValidationError("OpConcat: input dimension not matching output dimension"); return 1; } } } ERROR_IF(output_dim_on_axis != outputs[0]->getShape()[attribute->axis()], "OpConcat: sum of input dimension on axis not equal to output dimension on axis"); out = dynamic_cast*>(outputs[0]); return 0; } template int OpConcat::eval() { int32_t reversed_axis = Rank - 1 - attribute->axis(); for (int32_t d = 0; d < Rank; d++) { reverser[d] = Rank - 1 - d; } TIn result = ins[0]->getTensor().shuffle(reverser); for (size_t i = 1; i < ins.size(); i++) { TIn in_reversed = ins[i]->getTensor().shuffle(reverser); TIn temp = result.concatenate(in_reversed, reversed_axis); result = temp; } out->getTensor() = result.shuffle(reverser); return GraphNode::eval(); } template OpPad::OpPad(SubgraphTraverser* sgt_, TosaAttributeBase* attribute_, uint64_t id_) : GraphNode(sgt_, Op_PAD, id_) { setRequiredOperands(1, 1); setRequiredRank(0, 6); INIT_ATTRIBUTE(Pad); } template OpPad::~OpPad() { } template int OpPad::checkTensorAttributes() { if (validateRequiredOperands()) return 1; if (validateRequiredRank(inputs[0]) || validateRequiredRank(outputs[0])) { return 1; } // output and input must be the same types if (inputs[0]->matchRankType(*outputs[0])) { printNodeValidationError("Failure to match input and output type and rank"); return 1; } in = dynamic_cast*>(inputs[0]); out = dynamic_cast*>(outputs[0]); ASSERT_MEM(in && out); // padding in spec is 2D array in shape of [Rank, 2] // Reference model implement this as 1D array of [Rank * 2], with ordering: // [Rank0_front, Rank0_back, Rank1_front, Rank1_back, ..., Rank(N-1)_front, Rank(N-1)_back] ERROR_IF(attribute->padding().size() != (Rank * 2), "OpPad: padding length needs to be (rank(input1) * 2)"); for (int i = 0; i < Rank; i++) { int32_t pad_front = attribute->padding()[2 * i]; int32_t pad_back = attribute->padding()[2 * i + 1]; ERROR_IF((pad_front < 0) || (pad_back < 0), "OpPad: padding can't be smaller than 0"); ERROR_IF(out->getShape()[i] != pad_front + in->getShape()[i] + pad_back, "OpPad: output shape not equal to input plus padding"); paddings_array[i] = std::make_pair(pad_front, pad_back); } return 0; } template int OpPad::eval() { InEigenType pad_value = 0; switch (Dtype) { case DType_BOOL: case DType_INT8: case DType_INT16: case DType_INT32: pad_value = (InEigenType)attribute->pad_const_int(); break; case DType_FP16: case DType_FLOAT: pad_value = (InEigenType)attribute->pad_const_fp(); break; default: printNodeValidationError("Unsupported data type"); break; } this->out->getTensor() = this->in->getTensor().pad(this->paddings_array, pad_value); return GraphNode::eval(); } template OpReshape::OpReshape(SubgraphTraverser* sgt_, TosaAttributeBase* attribute_, uint64_t id_) : GraphNode(sgt_, Op_RESHAPE, id_) { setRequiredOperands(1, 1); setRequiredRank(0, 6); INIT_ATTRIBUTE(Reshape); } template OpReshape::~OpReshape() { if (attribute) delete attribute; } template int OpReshape::checkTensorAttributes() { if (validateRequiredOperands()) return 1; if (validateRequiredRank(inputs[0]) || validateRequiredRank(outputs[0])) { return 1; } // output and input must be the same types if (inputs[0]->matchType(*outputs[0])) { printNodeValidationError("OpReshape: Input and output types must match"); return 1; } ERROR_IF(inputs[0]->getElementCount() != outputs[0]->getElementCount(), "Input tensor size does not match output tensor size"); for (uint32_t d = 0; d < OutRank; d++) { ERROR_IF(attribute->new_shape()[d] != outputs[0]->getShape()[d], "OpReshape: new_shape doesn't match output shape"); } in = dynamic_cast*>(inputs[0]); out = dynamic_cast*>(outputs[0]); return 0; } template int OpReshape::eval() { for (int32_t d = 0; d < OutRank; d++) { array_shape[d] = attribute->new_shape()[OutRank - 1 - d]; out_reverser[d] = OutRank - 1 - d; } for (int32_t d = 0; d < InRank; d++) { in_reverser[d] = InRank - 1 - d; } // Eigen Tensor is col-major, and we're referencing row-major result // need to reverse it to row-major before reshape, and perform another reverse afterward // input tensor rank 0 can't do .shuffle(), need to be handled otherwise TIn in_reversed; if (InRank > 1) { in_reversed = in->getTensor().shuffle(in_reverser); } else { in_reversed = in->getTensor(); } TOut in_reshaped = in_reversed.reshape(array_shape); // output tensor can be rank 0, .reshape() and .shuffle() don't work, need to be handled otherwise if (OutRank > 1) { out->getTensor() = in_reshaped.shuffle(out_reverser); } else { out->getTensor() = in_reshaped; } return GraphNode::eval(); } template OpReverse::OpReverse(SubgraphTraverser* sgt_, TosaAttributeBase* attribute_, uint64_t id_) : GraphNode(sgt_, Op_REVERSE, id_) { setRequiredOperands(1, 1); setRequiredRank(1, 6); INIT_ATTRIBUTE(Axis); } template OpReverse::~OpReverse() { if (attribute) delete attribute; } template int OpReverse::checkTensorAttributes() { if (validateRequiredOperands()) return 1; if (validateRequiredRank(inputs[0]) || validateRequiredRank(outputs[0])) { return 1; } // output and input must be the same types if (inputs[0]->matchRankTypeShape(*outputs[0])) { printNodeValidationError("Failure to match input and output rank/type/shape"); return 1; } in = dynamic_cast*>(inputs[0]); out = dynamic_cast*>(outputs[0]); ASSERT_MEM(in && out); if (attribute->axis() < 0 || attribute->axis() >= inputs[0]->getRank()) { printNodeValidationError("Reverse axis must between [0, input_rank - 1]"); return 1; } // transform list of axis into true or false list // e.g. rank=4, axis=[1,2], reverse array would be [false, true, true, false] for (int i = 0; i < Rank; i++) { reverse_array[i] = false; } reverse_array[attribute->axis()] = true; return 0; } template int OpReverse::eval() { out->getTensor() = in->getTensor().reverse(reverse_array); return GraphNode::eval(); } template OpSlice::OpSlice(SubgraphTraverser* sgt_, TosaAttributeBase* attribute_, uint64_t id_) : GraphNode(sgt_, Op_SLICE, id_) { setRequiredOperands(1, 1); setRequiredRank(1, 4); INIT_ATTRIBUTE(Slice); } template OpSlice::~OpSlice() { if (attribute) delete attribute; } template int OpSlice::checkTensorAttributes() { if (validateRequiredOperands()) return 1; if (validateRequiredRank(inputs[0]) || validateRequiredRank(outputs[0])) { return 1; } // output and input must be the same types if (inputs[0]->matchType(*outputs[0])) { printNodeValidationError("Failure to match input and output type"); return 1; } in = dynamic_cast*>(inputs[0]); out = dynamic_cast*>(outputs[0]); ERROR_IF((int32_t)attribute->start().size() != in->getRank(), "OpSlice: begin array length needs to be rank(input)"); ERROR_IF((int32_t)attribute->size().size() != in->getRank(), "OpSlice: size array length needs to be rank(input)"); for (int32_t i = 0; i < in->getRank(); i++) { int32_t b = attribute->start()[i]; int32_t s = attribute->size()[i]; ERROR_IF(b < 0 || b >= in->getShape()[i], "OpSlice: start out of boundary"); ERROR_IF((b + s) < 0 || (b + s) > in->getShape()[i], "OpSlice: (start+size) out of boundary"); ERROR_IF(s <= 0, "OpSlice: output must be positive"); ERROR_IF(s != out->getShape()[i], "OpSlice: size doesn't match output tensor dimension"); begin_array[i] = b; size_array[i] = s; } return 0; } template int OpSlice::eval() { out->getTensor() = in->getTensor().slice(begin_array, size_array); return GraphNode::eval(); } template OpTileBase::OpTileBase(SubgraphTraverser* sgt_, TosaAttributeBase* attribute_, uint64_t id_) : GraphNode(sgt_, Op_TILE, id_) { setRequiredOperands(1, 1); setRequiredRank(0, 6); INIT_ATTRIBUTE(Tile); } template OpTileBase::~OpTileBase() { if (attribute) delete attribute; } template int OpTileBase::checkTensorAttributes() { if (validateRequiredOperands()) return 1; if (validateRequiredRank(inputs[0]) || validateRequiredRank(outputs[0])) { return 1; } // output and input must be the same ranks and types if (inputs[0]->matchRankType(*outputs[0])) { printNodeValidationError("Failure to match input and output rank or type"); return 1; } in = dynamic_cast*>(inputs[0]); out = dynamic_cast*>(outputs[0]); if (attribute->multiples().size() != Rank) { printNodeValidationError("1D list 'multiples' must have size equal to input rank"); return 1; } for (int32_t d = 0; d < Rank; d++) { ERROR_IF(in->getShape()[d] * attribute->multiples()[d] != out->getShape()[d], "Output shape not equal to input * multiples;") } return 0; } template int OpTile::eval() { // primary template shouldn't be called FATAL_ERROR("OpTile rank=%i, dtype=%s: not implemented yet", Rank, EnumNamesDType()[Dtype]); } template int OpTile<1, Dtype>::eval() { for (int32_t od0 = 0; od0 < this->out->getShape()[0]; od0++) { int32_t id0 = od0 % this->in->getShape()[0]; this->out->getTensor()(od0) = this->in->getTensor()(id0); } return GraphNode::eval(); } template int OpTile<2, Dtype>::eval() { for (int32_t od0 = 0; od0 < this->out->getShape()[0]; od0++) { int32_t id0 = od0 % this->in->getShape()[0]; for (int32_t od1 = 0; od1 < this->out->getShape()[1]; od1++) { int32_t id1 = od1 % this->in->getShape()[1]; this->out->getTensor()(od0, od1) = this->in->getTensor()(id0, id1); } } return GraphNode::eval(); } template int OpTile<3, Dtype>::eval() { for (int32_t od0 = 0; od0 < this->out->getShape()[0]; od0++) { int32_t id0 = od0 % this->in->getShape()[0]; for (int32_t od1 = 0; od1 < this->out->getShape()[1]; od1++) { int32_t id1 = od1 % this->in->getShape()[1]; for (int32_t od2 = 0; od2 < this->out->getShape()[2]; od2++) { int32_t id2 = od2 % this->in->getShape()[2]; this->out->getTensor()(od0, od1, od2) = this->in->getTensor()(id0, id1, id2); } } } return GraphNode::eval(); } template int OpTile<4, Dtype>::eval() { for (int32_t od0 = 0; od0 < this->out->getShape()[0]; od0++) { int32_t id0 = od0 % this->in->getShape()[0]; for (int32_t od1 = 0; od1 < this->out->getShape()[1]; od1++) { int32_t id1 = od1 % this->in->getShape()[1]; for (int32_t od2 = 0; od2 < this->out->getShape()[2]; od2++) { int32_t id2 = od2 % this->in->getShape()[2]; for (int32_t od3 = 0; od3 < this->out->getShape()[3]; od3++) { int32_t id3 = od3 % this->in->getShape()[3]; this->out->getTensor()(od0, od1, od2, od3) = this->in->getTensor()(id0, id1, id2, id3); } } } } return GraphNode::eval(); } template OpTranspose::OpTranspose(SubgraphTraverser* sgt_, TosaAttributeBase* attribute_, uint64_t id_) : GraphNode(sgt_, Op_TRANSPOSE, id_) { setRequiredOperands(1, 1); setRequiredRank(0, 6); INIT_ATTRIBUTE(Transpose); } template OpTranspose::~OpTranspose() {} template int OpTranspose::checkTensorAttributes() { if (validateRequiredOperands()) return 1; if (validateRequiredRank(inputs[0]) || validateRequiredRank(outputs[0])) { return 1; } // output and input must be the same types if (inputs[0]->matchRankType(*outputs[0])) { printNodeValidationError("Failure to match input and output rank and type"); return 1; } if (inputs[0]->getElementCount() != outputs[0]->getElementCount()) { printNodeValidationError("Failure to match input and output total element count"); return 1; } in = dynamic_cast*>(inputs[0]); out = dynamic_cast*>(outputs[0]); ASSERT_MEM(in && out); ERROR_IF(attribute->perms().size() != Rank, "OpTranspose: perms array size needs to match rank(input)"); std::array index_used; index_used.fill(false); for (int32_t d = 0; d < Rank; d++) { int32_t index = attribute->perms()[d]; ERROR_IF(index < 0 or index >= Rank, "OpTranspose: index out of boundary"); ERROR_IF(index_used[index], "OpTranspose: index duplicated in perm attribute"); index_used[index] = true; ERROR_IF(in->getShape()[index] != out->getShape()[d], "OpTranspose: input output shape mismatch"); perm_array[d] = index; } return 0; } template int OpTranspose::eval() { out->getTensor() = in->getTensor().shuffle(perm_array); return GraphNode::eval(); } // template explicit instantiation DEF_INSTANTIATE_RANK1_6_ONE_RANK_ONE_TYPE(OpConcat, FP16) DEF_INSTANTIATE_RANK1_6_ONE_RANK_ONE_TYPE(OpConcat, FLOAT) DEF_INSTANTIATE_RANK1_6_ONE_RANK_ONE_TYPE(OpConcat, INT8) DEF_INSTANTIATE_RANK1_6_ONE_RANK_ONE_TYPE(OpConcat, INT16) DEF_INSTANTIATE_RANK1_6_ONE_RANK_ONE_TYPE(OpConcat, INT32) DEF_INSTANTIATE_RANK1_6_ONE_RANK_ONE_TYPE(OpConcat, BOOL) DEF_INSTANTIATE_RANK1_6_ONE_RANK_ONE_TYPE(OpPad, FP16); DEF_INSTANTIATE_RANK1_6_ONE_RANK_ONE_TYPE(OpPad, FLOAT); DEF_INSTANTIATE_RANK1_6_ONE_RANK_ONE_TYPE(OpPad, INT8); DEF_INSTANTIATE_RANK1_6_ONE_RANK_ONE_TYPE(OpPad, INT16); DEF_INSTANTIATE_RANK1_6_ONE_RANK_ONE_TYPE(OpPad, INT32); DEF_INSTANTIATE_RANK1_6_ONE_RANK_ONE_TYPE(OpPad, BOOL); DEF_INSTANTIATE_RESHAPE(OpReshape, FP16); DEF_INSTANTIATE_RESHAPE(OpReshape, FLOAT); DEF_INSTANTIATE_RESHAPE(OpReshape, INT8); DEF_INSTANTIATE_RESHAPE(OpReshape, INT16); DEF_INSTANTIATE_RESHAPE(OpReshape, INT32); DEF_INSTANTIATE_RESHAPE(OpReshape, BOOL); DEF_INSTANTIATE_RANK1_6_ONE_RANK_ONE_TYPE(OpReverse, FP16); DEF_INSTANTIATE_RANK1_6_ONE_RANK_ONE_TYPE(OpReverse, FLOAT); DEF_INSTANTIATE_RANK1_6_ONE_RANK_ONE_TYPE(OpReverse, INT8); DEF_INSTANTIATE_RANK1_6_ONE_RANK_ONE_TYPE(OpReverse, INT16); DEF_INSTANTIATE_RANK1_6_ONE_RANK_ONE_TYPE(OpReverse, INT32); DEF_INSTANTIATE_RANK1_6_ONE_RANK_ONE_TYPE(OpReverse, BOOL); DEF_INSTANTIATE_RANK0_6_ONE_RANK_ONE_TYPE(OpSlice, FP16); DEF_INSTANTIATE_RANK0_6_ONE_RANK_ONE_TYPE(OpSlice, FLOAT); DEF_INSTANTIATE_RANK0_6_ONE_RANK_ONE_TYPE(OpSlice, INT8); DEF_INSTANTIATE_RANK0_6_ONE_RANK_ONE_TYPE(OpSlice, INT16); DEF_INSTANTIATE_RANK0_6_ONE_RANK_ONE_TYPE(OpSlice, INT32); DEF_INSTANTIATE_RANK0_6_ONE_RANK_ONE_TYPE(OpSlice, BOOL); DEF_INSTANTIATE_RANK0_6_ONE_RANK_ONE_TYPE(OpTile, FP16); DEF_INSTANTIATE_RANK0_6_ONE_RANK_ONE_TYPE(OpTile, FLOAT); DEF_INSTANTIATE_RANK0_6_ONE_RANK_ONE_TYPE(OpTile, INT8); DEF_INSTANTIATE_RANK0_6_ONE_RANK_ONE_TYPE(OpTile, INT16); DEF_INSTANTIATE_RANK0_6_ONE_RANK_ONE_TYPE(OpTile, INT32); DEF_INSTANTIATE_RANK0_6_ONE_RANK_ONE_TYPE(OpTile, BOOL); DEF_INSTANTIATE_RANK0_6_ONE_RANK_ONE_TYPE(OpTranspose, FP16); DEF_INSTANTIATE_RANK0_6_ONE_RANK_ONE_TYPE(OpTranspose, FLOAT); DEF_INSTANTIATE_RANK0_6_ONE_RANK_ONE_TYPE(OpTranspose, INT8); DEF_INSTANTIATE_RANK0_6_ONE_RANK_ONE_TYPE(OpTranspose, INT16); DEF_INSTANTIATE_RANK0_6_ONE_RANK_ONE_TYPE(OpTranspose, INT32); DEF_INSTANTIATE_RANK0_6_ONE_RANK_ONE_TYPE(OpTranspose, BOOL);