mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-11-11 14:22:09 +01:00
Max Krasnyansky
39fb81f875
* hexagon: refactor set/get/sum-rows ops to use local context * hexagon: refactor ROPE and Softmax Ops to use local context Improves performance a bit by precomputing things and saving in the context. * hexagon: refactor activation ops to use local context struct * hexagon: refactor unary ops to use local context struct and DMA/VTCM * hexagon: use aligned hvx_scale function * hexagon: remove unused fields from op_context * hexagon: rewrite ROPE to use DMA and VTCM scratchpad * hex-rope: keep N rows in scratchpad (instead of just two) * hex-rope: introduce rowidx cache * hex-rope: remove unused fields * hex-rope: rewrite dma prefetch logic to allow for multi-row fetch/compute also removes the need for fastdiv. * hex-rope: minor formatting * hex-rope: use indices and unroll the loops * hex-rope: more updates to cleanup rope-block handling * hexagon: cleanup supported type/dims checks * hexagon: all reduce funcs replicated across lanes There is no need to explicitly replicate the first value. * snapdragon: update adb and windows scripts to use ubatch-size 256 Updated Ops support handles larger ubatches.
112 lines
3.8 KiB
C
112 lines
3.8 KiB
C
#pragma clang diagnostic ignored "-Wunused-variable"
|
|
#pragma clang diagnostic ignored "-Wunused-function"
|
|
#pragma clang diagnostic ignored "-Wunused-but-set-variable"
|
|
|
|
#include <HAP_farf.h>
|
|
#include <HAP_perf.h>
|
|
|
|
#include <math.h>
|
|
#include <string.h>
|
|
|
|
#define GGML_COMMON_DECL_C
|
|
#include "ggml-common.h"
|
|
#include "htp-ctx.h"
|
|
#include "htp-msg.h"
|
|
#include "htp-ops.h"
|
|
#include "hvx-utils.h"
|
|
|
|
struct get_rows_context {
|
|
struct htp_ops_context * octx;
|
|
uint32_t src1_nrows_per_thread;
|
|
struct fastdiv_values get_rows_div_ne10;
|
|
struct fastdiv_values get_rows_div_ne10_ne11;
|
|
};
|
|
|
|
#define get_rows_preamble \
|
|
const uint32_t ne00 = octx->src0.ne[0]; \
|
|
const uint32_t ne01 = octx->src0.ne[1]; \
|
|
const uint32_t ne02 = octx->src0.ne[2]; \
|
|
const uint32_t ne03 = octx->src0.ne[3]; \
|
|
\
|
|
const uint32_t ne10 = octx->src1.ne[0]; \
|
|
const uint32_t ne11 = octx->src1.ne[1]; \
|
|
const uint32_t ne12 = octx->src1.ne[2]; \
|
|
\
|
|
const uint32_t nb01 = octx->src0.nb[1]; \
|
|
const uint32_t nb02 = octx->src0.nb[2]; \
|
|
const uint32_t nb03 = octx->src0.nb[3]; \
|
|
\
|
|
const uint32_t nb10 = octx->src1.nb[0]; \
|
|
const uint32_t nb11 = octx->src1.nb[1]; \
|
|
const uint32_t nb12 = octx->src1.nb[2]; \
|
|
\
|
|
const uint32_t nb1 = octx->dst.nb[1]; \
|
|
const uint32_t nb2 = octx->dst.nb[2]; \
|
|
const uint32_t nb3 = octx->dst.nb[3]; \
|
|
\
|
|
const uint32_t nr = ne10 * ne11 * ne12;
|
|
|
|
static void get_rows_thread_f32_f32(unsigned int nth, unsigned int ith, void *data) {
|
|
struct get_rows_context * grctx = (struct get_rows_context *)data;
|
|
struct htp_ops_context * octx = grctx->octx;
|
|
get_rows_preamble;
|
|
|
|
// parallelize by src1 elements (which correspond to dst rows)
|
|
const uint32_t dr = grctx->src1_nrows_per_thread;
|
|
const uint32_t ir0 = dr * ith;
|
|
const uint32_t ir1 = (ir0 + dr < nr) ? (ir0 + dr) : nr;
|
|
|
|
const bool is_i32 = (octx->src1.type == HTP_TYPE_I32);
|
|
|
|
for (uint32_t i = ir0; i < ir1; ++i) {
|
|
const uint32_t i12 = fastdiv(i, &grctx->get_rows_div_ne10_ne11);
|
|
const uint32_t rem = i - i12 * ne11 * ne10;
|
|
const uint32_t i11 = fastdiv(rem, &grctx->get_rows_div_ne10);
|
|
const uint32_t i10 = rem - i11 * ne10;
|
|
|
|
const uintptr_t src1_addr = octx->src1.data + i10*nb10 + i11*nb11 + i12*nb12;
|
|
|
|
uint32_t i01 = is_i32 ? *(int32_t *)src1_addr : *(int64_t *)src1_addr;
|
|
|
|
if (i01 >= ne01) {
|
|
// invalid index, skip for now to avoid crash
|
|
continue;
|
|
}
|
|
|
|
const uintptr_t src0_ptr = octx->src0.data + i01*nb01 + i11*nb02 + i12*nb03;
|
|
const uintptr_t dst_ptr = octx->dst.data + i10*nb1 + i11*nb2 + i12*nb3;
|
|
hvx_copy_f32_uu((uint8_t *)dst_ptr, (const uint8_t *)src0_ptr, ne00);
|
|
}
|
|
}
|
|
|
|
int op_get_rows(struct htp_ops_context * octx) {
|
|
get_rows_preamble;
|
|
|
|
if (octx->src0.type != HTP_TYPE_F32) {
|
|
return HTP_STATUS_NO_SUPPORT;
|
|
}
|
|
|
|
if (octx->dst.type != HTP_TYPE_F32) {
|
|
return HTP_STATUS_NO_SUPPORT;
|
|
}
|
|
|
|
if (octx->src1.type != HTP_TYPE_I32 && octx->src1.type != HTP_TYPE_I64) {
|
|
return HTP_STATUS_NO_SUPPORT;
|
|
}
|
|
|
|
if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) {
|
|
return HTP_STATUS_OK;
|
|
}
|
|
|
|
struct get_rows_context grctx;
|
|
grctx.octx = octx;
|
|
grctx.get_rows_div_ne10 = init_fastdiv_values(octx->src1.ne[0]);
|
|
grctx.get_rows_div_ne10_ne11 = init_fastdiv_values(octx->src1.ne[0] * octx->src1.ne[1]);
|
|
|
|
const uint32_t n_jobs = MIN(nr, octx->n_threads);
|
|
grctx.src1_nrows_per_thread = (nr + n_jobs - 1) / n_jobs;
|
|
|
|
worker_pool_run_func(octx->ctx->worker_pool, get_rows_thread_f32_f32, &grctx, n_jobs);
|
|
return HTP_STATUS_OK;
|
|
}
|