koboldcpp/kcpp_backend.h at concedo · LostRuins/koboldcpp · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
#pragma once

#include "ggml-backend.h"
#include "ggml_v2.h"
#include "ggml_v3.h"

// backends with GGML_USE_CUDA
#define KCPP_BACKENDS_USE_CUDA "cuda|rocm"

// Metal backend aliases
#define KCPP_BACKENDS_METAL "metal|mtl"

// backends that support blas
#define KCPP_BACKENDS_BLAS "blas|cuda|rocm|vulkan|sycl"

// checks if the provided backend (or the first one) matches a |-separated name list
int kcpp_backend_check(const char* name_list, ggml_backend_t backend = nullptr);

ggml_backend_dev_t kcpp_backend_get_device(int index);
ggml_backend_dev_t kcpp_backend_get_gpu_device(int index);

// per-backend aux functions

void kcpp_backend_cuda_ggmlv2_transform_tensor(ggml_v2_tensor * tensor);

void kcpp_backend_cuda_ggmlv3_assign_buffers(struct ggml_v3_tensor * tensor);
void kcpp_backend_cuda_ggmlv3_assign_buffers_force_inplace(struct ggml_v3_tensor * tensor);
void kcpp_backend_cuda_ggmlv3_assign_buffers_no_scratch(struct ggml_v3_tensor * tensor);
void kcpp_backend_cuda_ggmlv3_free_data(struct ggml_v3_tensor * tensor);
void kcpp_backend_cuda_ggmlv3_free_scratch(void);
void kcpp_backend_cuda_ggmlv3_set_main_device(int device);
void kcpp_backend_cuda_ggmlv3_set_mul_mat_q(bool mul_mat_q);
void kcpp_backend_cuda_ggmlv3_set_scratch_size(size_t scratch_size);
void kcpp_backend_cuda_ggmlv3_set_tensor_split(const float * tensor_split);
void kcpp_backend_cuda_ggmlv3_transform_tensor(void * data, struct ggml_v3_tensor * tensor);

void kcpp_backend_cuda_set_mul_mat_q(int use_mmq);

void kcpp_backend_hip_initialize();

bool kcpp_backend_metal_supports_family(ggml_backend_t backend, int family);