Skip to content

FP8 weight surpport for v79+ and dit optimizations #28952

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
happyyzy wants to merge 1 commit into
base: master
Choose a base branch
from
Open

FP8 weight surpport for v79+ and dit optimizations #28952

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 62 additions & 1 deletion ggml/include/ggml.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -430,7 +430,8 @@ extern "C" {
GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale)
GGML_TYPE_Q1_0 = 41,
GGML_TYPE_Q2_0 = 42,
GGML_TYPE_COUNT = 43,
GGML_TYPE_F8_E4M3 = 43,
GGML_TYPE_COUNT = 44,
};

// [TAG_GGML_PREC]
Expand Down Expand Up @@ -601,6 +602,12 @@ extern "C" {

GGML_OP_GLU,

GGML_OP_MUL_MAT_SEGMENTED,
GGML_OP_QKNORM_ROPE,
GGML_OP_GROUP_NORM_AFFINE_SILU,
GGML_OP_CONV_2D_BIAS,
GGML_OP_CONV_2D_UPSCALE,

GGML_OP_COUNT,
};

Expand Down Expand Up @@ -1406,6 +1413,13 @@ extern "C" {
struct ggml_tensor * a,
float eps);

GGML_API struct ggml_tensor * ggml_qknorm_rope(
struct ggml_context * ctx,
struct ggml_tensor * a,
struct ggml_tensor * weight,
struct ggml_tensor * theta,
float eps);

// group normalize along ne0*ne1*n_groups
// used in stable-diffusion
GGML_API struct ggml_tensor * ggml_group_norm(
Expand All @@ -1420,6 +1434,22 @@ extern "C" {
int n_groups,
float eps);

GGML_API struct ggml_tensor * ggml_group_norm_affine_silu(
struct ggml_context * ctx,
struct ggml_tensor * a,
struct ggml_tensor * weight,
struct ggml_tensor * bias,
int n_groups,
float eps);

GGML_API struct ggml_tensor * ggml_group_norm_affine_silu_inplace(
struct ggml_context * ctx,
struct ggml_tensor * a,
struct ggml_tensor * weight,
struct ggml_tensor * bias,
int n_groups,
float eps);

// l2 normalize along rows
// used in rwkv v7
GGML_API struct ggml_tensor * ggml_l2_norm(
Expand Down Expand Up @@ -1484,6 +1514,12 @@ extern "C" {
struct ggml_tensor * a,
struct ggml_tensor * b);

GGML_API struct ggml_tensor * ggml_mul_mat_segmented(
struct ggml_context * ctx,
struct ggml_tensor * a,
struct ggml_tensor * b0,
struct ggml_tensor * b1);

// change the precision of a matrix multiplication
// set to GGML_PREC_F32 for higher precision (useful for phi-2)
GGML_DEPRECATED(GGML_API void ggml_mul_mat_set_prec(
Expand Down Expand Up @@ -2255,6 +2291,31 @@ extern "C" {
int d0, // dilation dimension 0
int d1); // dilation dimension 1

GGML_API struct ggml_tensor * ggml_conv_2d_direct_bias(
struct ggml_context * ctx,
struct ggml_tensor * a,
struct ggml_tensor * b,
struct ggml_tensor * bias,
int s0,
int s1,
int p0,
int p1,
int d0,
int d1);

GGML_API struct ggml_tensor * ggml_conv_2d_direct_upscale(
struct ggml_context * ctx,
struct ggml_tensor * a,
struct ggml_tensor * b,
struct ggml_tensor * bias,
int upscale_factor,
int s0,
int s1,
int p0,
int p1,
int d0,
int d1);

GGML_API struct ggml_tensor * ggml_conv_3d_direct(
struct ggml_context * ctx,
struct ggml_tensor * a, // kernel [KW, KH, KD, IC * OC]
Expand Down
14 changes: 14 additions & 0 deletions ggml/src/ggml-cpu/ggml-cpu.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2141,6 +2141,12 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm
ggml_compute_forward_opt_step_sgd(params, tensor);
}
break;
case GGML_OP_MUL_MAT_SEGMENTED:
case GGML_OP_QKNORM_ROPE:
case GGML_OP_GROUP_NORM_AFFINE_SILU:
case GGML_OP_CONV_2D_BIAS:
case GGML_OP_CONV_2D_UPSCALE:
GGML_ABORT("operation is not supported by the CPU backend");
case GGML_OP_NONE:
{
// nop
Expand Down Expand Up @@ -2489,6 +2495,14 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
{
n_tasks = n_threads;
} break;
case GGML_OP_MUL_MAT_SEGMENTED:
case GGML_OP_QKNORM_ROPE:
case GGML_OP_GROUP_NORM_AFFINE_SILU:
case GGML_OP_CONV_2D_BIAS:
case GGML_OP_CONV_2D_UPSCALE:
{
n_tasks = 1;
} break;
case GGML_OP_NONE:
{
n_tasks = 1;
Expand Down
6 changes: 6 additions & 0 deletions ggml/src/ggml-cpu/ggml-cpu.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -472,6 +472,12 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st
src1->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32;
case GGML_OP_CONV_2D:
return ggml_is_contiguous(op->src[0]);
case GGML_OP_MUL_MAT_SEGMENTED:
case GGML_OP_QKNORM_ROPE:
case GGML_OP_GROUP_NORM_AFFINE_SILU:
case GGML_OP_CONV_2D_BIAS:
case GGML_OP_CONV_2D_UPSCALE:
return false;
case GGML_OP_SSM_SCAN:
return ggml_get_op_params_i32(op, 0) == 1 || op->src[3]->ne[0] == 1;
default:
Expand Down
1 change: 1 addition & 0 deletions ggml/src/ggml-cpu/ops.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5907,6 +5907,7 @@ void ggml_compute_forward_clamp(
ggml_compute_forward_clamp_f16(params, dst);
} break;
case GGML_TYPE_BF16:
case GGML_TYPE_F8_E4M3:
case GGML_TYPE_Q1_0:
case GGML_TYPE_Q2_0:
case GGML_TYPE_Q4_0:
Expand Down
Loading