update ggml and propogate changes; embed and textgen works

Author: iamlemec

gadget/compute.py

@@ -21,15 +21,13 @@
     ggml_backend_free,
     ggml_backend_alloc_ctx_tensors,
     ggml_backend_get_default_buffer_type,
-    ggml_backend_cpu_set_n_threads,
     ggml_backend_graph_compute,
     ggml_gallocr_new,
     ggml_gallocr_reserve,
     ggml_gallocr_alloc_graph,
     ggml_gallocr_free,
 )
 from .tensor import (
-    get_framework,
     get_tensor_name,
     get_tensor_info,
     create_tensor,
@@ -295,7 +293,6 @@ def test_torch(input_dim=256, output_dim=32, batch_size=16, qtype=T.F32, backend
 
     # define model function
     def test_model(ctx, par, ten):
-        n, m = par['input_dim'], par['output_dim']
         a, b, x = ten['a'], ten['b'], ten['x']
         x1 = ggml_mul_mat(ctx, a, x, name=f'x1')
         x2 = ggml_add(ctx, x1, b, name=f'x2')

gadget/ggml.py

@@ -50,7 +50,7 @@
     ggml_element_size,
     ggml_nelements,
     ggml_nbytes,
-    ggml_internal_get_type_traits,
+    ggml_get_type_traits,
     # tensor ops
     ggml_dup,
     ggml_dup_inplace,
@@ -153,17 +153,13 @@
     ggml_soft_max,
     ggml_soft_max_inplace,
     ggml_soft_max_ext,
-    ggml_soft_max_back,
-    ggml_soft_max_back_inplace,
     ggml_rope,
     ggml_rope_inplace,
     ggml_rope_ext,
     ggml_rope_ext_inplace,
     ggml_rope_yarn_corr_dims,
-    ggml_rope_back,
     ggml_clamp,
     ggml_im2col,
-    ggml_conv_depthwise_2d,
     ggml_conv_1d,
     ggml_conv_1d_ph,
     ggml_conv_transpose_1d,

gadget/libs/_libggml.py

@@ -268,23 +268,18 @@ class ggml_backend_buffer_type_context(ctypes.Structure): ...
     ("op"       , ctypes.c_int                           ),
     ("op_params", ctypes.c_int32 * GGML_MAX_OP_PARAMS_INT),
     ("flags"    , ctypes.c_int32                         ),
-    ("grad"     , ggml_tensor_p                          ),
     ("src"      , ggml_tensor_p * GGML_MAX_SRC           ),
     ("view_src" , ggml_tensor_p                          ),
     ("view_offs", ctypes.c_size_t                        ),
     ("data"     , ctypes.c_void_p                        ),
     ("name"     , ctypes.c_char * GGML_MAX_NAME          ),
     ("extra"    , ctypes.c_void_p                        ),
+    ("padding"  , ctypes.c_char * 8                      ),
 ]
 
 # types and quantization
 ggml_to_float_p          = ctypes.CFUNCTYPE(None, ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_int64)
 ggml_from_float_p        = ctypes.CFUNCTYPE(None, ctypes.POINTER(ctypes.c_float), ctypes.c_void_p, ctypes.c_int64)
-ggml_from_float_to_mat_p = ctypes.CFUNCTYPE(None, ctypes.POINTER(ctypes.c_float), ctypes.c_void_p, ctypes.c_int64, ctypes.c_int64, ctypes.c_int64)
-ggml_vec_dot_p           = ctypes.CFUNCTYPE(None, ctypes.c_int, ctypes.POINTER(ctypes.c_float), ctypes.c_size_t, ctypes.c_void_p, ctypes.c_size_t, ctypes.c_void_p, ctypes.c_size_t, ctypes.c_int)
-ggml_vec_dot_p           = ctypes.CFUNCTYPE(None, ctypes.c_int, ctypes.POINTER(ctypes.c_float), ctypes.c_size_t, ctypes.c_void_p, ctypes.c_size_t, ctypes.c_void_p, ctypes.c_size_t, ctypes.c_int)
-ggml_gemv_p              = ctypes.CFUNCTYPE(None, ctypes.c_int, ctypes.POINTER(ctypes.c_float), ctypes.c_size_t, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_int)
-ggml_gemm_p              = ctypes.CFUNCTYPE(None, ctypes.c_int, ctypes.POINTER(ctypes.c_float), ctypes.c_size_t, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_int)
 
 class ggml_type_traits(ctypes.Structure):
     _fields_ = [
@@ -294,15 +289,7 @@ class ggml_type_traits(ctypes.Structure):
         ("type_size"           , ctypes.c_size_t         ),
         ("is_quantized"        , ctypes.c_bool           ),
         ("to_float"            , ggml_to_float_p         ),
-        ("from_float"          , ggml_from_float_p       ),
         ("from_float_ref"      , ggml_from_float_p       ),
-        ("from_float_to_mat"   , ggml_from_float_to_mat_p),
-        ("vec_dot"             , ggml_vec_dot_p          ),
-        ("vec_dot_type"        , ctypes.c_int            ),
-        ("nrows"               , ctypes.c_int64          ),
-        ("ncols"               , ctypes.c_int64          ),
-        ("gemv"                , ggml_gemv_p             ),
-        ("gemm"                , ggml_gemm_p             ),
     ]
 ggml_type_traits_p = ctypes.POINTER(ggml_type_traits)
 
@@ -660,9 +647,9 @@ def ggml_is_contiguous(tensor): ...
 
 @ctypes_function(_ggml,
     [ctypes.c_int],
-    ggml_type_traits
+    ggml_type_traits_p
 )
-def ggml_internal_get_type_traits(ttype): ...
+def ggml_get_type_traits(ttype): ...
 
 ## graphs
 
@@ -1306,18 +1293,6 @@ def ggml_soft_max_inplace(ctx, a): ...
 )
 def ggml_soft_max_ext(ctx, a, mask, scale, max_bias): ...
 
-@ctypes_function(_ggml,
-    [ggml_context_p, ggml_tensor_p, ggml_tensor_p],
-    ggml_tensor_p
-)
-def ggml_soft_max_back(ctx, a, b): ...
-
-@ctypes_function(_ggml,
-    [ggml_context_p, ggml_tensor_p, ggml_tensor_p],
-    ggml_tensor_p
-)
-def ggml_soft_max_back_inplace(ctx, a, b): ...
-
 @ctypes_function(_ggml,
     [ggml_context_p, ggml_tensor_p, ggml_tensor_p, ctypes.c_int, ctypes.c_int],
     ggml_tensor_p
@@ -1348,12 +1323,6 @@ def ggml_rope_ext_inplace(ctx, a, b, c, n_dims, mode, n_ctx_orig, freq_base, fre
 )
 def ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, dims): ...
 
-@ctypes_function(_ggml,
-    [ggml_context_p, ggml_tensor_p, ggml_tensor_p, ggml_tensor_p, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_float, ctypes.c_float, ctypes.c_float, ctypes.c_float, ctypes.c_float, ctypes.c_float],
-    ggml_tensor_p
-)
-def ggml_rope_back(ctx, a, b, c, n_dims, mode, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow): ...
-
 @ctypes_function(_ggml,
     [ggml_context_p, ggml_tensor_p, ctypes.c_float, ctypes.c_float],
     ggml_tensor_p
@@ -1366,12 +1335,6 @@ def ggml_clamp(ctx, a, min, max): ...
 )
 def ggml_im2col(ctx, a, b, s0, s1, p0, p1, d0, d1, is_2D, dst_type): ...
 
-@ctypes_function(_ggml,
-    [ggml_context_p, ggml_tensor_p, ggml_tensor_p, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int],
-    ggml_tensor_p
-)
-def ggml_conv_depthwise_2d(ctx, a, b, s0, s1, p0, p1, d0, d1): ...
-
 @ctypes_function(_ggml,
     [ggml_context_p, ggml_tensor_p, ggml_tensor_p, ctypes.c_int, ctypes.c_int, ctypes.c_int],
     ggml_tensor_p

gadget/libs/constants.py

@@ -55,6 +55,10 @@ class GGMLQuantizationType(IntEnum):
     F64     = 28
     IQ1_M   = 29
     BF16    = 30
+    TQ1_0   = 34
+    TQ2_0   = 35
+    MXFP4   = 39
+    COUNT   = 40
 
 # embed pooling types
 class LlamaPoolingType(IntEnum):

gadget/models/bert.py

@@ -1,9 +1,6 @@
 # bert implementation
 
-import numpy as np
-
 from ..ggml import (
-    ggml_add,
     ggml_add_inplace,
     ggml_get_rows,
     ggml_view_1d,
@@ -13,7 +10,6 @@
 )
 from ..model import GgmlModel, Parameter, State, Tensor
 from .layers import (
-    linear_layer,
     norm_layer,
     attention_layer,
     feed_forward_layer,

gadget/models/llama.py

@@ -1,17 +1,16 @@
-# llama implementation
+# llama implementation (llama-3.1)
 
 import numpy as np
 
 from ..ggml import (
     ggml_element_size,
     ggml_add_inplace,
     ggml_get_rows,
-    ggml_transpose,
     ggml_view_1d,
     ggml_view_2d,
     ggml_cont,
 )
-from ..tensor import get_tensor_shape, get_tensor_info
+from ..tensor import get_tensor_shape
 from ..model import GgmlModel, Parameter, State, Tensor
 from .cache import KVCache
 from .layers import (
@@ -107,7 +106,6 @@ def forward(self):
         ctx = self.ctx_graph
 
         # get runtime state
-        batch_size, context_length = self.params['batch_size', 'context_length']
         n_past, n_tokens = self.state['n_past', 'n_tokens']
 
         # get params

gadget/tensor.py

@@ -17,7 +17,7 @@
     ggml_new_tensor_4d,
     ggml_backend_tensor_set,
     ggml_backend_tensor_get,
-    ggml_internal_get_type_traits,
+    ggml_get_type_traits,
     ggml_backend_buffer_is_host,
 )
 
@@ -120,8 +120,8 @@ def trim_nelem(shape):
     return shape[:dims]
 
 def get_type_traits(ttype):
-    traits = ggml_internal_get_type_traits(ttype)
-    return traits.blck_size, traits.type_size
+    traits = ggml_get_type_traits(ttype)
+    return traits.contents.blck_size, traits.contents.type_size
 
 def get_tensor_name(tensor):
     value = tensor.contents
@@ -274,7 +274,7 @@ def array_to_tensor(array, tensor, offset=0, strict=True):
         src_p = ctypes.cast(src, ctypes.POINTER(ctypes.c_float))
         dst_p = ctypes.cast(dst, ctypes.c_void_p)
         size = ggml_nelements(tensor)
-        traits = ggml_internal_get_type_traits(ttype)
+        traits = ggml_get_type_traits(ttype)
         traits.from_float(src_p, dst_p, size)
     else:
         src_p = ctypes.cast(src, ctypes.c_void_p)
@@ -306,7 +306,7 @@ def tensor_to_array(tensor, framework='numpy', device='cpu'):
         src_p = ctypes.cast(src, ctypes.c_void_p)
         dst_p = ctypes.cast(dst, ctypes.POINTER(ctypes.c_float))
         size = ggml_nelements(tensor)
-        traits = ggml_internal_get_type_traits(ttype)
+        traits = ggml_get_type_traits(ttype)
         traits.to_float(src_p, dst_p, size)
     else:
         dst_p = ctypes.cast(dst, ctypes.c_void_p)

gadget/textgen.py

@@ -1,6 +1,5 @@
 # text generation
 
-import numpy as np
 from transformers import AutoTokenizer
 
 from .loader import GgufFile
@@ -100,10 +99,10 @@ def stream_chat(self, message, prefill=None, add_generation_prompt=True, **kwarg
         self.history.append({'role': 'assistant', 'content': reply})
 
     def generate_chat(self, message, **kwargs):
-        tokens = []
-        for tok in self.stream_chat(message, **kwargs):
-            tokens += [tok]
-        return self.detokenize(tokens)
+        reply = ''
+        for chunk in self.stream_chat(message, **kwargs):
+            reply += chunk
+        return reply
 
 def test_logits(gguf_path, model_id, model_class=LlamaModel, batch_size=128, **kwargs):
     model = TextGen(gguf_path, model_id, model_class=model_class, batch_size=batch_size, **kwargs)