From ec9bc0f12cb8a9335fc6979a0d5abf3a9d2a8207 Mon Sep 17 00:00:00 2001 From: gspark-etri Date: Fri, 30 Oct 2026 15:51:51 +0900 Subject: [PATCH 1/2] Decide sub-chunking by file size, report placement classes (#74) New stores sub-chunk new chunks of files of at least --sub-min-file (default 1 MiB) instead of by ATOM file kind; stores that recorded sub_kinds keep them. --sub-probe is an experiment for the measurement. dedup-report --classify reports referenced / embedded-rearranged / embedded-shifted / new per file from chunk shares alone. Co-Authored-By: Claude Opus 5.5 --- cmd/onggi/csi.go | 2 +- cmd/onggi/main.go | 34 ++++++ cmd/onggi/store.go | 34 ++++-- pkg/dedup/classify.go | 232 +++++++++++++++++++++++++++++++++++++ pkg/dedup/classify_test.go | 97 ++++++++++++++++ pkg/dedup/report.go | 37 +++++- pkg/store/format.go | 17 ++- pkg/store/push.go | 18 ++- pkg/store/store.go | 47 +++++--- pkg/store/sub_test.go | 70 +++++++++++ 10 files changed, 553 insertions(+), 35 deletions(-) create mode 100644 pkg/dedup/classify.go create mode 100644 pkg/dedup/classify_test.go diff --git a/cmd/onggi/csi.go b/cmd/onggi/csi.go index 3c86d4e..1485479 100644 --- a/cmd/onggi/csi.go +++ b/cmd/onggi/csi.go @@ -76,7 +76,7 @@ func csiNode(args []string, stderr io.Writer) int { return nil, err } return store.OpenWith(ctx, cb, store.Create{Chunker: s.Config.Chunker, Sub: s.Config.Sub, - SubKinds: s.Config.SubKinds, PackSize: s.Config.PackSize}) + SubKinds: s.Config.SubKinds, SubMinFile: s.Config.SubMinFile, PackSize: s.Config.PackSize}) } cache, err := openCache(ctx) if err == nil { diff --git a/cmd/onggi/main.go b/cmd/onggi/main.go index ae9f0bc..c325c71 100644 --- a/cmd/onggi/main.go +++ b/cmd/onggi/main.go @@ -13,6 +13,7 @@ import ( "text/tabwriter" "time" + "github.com/ai-computing/onggi/pkg/chunker" "github.com/ai-computing/onggi/pkg/dedup" ) @@ -95,6 +96,16 @@ func dedupReport(args []string, stdout, stderr io.Writer) int { fixed := fs.String("fixed-block", "", "also measure fixed-size blocks of this size (e.g. 1MiB)") asJSON := fs.Bool("json", false, "print JSON instead of tables") quiet := fs.Bool("quiet", false, "no progress on stderr") + classify := fs.Bool("classify", false, "report each file's weight-placement class from its bytes: referenced, embedded-rearranged, embedded-shifted, new (#74)") + var refs []string + fs.Func("reference", "with --classify: a file or directory the set may reference, such as the source checkpoint (repeatable; links are followed)", func(v string) error { + refs = append(refs, v) + return nil + }) + fine := fs.String("fine", "4KiB", "with --classify: average fine chunk size; min and max are a quarter and four times it") + refShare := fs.Float64("ref-share", dedup.DefaultRefShare, "with --classify: referenced when at least this share of a file is in the references") + gap := fs.Float64("rearranged-gap", dedup.DefaultRearrangeGap, "with --classify: embedded-rearranged when fine chunks find at least this much more of a file elsewhere than coarse chunks") + shifted := fs.Float64("shifted-share", dedup.DefaultShiftedShare, "with --classify: embedded-shifted when coarse chunks find at least this share of a file elsewhere") fs.Usage = func() { fmt.Fprintln(stderr, "usage: onggi dedup-report [flags] ...") fs.PrintDefaults() @@ -125,6 +136,16 @@ func dedupReport(args []string, stdout, stderr io.Writer) int { if err == nil { err = o.Params.Validate() } + if err == nil && (*classify || len(refs) > 0) { + var avg int + if avg, err = size(*fine); err == nil { + o.Classify = &dedup.Classify{Fine: chunker.Params{Min: avg / 4, Avg: avg, Max: avg * 4}, References: refs, + RefShare: *refShare, RearrangedGap: *gap, ShiftedShare: *shifted} + if err = o.Classify.Fine.Validate(); err == nil && o.Classify.Fine.Max > o.Params.Min { + err = fmt.Errorf("fine chunk max %d exceeds chunk min %d", o.Classify.Fine.Max, o.Params.Min) + } + } + } if err != nil { fmt.Fprintln(stderr, "onggi:", err) return 2 @@ -190,6 +211,19 @@ func printReport(w io.Writer, r *dedup.Report, took time.Duration) { } tw.Flush() } + if c := r.Classify; c != nil { + fmt.Fprintf(w, "\nplacement (fine chunks %s; referenced >= %.2f in %d reference files, rearranged: fine - coarse >= %.2f, shifted: coarse >= %.2f):\n", + human(int64(c.Fine.Avg)), c.RefShare, c.RefFiles, c.RearrangedGap, c.ShiftedShare) + tw = tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) + fmt.Fprintln(tw, " CLASS\tBYTES\tREF\tCOARSE\tFINE\tPATH") + for _, f := range r.Files { + if f.Shares != nil { + fmt.Fprintf(tw, " %s\t%s\t%.1f %%\t%.1f %%\t%.1f %%\t%s\n", f.Class, human(f.Bytes), 100*f.Shares.Ref, + 100*f.Shares.Coarse, 100*f.Shares.Fine, f.Path) + } + } + tw.Flush() + } if len(r.Skipped) > 0 { fmt.Fprintf(w, "\nskipped %d symlinks or unreadable entries\n", len(r.Skipped)) } diff --git a/cmd/onggi/store.go b/cmd/onggi/store.go index 21ba66c..0b5fc58 100644 --- a/cmd/onggi/store.go +++ b/cmd/onggi/store.go @@ -25,7 +25,7 @@ import ( ) type storeFlags struct { - url, durability, min, avg, max, sub, subKinds, packSize, frame *string + url, durability, min, avg, max, sub, subKinds, subMinFile, subProbe, packSize, frame *string } func addStoreFlags(fs *flag.FlagSet) storeFlags { @@ -36,7 +36,9 @@ func addStoreFlags(fs *flag.FlagSet) storeFlags { avg: fs.String("avg", "64KiB", "average chunk size (only when creating a store)"), max: fs.String("max", "256KiB", "maximum chunk size (only when creating a store)"), sub: fs.String("sub", "4KiB", "average sub-chunk size for two-level chunking, 0 for one level (only when creating a store)"), - subKinds: fs.String("sub-kinds", "atom-prefill,atom-decode,atom-other", "file kinds that get sub-chunks, or all (only when creating a store)"), + subKinds: fs.String("sub-kinds", "", "file kinds (pkg/layout, by name) that get sub-chunks, or all; overrides --sub-min-file (only when creating a store; the behaviour before #74)"), + subMinFile: fs.String("sub-min-file", "1MiB", "sub-chunk new chunks of files of at least this size, whatever their name or format (only when creating a store)"), + subProbe: fs.String("sub-probe", "", "experiment: N:SHARE stops sub-chunking a file when its first N new chunks find less than SHARE of their sub-chunk bytes stored (only when creating a store)"), packSize: fs.String("pack-size", "64MiB", "pack object size (only when creating a store)"), frame: fs.String("frame", "256KiB", "compress new chunks of sub-chunked files together up to this size, 0 for one by one (only when creating a store)"), } @@ -95,11 +97,26 @@ func (f storeFlags) open(ctx context.Context) (*store.Store, backend.Refs, error } c.Frame = int64(fr) } - if *f.subKinds != "all" { + switch *f.subKinds { + case "all": + case "": + mf, err := size(*f.subMinFile) + if err != nil { + return nil, nil, err + } + c.SubMinFile = int64(mf) + default: for _, k := range strings.Split(*f.subKinds, ",") { c.SubKinds = append(c.SubKinds, layout.Kind(strings.TrimSpace(k))) } } + if *f.subProbe != "" { + var p store.SubProbe + if _, err := fmt.Sscanf(*f.subProbe, "%d:%g", &p.Chunks, &p.MinShare); err != nil || p.Chunks < 1 { + return nil, nil, fmt.Errorf("bad --sub-probe %q: want N:SHARE", *f.subProbe) + } + c.SubProbe = &p + } } s, err := store.OpenWith(ctx, b, c) return s, b, err @@ -226,7 +243,7 @@ func pull(args []string, stdout, stderr io.Writer) int { cb, err := bfs.Open(*cacheDir, backend.DurabilityNone) if err == nil { o.Cache, err = store.OpenWith(ctx, cb, store.Create{Chunker: s.Config.Chunker, Sub: s.Config.Sub, - SubKinds: s.Config.SubKinds, PackSize: s.Config.PackSize}) + SubKinds: s.Config.SubKinds, SubMinFile: s.Config.SubMinFile, PackSize: s.Config.PackSize}) } if err != nil { fmt.Fprintln(stderr, "onggi: node cache:", err) @@ -311,9 +328,12 @@ func ls(args []string, stdout, stderr io.Writer) int { fmt.Fprintf(stdout, "\nstore: %d chunks in %d packs, %s; chunker %s/%s/%s; durability %s\n", chunks, packs, human(packBytes), human(int64(c.Min)), human(int64(c.Avg)), human(int64(c.Max)), s.B.Caps().Durability) if sc := s.Config.Sub; sc != nil { - kinds := "all kinds" - if len(s.Config.SubKinds) > 0 { - kinds = fmt.Sprint(s.Config.SubKinds) + kinds := "every file" + switch { + case len(s.Config.SubKinds) > 0: + kinds = "kinds " + fmt.Sprint(s.Config.SubKinds) + case s.Config.SubMinFile > 0: + kinds = "files of at least " + human(s.Config.SubMinFile) } fmt.Fprintf(stdout, "two-level: %d chunks held as sub-chunks of %s/%s/%s (%s)\n", s.Composites(), human(int64(sc.Min)), human(int64(sc.Avg)), human(int64(sc.Max)), kinds) diff --git a/pkg/dedup/classify.go b/pkg/dedup/classify.go new file mode 100644 index 0000000..9dca371 --- /dev/null +++ b/pkg/dedup/classify.go @@ -0,0 +1,232 @@ +package dedup + +import ( + "crypto/sha256" + "encoding/binary" + "errors" + "io/fs" + "os" + "path/filepath" + "sync" + + "github.com/ai-computing/onggi/pkg/chunker" +) + +// Placement classes (#74). Each names how the bytes a file shares with the rest of the set and +// with the reference files can be found, which bytes alone reveal (ADR-0009: no format is read): +// +// - referenced: at least RefShare of the file's bytes are in the reference files (a checkpoint) +// at coarse chunks; any chunking finds them there. +// - embedded-rearranged: fine chunks find at least RearrangedGap more of the file's bytes +// elsewhere than coarse chunks do; the shared bytes lie in runs shorter than a coarse chunk. +// - embedded-shifted: coarse chunks find at least ShiftedShare of the file's bytes elsewhere; +// they are copied whole, at whatever offsets. +// - new: none of the above. +// +// "Elsewhere" is any other file of the set or a reference file. The rules apply in this order. +const ( + Referenced = "referenced" + EmbeddedRearranged = "embedded-rearranged" + EmbeddedShifted = "embedded-shifted" + New = "new" + DefaultRefShare = 0.95 + DefaultRearrangeGap = 0.25 + DefaultShiftedShare = 0.5 +) + +// DefaultFine is the fine level: the store's sub-chunks (1 / 4 / 16 KiB). +var DefaultFine = chunker.Params{Min: 1 << 10, Avg: 4 << 10, Max: 16 << 10} + +// Classify configures the placement report. +type Classify struct { + Fine chunker.Params `json:"fine_params"` // each coarse chunk is split at these + References []string `json:"references,omitempty"` + RefShare float64 `json:"ref_share"` + RearrangedGap float64 `json:"rearranged_gap"` + ShiftedShare float64 `json:"shifted_share"` + RefFiles int `json:"reference_files"` + RefBytes int64 `json:"reference_bytes"` +} + +// Shares are the shares of a file's bytes found in the reference files (coarse chunks) and +// anywhere else (coarse and fine chunks). +type Shares struct { + Ref float64 `json:"ref"` + Coarse float64 `json:"coarse"` + Fine float64 `json:"fine"` +} + +// Class applies the rules above. +func (c *Classify) Class(s Shares) string { + switch { + case s.Ref >= c.RefShare: + return Referenced + case s.Fine-s.Coarse >= c.RearrangedGap: + return EmbeddedRearranged + case s.Coarse >= c.ShiftedShare: + return EmbeddedShifted + } + return New +} + +// fineKey names a fine chunk by the first 8 bytes of its SHA-256: a report needs no more, and it +// halves the memory of the fine index (a collision only moves bytes between shares). +func fineKey(h [32]byte) uint64 { return binary.LittleEndian.Uint64(h[:8]) } + +// splitFine hashes the fine chunks of one coarse chunk. +func splitFine(p chunker.Params, ch []byte, cl *chunkList) { + p.Split(ch, func(sub []byte) error { + cl.fine = append(cl.fine, fineKey(sha256.Sum256(sub))) + cl.fineSizes = append(cl.fineSizes, int32(len(sub))) + return nil + }) +} + +// refSets chunks the reference files and returns their coarse and fine chunk sets. +func refSets(c *Classify, o Options) (map[[32]byte]struct{}, map[uint64]struct{}, error) { + var paths []string + seen := map[string]bool{} + for _, root := range c.References { + // references may be a model hub snapshot, whose files are links to blobs: follow them + err := filepath.WalkDir(root, func(p string, d fs.DirEntry, werr error) error { + if werr != nil { + return werr + } + if d.IsDir() { + return nil + } + info, err := os.Stat(p) + if err != nil || !info.Mode().IsRegular() { + return nil + } + real, err := filepath.EvalSymlinks(p) + if err != nil { + return nil + } + if !seen[real] { + seen[real] = true + paths = append(paths, real) + c.RefFiles++ + c.RefBytes += info.Size() + } + return nil + }) + if err != nil { + return nil, nil, err + } + } + coarse, fine := map[[32]byte]struct{}{}, map[uint64]struct{}{} + var mu sync.Mutex + var firstErr error + jobs := make(chan string) + var wg sync.WaitGroup + for w := 0; w < max(o.Workers, 1); w++ { + wg.Add(1) + go func() { + defer wg.Done() + for p := range jobs { + cl := chunkFile(p, Options{Params: o.Params, Classify: o.Classify}) + mu.Lock() + if cl.err != nil && firstErr == nil { + firstErr = cl.err + } + for _, h := range cl.hashes { + coarse[h] = struct{}{} + } + for _, h := range cl.fine { + fine[h] = struct{}{} + } + mu.Unlock() + } + }() + } + for _, p := range paths { + jobs <- p + } + close(jobs) + wg.Wait() + if firstErr != nil { + return nil, nil, firstErr + } + if len(c.References) > 0 && len(paths) == 0 { + return nil, nil, errors.New("no reference files found") + } + return coarse, fine, nil +} + +// placement tracks, while files are merged in report order, which chunks occur in more than one +// file of the set, so each file's shares count only bytes found elsewhere. +type placement struct { + c *Classify + refCoarse map[[32]byte]struct{} + refFine map[uint64]struct{} + owner []int32 // per coarse chunk id: the one file holding it, or -1 for several + fineOwner map[uint64]int32 // the same for fine chunks + inRef [][]bool // per file and coarse chunk: in the reference files + refBytes []int64 + fine [][]uint64 + fineSizes [][]int32 +} + +func newPlacement(c *Classify, o Options, files int) (*placement, error) { + rc, rf, err := refSets(c, o) + if err != nil { + return nil, err + } + return &placement{c: c, refCoarse: rc, refFine: rf, fineOwner: map[uint64]int32{}, + inRef: make([][]bool, files), refBytes: make([]int64, files), + fine: make([][]uint64, files), fineSizes: make([][]int32, files)}, nil +} + +// add records file i's chunks; ids are its coarse chunk ids, new ids are numbered in order. +func (p *placement) add(i int, cl chunkList, ids []int32) { + in := make([]bool, len(cl.hashes)) + for j, h := range cl.hashes { + if _, ok := p.refCoarse[h]; ok { + in[j] = true + p.refBytes[i] += int64(cl.sizes[j]) + } + id := ids[j] + switch { + case int(id) == len(p.owner): + p.owner = append(p.owner, int32(i)) + case p.owner[id] != int32(i): + p.owner[id] = -1 + } + } + p.inRef[i] = in + for _, h := range cl.fine { + if o, ok := p.fineOwner[h]; !ok { + p.fineOwner[h] = int32(i) + } else if o != int32(i) { + p.fineOwner[h] = -1 + } + } + p.fine[i], p.fineSizes[i] = cl.fine, cl.fineSizes +} + +// finish sets each file's shares and class. +func (p *placement) finish(files []File) { + for i := range files { + f := &files[i] + if f.Bytes == 0 { + continue + } + var coarse, fine int64 + for j, id := range f.ids { + if p.owner[id] < 0 || p.inRef[i][j] { + coarse += int64(f.sizes[j]) + } + } + for j, h := range p.fine[i] { + _, ref := p.refFine[h] + if p.fineOwner[h] < 0 || ref { + fine += int64(p.fineSizes[i][j]) + } + } + b := float64(f.Bytes) + f.Shares = &Shares{Ref: float64(p.refBytes[i]) / b, Coarse: float64(coarse) / b, Fine: float64(fine) / b} + f.Class = p.c.Class(*f.Shares) + p.fine[i], p.fineSizes[i], p.inRef[i] = nil, nil, nil + } +} diff --git a/pkg/dedup/classify_test.go b/pkg/dedup/classify_test.go new file mode 100644 index 0000000..a40f9d7 --- /dev/null +++ b/pkg/dedup/classify_test.go @@ -0,0 +1,97 @@ +package dedup + +import ( + "math/rand" + "os" + "path/filepath" + "testing" + + "github.com/ai-computing/onggi/pkg/chunker" +) + +// rearrange returns the 32 KiB blocks of data in another order: the same bytes in runs shorter +// than a coarse chunk, as between two ATOM prefill variants. +func rearrange(data []byte, seed int64) []byte { + const blk = 32 << 10 + idx := rand.New(rand.NewSource(seed)).Perm(len(data) / blk) + out := make([]byte, 0, len(data)) + for _, i := range idx { + out = append(out, data[i*blk:(i+1)*blk]...) + } + return out +} + +func classify(t *testing.T, refs []string, paths ...string) map[string]File { + t.Helper() + files, skipped, err := Collect(paths, 0) + if err != nil { + t.Fatal(err) + } + c := &Classify{Fine: DefaultFine, References: refs, RefShare: DefaultRefShare, RearrangedGap: DefaultRearrangeGap, + ShiftedShare: DefaultShiftedShare} + rep, err := Run(files, skipped, Options{Params: chunker.Default, Workers: 3, Classify: c}, nil) + if err != nil { + t.Fatal(err) + } + out := map[string]File{} + for _, f := range rep.Files { + out[filepath.Base(f.Path)] = f + t.Logf("%-16s %-20s ref %.3f coarse %.3f fine %.3f", filepath.Base(f.Path), f.Class, f.Shares.Ref, f.Shares.Coarse, f.Shares.Fine) + } + return out +} + +// The three placement classes and "new", from bytes alone: no name or format is read. +func TestPlacementClasses(t *testing.T) { + dir := t.TempDir() + w := random(31, 6<<20) + write(t, filepath.Join(dir, "a.bin"), append(random(32, 300_001), w...)) + write(t, filepath.Join(dir, "b.bin"), append(random(33, 777_777), w...)) + write(t, filepath.Join(dir, "c.bin"), rearrange(w, 34)) + write(t, filepath.Join(dir, "d.bin"), random(35, 2<<20)) + got := classify(t, nil, dir) + for name, want := range map[string]string{"a.bin": EmbeddedShifted, "b.bin": EmbeddedShifted, "c.bin": EmbeddedRearranged, "d.bin": New} { + if got[name].Class != want { + t.Errorf("%s: class %q, want %q", name, got[name].Class, want) + } + } + if s := got["c.bin"].Shares; s.Coarse > 0.2 || s.Fine < 0.6 { + t.Errorf("rearranged file: coarse %.3f, fine %.3f", s.Coarse, s.Fine) + } +} + +// A file that holds the reference bytes behind a different header is referenced; the reference +// may be a link, as in a model hub snapshot. +func TestPlacementReferenced(t *testing.T) { + dir, ref := t.TempDir(), t.TempDir() + w := random(41, 4<<20) + write(t, filepath.Join(ref, "blobs", "x"), append([]byte("header of the checkpoint"), w...)) + if err := os.Symlink(filepath.Join(ref, "blobs", "x"), filepath.Join(ref, "model.safetensors")); err != nil { + t.Skip("no symlinks:", err) + } + write(t, filepath.Join(dir, "copy.bin"), append([]byte("another header, 16"), w...)) + write(t, filepath.Join(dir, "other.bin"), random(42, 1<<20)) + got := classify(t, []string{filepath.Join(ref, "model.safetensors")}, dir) + if got["copy.bin"].Class != Referenced || got["other.bin"].Class != New { + t.Fatalf("copy %q, other %q", got["copy.bin"].Class, got["other.bin"].Class) + } +} + +func TestClassRulesAndThresholds(t *testing.T) { + c := &Classify{RefShare: DefaultRefShare, RearrangedGap: DefaultRearrangeGap, ShiftedShare: DefaultShiftedShare} + for _, tc := range []struct { + s Shares + want string + }{ + {Shares{Ref: 0.95, Coarse: 0.95, Fine: 0.95}, Referenced}, + {Shares{Ref: 0.94, Coarse: 0.99, Fine: 0.99}, EmbeddedShifted}, + {Shares{Coarse: 0.37, Fine: 0.86}, EmbeddedRearranged}, // two ATOM prefill variants (#15) + {Shares{Coarse: 0.60, Fine: 0.84}, EmbeddedShifted}, // gap below 0.25 + {Shares{Coarse: 0.10, Fine: 0.30}, New}, + {Shares{Coarse: 0.49, Fine: 0.50}, New}, + } { + if got := c.Class(tc.s); got != tc.want { + t.Errorf("%+v: %q, want %q", tc.s, got, tc.want) + } + } +} diff --git a/pkg/dedup/report.go b/pkg/dedup/report.go index 2b65864..87eaef1 100644 --- a/pkg/dedup/report.go +++ b/pkg/dedup/report.go @@ -21,10 +21,11 @@ import ( // Options controls a report. type Options struct { Params chunker.Params - Workers int // files chunked at once - MinSize int64 // skip smaller files - Pairs int // pairwise shares among this many largest files (0: none) - FixedBlock int // also measure fixed-size blocks of this many bytes (0: no) + Workers int // files chunked at once + MinSize int64 // skip smaller files + Pairs int // pairwise shares among this many largest files (0: none) + FixedBlock int // also measure fixed-size blocks of this many bytes (0: no) + Classify *Classify // also report each file's placement class (#74); nil: no } // File is one input file's result. @@ -34,6 +35,8 @@ type File struct { Bytes int64 `json:"bytes"` NewBytes int64 `json:"new_bytes"` // bytes not seen in any earlier file Chunks int `json:"chunks"` + Class string `json:"class,omitempty"` // placement class (classify.go), with Options.Classify + Shares *Shares `json:"shares,omitempty"` // what the class was decided from ids []int32 sizes []int32 } @@ -67,6 +70,7 @@ type Report struct { Pairs []Pair `json:"pairs,omitempty"` Fixed *FixedResult `json:"fixed_block,omitempty"` Skipped []string `json:"skipped,omitempty"` // symlinks and unreadable entries + Classify *Classify `json:"classify,omitempty"` } // FixedResult is the same measurement with fixed-size blocks, for comparison. @@ -126,7 +130,10 @@ type chunkList struct { hashes [][32]byte sizes []int32 fixed [][32]byte - err error + // fine chunks of every coarse chunk, with Options.Classify + fine []uint64 + fineSizes []int32 + err error } func chunkFile(path string, o Options) (cl chunkList) { @@ -154,6 +161,9 @@ func chunkFile(path string, o Options) (cl chunkList) { } cl.hashes = append(cl.hashes, sha256.Sum256(ch)) cl.sizes = append(cl.sizes, int32(len(ch))) + if o.Classify != nil { + splitFine(o.Classify.Fine, ch, &cl) + } } if fixed != nil { cl.fixed = fixed.done() @@ -200,6 +210,16 @@ func Run(files []File, skipped []string, o Options, progress func(done, total in if o.Workers < 1 { o.Workers = 1 } + var place *placement + if c := o.Classify; c != nil { + if err := c.Fine.Validate(); err != nil { + return nil, err + } + var err error + if place, err = newPlacement(c, o, len(files)); err != nil { + return nil, err + } + } lists := make([]chunkList, len(files)) ready := make([]chan struct{}, len(files)) for i := range ready { @@ -274,6 +294,9 @@ func Run(files []File, skipped []string, o Options, progress func(done, total in fixedUnique += size } } + if place != nil { + place.add(i, cl, f.ids) + } f.Chunks = len(cl.hashes) g.Files++ g.Bytes += f.Bytes @@ -295,6 +318,10 @@ func Run(files []File, skipped []string, o Options, progress func(done, total in if o.FixedBlock > 0 { rep.Fixed = &FixedResult{Block: o.FixedBlock, UniqueBytes: fixedUnique, Ratio: ratio(fixedBytes, fixedUnique)} } + if place != nil { + place.finish(files) + rep.Classify = o.Classify + } rep.Pairs = pairs(files, o.Pairs) rep.Files = files return rep, nil diff --git a/pkg/store/format.go b/pkg/store/format.go index 5a8bc91..5c3632d 100644 --- a/pkg/store/format.go +++ b/pkg/store/format.go @@ -230,15 +230,28 @@ type Config struct { // Sub, when set, makes push split every chunk the store lacks into content-defined // sub-chunks (two-level chunking, #15). Stores created before it have none. Sub *chunker.Params `json:"sub_chunker,omitempty"` - // SubKinds limits sub-chunking to these file kinds (by name, pkg/layout); empty means all. - // Weights that equal their HF source dedup whole at 64 KiB, and finer chunks only add metadata. + // SubKinds limits sub-chunking to these file kinds (by name, pkg/layout). Stores created + // before #74 record it; it is kept for them and excludes SubMinFile. SubKinds []layout.Kind `json:"sub_kinds,omitempty"` + // SubMinFile limits sub-chunking to files of at least this many bytes, whatever their name or + // format (#74). With neither SubKinds nor SubMinFile, every file gets sub-chunks; that is also + // what binaries from before #74 do on a store that records only SubMinFile. + SubMinFile int64 `json:"sub_min_file,omitempty"` + // SubProbe is an experiment (#74); binaries from before it ignore it like SubMinFile. + SubProbe *SubProbe `json:"sub_probe,omitempty"` // Frame, when set, makes push compress consecutive new chunks of a sub-chunked file // together, up to this many raw bytes (#52). Stores created before it have none. Frame int64 `json:"frame,omitempty"` Created time.Time `json:"created"` } +// SubProbe stops sub-chunking a file when the first Chunks new chunks it sub-chunks find less +// than MinShare of their bytes already stored (#74 experiment). +type SubProbe struct { + Chunks int `json:"chunks"` + MinShare float64 `json:"min_share"` +} + // File is one file of an artifact. type File struct { Path string `json:"path"` // slash-separated, relative to the artifact root diff --git a/pkg/store/push.go b/pkg/store/push.go index 4620e11..22853a9 100644 --- a/pkg/store/push.go +++ b/pkg/store/push.go @@ -269,7 +269,7 @@ func (s *Store) Push(ctx context.Context, root string, o PushOptions) (digest.Di if ctx.Err() != nil { continue } - list, nb, err := s.chunkFile(ctx, files[i].abs, s.subChunks(files[i].rel), o.Compress, claim, addComp, chunks) + list, nb, err := s.chunkFile(ctx, files[i].abs, s.subChunks(files[i].rel, files[i].Size), o.Compress, claim, addComp, chunks) if err != nil { fail(fmt.Errorf("%s: %w", files[i].rel, err)) continue @@ -411,6 +411,9 @@ func (s *Store) chunkFile(ctx context.Context, path string, twoLevel bool, mode if twoLevel && s.Config.Frame > 0 { fb = &frameBuilder{limit: int(s.Config.Frame), mode: mode} } + probe := s.Config.SubProbe // experiment (#74) + var probed int + var probeBytes, probeFound int64 for { ch, err := c.Next() if errors.Is(err, io.EOF) { @@ -442,7 +445,11 @@ func (s *Store) chunkFile(ctx context.Context, path string, twoLevel bool, mode err = s.Config.Sub.Split(ch, func(sub []byte) error { sd := digest.Of(sub) subs = append(subs, ChunkRef{Digest: sd, Length: uint32(len(sub))}) - if sd == d || claim(sd) { // sd == d: the whole chunk is one sub-chunk; this push owns d already + fresh := sd == d || claim(sd) + if !fresh { + probeFound += int64(len(sub)) + } + if fresh { // sd == d: the whole chunk is one sub-chunk; this push owns d already if fb != nil { n, err := fb.add(ctx, sub, sd, out) nb += n @@ -459,6 +466,13 @@ func (s *Store) chunkFile(ctx context.Context, path string, twoLevel bool, mode if err != nil { return nil, 0, err } + if probe != nil && probed < probe.Chunks { + probed++ + probeBytes += int64(len(ch)) + if probed == probe.Chunks && float64(probeFound) < probe.MinShare*float64(probeBytes) { + twoLevel = false // the rest of the file goes out as whole chunks + } + } if len(subs) == 1 && subs[0].Digest == d { continue // one sub-chunk equal to the chunk: it is stored as bytes under d } diff --git a/pkg/store/store.go b/pkg/store/store.go index 124161f..b5533e0 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -13,6 +13,7 @@ import ( "errors" "fmt" "io" + "slices" "sync" "time" @@ -49,14 +50,24 @@ var DefaultSub = chunker.Params{Min: 1 << 10, Avg: 4 << 10, Max: 16 << 10} type Create struct { Chunker chunker.Params Sub *chunker.Params // nil: one level of chunks - SubKinds []layout.Kind // file kinds that get sub-chunks; empty: all - PackSize int64 - Frame int64 // compression frames for sub-chunked files (#52); used only with Sub + SubKinds []layout.Kind // file kinds that get sub-chunks (by name, before #74); excludes SubMinFile + // SubMinFile sub-chunks the new chunks of every file of at least this many bytes (#74); used + // only with Sub. With neither SubKinds nor SubMinFile, every file gets sub-chunks. + SubMinFile int64 + SubProbe *SubProbe // experiment (#74): stop sub-chunking a file whose first new chunks find nothing + PackSize int64 + Frame int64 // compression frames for sub-chunked files (#52); used only with Sub } -// DefaultSubKinds are the kinds whose compile variants rearrange the same bytes (#15). +// DefaultSubKinds are the kinds whose compile variants rearrange the same bytes (#15). Stores +// created before #74 recorded them; new stores use DefaultSubMinFile instead. var DefaultSubKinds = []layout.Kind{layout.ATOMPrefill, layout.ATOMDecode, layout.ATOMOther} +// DefaultSubMinFile is the file size from which new stores sub-chunk new chunks (#74). It looks +// at no name or format: weight-bearing files are large, and below it the second level only adds +// metadata. +const DefaultSubMinFile = 1 << 20 + // Open reads the store's config, creating it with p and packSize (one level of chunks) if the // store is empty, and loads the chunk index. func Open(ctx context.Context, b backend.Backend, p chunker.Params, packSize int64) (*Store, error) { @@ -75,6 +86,9 @@ func OpenWith(ctx context.Context, b backend.Backend, c Create) (*Store, error) if err := c.Sub.Validate(); err != nil { return nil, fmt.Errorf("sub-chunks: %v", err) } + if len(c.SubKinds) > 0 && c.SubMinFile > 0 { + return nil, errors.New("sub-chunks: choose file kinds or a minimum file size, not both") + } if c.Sub.Max > p.Min { return nil, fmt.Errorf("sub-chunk max %d exceeds chunk min %d", c.Sub.Max, p.Min) } @@ -84,7 +98,7 @@ func OpenWith(ctx context.Context, b backend.Backend, c Create) (*Store, error) } cfg = Config{Schema: "onggi.store.v1", Chunker: p, PackSize: packSize, Sub: c.Sub, Created: time.Now().UTC()} if c.Sub != nil { - cfg.SubKinds, cfg.Frame = c.SubKinds, c.Frame + cfg.SubKinds, cfg.SubMinFile, cfg.SubProbe, cfg.Frame = c.SubKinds, c.SubMinFile, c.SubProbe, c.Frame } data, _ := json.MarshalIndent(cfg, "", " ") if err := b.Put(ctx, configKey, bytes.NewReader(data), int64(len(data))); err != nil { @@ -209,21 +223,18 @@ func (s *Store) addComposites(cs []composite) { } } -// subChunks reports whether files of this path get sub-chunks. -func (s *Store) subChunks(rel string) bool { - if s.Config.Sub == nil { +// subChunks reports whether the new chunks of a file get sub-chunks. Stores created since #74 +// decide by size alone; stores that recorded file kinds keep deciding by name. +func (s *Store) subChunks(rel string, size int64) bool { + switch { + case s.Config.Sub == nil: return false + case len(s.Config.SubKinds) > 0: + return slices.Contains(s.Config.SubKinds, layout.Of(rel)) + case s.Config.SubMinFile > 0: + return size >= s.Config.SubMinFile } - if len(s.Config.SubKinds) == 0 { - return true - } - k := layout.Of(rel) - for _, x := range s.Config.SubKinds { - if x == k { - return true - } - } - return false + return true } // has reports whether the store can produce chunk d, as bytes or as sub-chunks. diff --git a/pkg/store/sub_test.go b/pkg/store/sub_test.go index b669c28..0f6a477 100644 --- a/pkg/store/sub_test.go +++ b/pkg/store/sub_test.go @@ -4,6 +4,7 @@ import ( "context" "os" "path/filepath" + "strings" "testing" "github.com/ai-computing/onggi/pkg/backend" @@ -262,3 +263,72 @@ func TestCompositesPublishedDuringPush(t *testing.T) { t.Fatalf("second push: %d new bytes, %v", st.NewBytes, err) } } + +// Since #74 a new store decides by size alone: a large file gets sub-chunks whatever its name or +// format, a small one does not. +func TestSubMinFileIgnoresNames(t *testing.T) { + ctx := context.Background() + src, storeDir, out := t.TempDir(), t.TempDir(), t.TempDir() + w := random(21, 4<<20) + write(t, filepath.Join(src, "rank0.engine"), w) + write(t, filepath.Join(src, "renamed.bin"), shuffled(w, 22)) + b, _ := bfs.Open(storeDir, backend.DurabilityNone) + sub := DefaultSub + s, err := OpenWith(ctx, b, Create{Chunker: chunker.Default, Sub: &sub, SubMinFile: 1 << 20, PackSize: 1 << 20}) + if err != nil { + t.Fatal(err) + } + md, st, err := s.Push(ctx, src, PushOptions{}) + if err != nil { + t.Fatal(err) + } + if s.Composites() == 0 || st.NewBytes > int64(len(w))*15/10 { + t.Fatalf("large files were not sub-chunked: %d composites, %d new bytes", s.Composites(), st.NewBytes) + } + if !s.subChunks("model.safetensors", 1<<20) || s.subChunks("prefill.rbln", 1<<20-1) { + t.Fatal("the decision depends on more than the size") + } + if _, err := openStore(t, storeDir).Pull(ctx, md, out, PullOptions{}); err != nil { + t.Fatal(err) + } + sameTree(t, src, out) +} + +// A store created before #74 keeps its file kinds, whatever a later Open asks for; a new store +// records its minimum file size; kinds and a size exclude each other. +func TestSubPolicyIsStoredWithTheStore(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + b, _ := bfs.Open(dir, backend.DurabilityNone) + old := `{"schema":"onggi.store.v1","chunker":{"min":16384,"avg":65536,"max":262144},"pack_size":1048576,` + + `"sub_chunker":{"min":1024,"avg":4096,"max":16384},"sub_kinds":["atom-prefill","atom-decode","atom-other"],` + + `"frame":262144,"created":"2026-10-22T00:00:00Z"}` + if err := b.Put(ctx, configKey, strings.NewReader(old), int64(len(old))); err != nil { + t.Fatal(err) + } + sub := DefaultSub + s, err := OpenWith(ctx, b, Create{Chunker: chunker.Default, Sub: &sub, SubMinFile: DefaultSubMinFile}) + if err != nil { + t.Fatal(err) + } + if s.Config.SubMinFile != 0 || len(s.Config.SubKinds) != 3 { + t.Fatalf("old store config changed: %+v", s.Config) + } + if s.subChunks("rank0.engine", 1<<30) || !s.subChunks("prefill_S256_C16.rbln", 1) { + t.Fatal("old store no longer decides by kind") + } + + b2, _ := bfs.Open(t.TempDir(), backend.DurabilityNone) + if _, err := OpenWith(ctx, b2, Create{Chunker: chunker.Default, Sub: &sub, SubMinFile: 8 << 20}); err != nil { + t.Fatal(err) + } + s2, err := OpenWith(ctx, b2, Create{Chunker: chunker.Default, Sub: &sub, SubKinds: DefaultSubKinds}) + if err != nil || s2.Config.SubMinFile != 8<<20 || len(s2.Config.SubKinds) != 0 { + t.Fatalf("new store lost its minimum file size: %+v %v", s2.Config, err) + } + + b3, _ := bfs.Open(t.TempDir(), backend.DurabilityNone) + if _, err := OpenWith(ctx, b3, Create{Chunker: chunker.Default, Sub: &sub, SubKinds: DefaultSubKinds, SubMinFile: 1}); err == nil { + t.Fatal("kinds and a minimum file size were both accepted") + } +} From 79571a8630aaf8f3ae943b98265512f2117b6843 Mon Sep 17 00:00:00 2001 From: gspark-etri Date: Fri, 30 Oct 2026 18:55:10 +0900 Subject: [PATCH 2/2] Choose the size policy by measurement; drop the probe experiment (#74) On the lab node the size policy (sub-chunk files >= 1 MiB) stores the ATOM set in 16.78 GB (kind policy 16.70 GB) and cuts the RNGD pfbatch -> preset switch from 4.33 to 0.65 MB, at +0.53 GB metadata on a first-time 16 GB checkpoint; the probe lost 10.2 GB on ATOM because the first file of a family is never sub-chunked. The probe code is removed (measured at ec9bc0f). Raw data in docs/data/, summary and placement classes in docs/sub-chunk-policy.md; README and docs updated. Co-Authored-By: Claude Opus 5.5 --- README.ko.md | 3 +- README.md | 7 +- cmd/onggi/main_test.go | 23 + cmd/onggi/store.go | 10 +- docs/artifact-formats.md | 3 +- docs/data/sub-chunk-policy-20261030.json | 3498 ++++++++++++++++++++++ docs/evaluation.md | 4 +- docs/sub-chunk-policy.md | 107 + pkg/store/format.go | 9 - pkg/store/push.go | 16 +- pkg/store/store.go | 6 +- 11 files changed, 3643 insertions(+), 43 deletions(-) create mode 100644 docs/data/sub-chunk-policy-20261030.json create mode 100644 docs/sub-chunk-policy.md diff --git a/README.ko.md b/README.ko.md index 6e88b91..cd7dc07 100644 --- a/README.ko.md +++ b/README.ko.md @@ -70,7 +70,8 @@ export ONGGI_STORE='s3://models/onggi?endpoint=s3.lab.example:8333&insecure=1' A ``` 연구실 노드에서 ATOM Llama-3.1-8B 빌드 묶음 264 GB가 **15.9 GB**로 저장됩니다(push 3분, pull 5분, 원본과 바이트 단위 -동일). 2단계 청크 분할이 새 ATOM 청크를 약 4 KiB 하위 청크로 다시 나눠서, 같은 가중치를 다른 순서로 담은 prefill +동일). 2단계 청크 분할이 1 MiB 이상인 모든 파일의 새 청크를 형식과 무관하게 크기만 보고 약 4 KiB 하위 청크로 다시 나눠서 +([docs/sub-chunk-policy.md](docs/sub-chunk-policy.md)), 같은 가중치를 다른 순서로 담은 prefill 변형끼리 중복을 찾습니다(새 데이터 46.8 GB → 24.9 GB). 여기에 청크 단위 압축이 1.6배를 더합니다. `--compress none`, `--sub 0`으로 각각 끌 수 있습니다. diff --git a/README.md b/README.md index b0cc0d6..7ff2e00 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,9 @@ It walks the paths (symlinks are not followed), sorts files by kind (HF source, decode/prefill `.rbln`) so source checkpoints come first, chunks files in parallel at content-defined boundaries (FastCDC, 16/64/256 KiB by default) and prints, per kind, the logical size, the distinct bytes and the bytes each kind adds on top of the earlier ones. `--fixed-block` repeats the measurement with fixed-size blocks for comparison; -`--pairs N` prints which share of each of the N largest files occurs in each other. +`--pairs N` prints which share of each of the N largest files occurs in each other; `--classify [--reference +CHECKPOINT]` adds each file's weight-placement class (referenced, embedded-rearranged, embedded-shifted, new) from +chunk shares alone. Store and restore an artifact (only chunks the store lacks are uploaded and are compressed per chunk; every chunk is hash-checked on the way back): @@ -78,7 +80,8 @@ export ONGGI_STORE='s3://models/onggi?endpoint=s3.lab.example:8333&insecure=1' A ``` On the lab node, the 264 GB ATOM Llama-3.1-8B build set is stored in **15.9 GB** (push 3 min, pull 5 min, -byte-identical). Two steps get it there. Two-level chunking re-splits new ATOM chunks into ~4 KiB sub-chunks, so prefill +byte-identical). Two steps get it there. Two-level chunking re-splits new chunks of every file of at least 1 MiB into ~4 KiB sub-chunks (by size, not by +format: [docs/sub-chunk-policy.md](docs/sub-chunk-policy.md)), so prefill variants that hold the same weights in a different order dedup (24.9 GB new instead of 46.8 GB). Per-chunk compression then takes 1.6× off. `--compress none` and `--sub 0` turn the two off. New chunks of sub-chunked files are compressed together in frames of up to 255 KiB where that is smaller (#52; `--frame 0` turns it off). diff --git a/cmd/onggi/main_test.go b/cmd/onggi/main_test.go index 3c12ba2..9b9c241 100644 --- a/cmd/onggi/main_test.go +++ b/cmd/onggi/main_test.go @@ -39,3 +39,26 @@ func TestDedupReportRuns(t *testing.T) { t.Fatalf("json output: exit %d\n%s", code, out.String()) } } + +func TestDedupReportClassify(t *testing.T) { + dir, ref := t.TempDir(), t.TempDir() + data := bytes.Repeat([]byte("onggi "), 200_000) + if err := os.WriteFile(filepath.Join(dir, "copy.bin"), data, 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(ref, "model.safetensors"), data, 0o644); err != nil { + t.Fatal(err) + } + var out, errb bytes.Buffer + if code := dedupReport([]string{"-quiet", "-json", "-reference", ref, dir}, &out, &errb); code != 0 || + !strings.Contains(out.String(), `"class": "referenced"`) || !strings.Contains(out.String(), `"rearranged_gap": 0.25`) { + t.Fatalf("classify json: exit %d %s\n%s", code, errb.String(), out.String()) + } + out.Reset() + if code := dedupReport([]string{"-quiet", "-classify", dir}, &out, &errb); code != 0 || !strings.Contains(out.String(), "placement") { + t.Fatalf("classify table: exit %d\n%s", code, out.String()) + } + if code := dedupReport([]string{"-quiet", "-classify", "-fine", "32KiB", dir}, &out, &errb); code != 2 { + t.Fatalf("fine chunks larger than the chunk minimum: exit %d", code) + } +} diff --git a/cmd/onggi/store.go b/cmd/onggi/store.go index 0b5fc58..689c250 100644 --- a/cmd/onggi/store.go +++ b/cmd/onggi/store.go @@ -25,7 +25,7 @@ import ( ) type storeFlags struct { - url, durability, min, avg, max, sub, subKinds, subMinFile, subProbe, packSize, frame *string + url, durability, min, avg, max, sub, subKinds, subMinFile, packSize, frame *string } func addStoreFlags(fs *flag.FlagSet) storeFlags { @@ -38,7 +38,6 @@ func addStoreFlags(fs *flag.FlagSet) storeFlags { sub: fs.String("sub", "4KiB", "average sub-chunk size for two-level chunking, 0 for one level (only when creating a store)"), subKinds: fs.String("sub-kinds", "", "file kinds (pkg/layout, by name) that get sub-chunks, or all; overrides --sub-min-file (only when creating a store; the behaviour before #74)"), subMinFile: fs.String("sub-min-file", "1MiB", "sub-chunk new chunks of files of at least this size, whatever their name or format (only when creating a store)"), - subProbe: fs.String("sub-probe", "", "experiment: N:SHARE stops sub-chunking a file when its first N new chunks find less than SHARE of their sub-chunk bytes stored (only when creating a store)"), packSize: fs.String("pack-size", "64MiB", "pack object size (only when creating a store)"), frame: fs.String("frame", "256KiB", "compress new chunks of sub-chunked files together up to this size, 0 for one by one (only when creating a store)"), } @@ -110,13 +109,6 @@ func (f storeFlags) open(ctx context.Context) (*store.Store, backend.Refs, error c.SubKinds = append(c.SubKinds, layout.Kind(strings.TrimSpace(k))) } } - if *f.subProbe != "" { - var p store.SubProbe - if _, err := fmt.Sscanf(*f.subProbe, "%d:%g", &p.Chunks, &p.MinShare); err != nil || p.Chunks < 1 { - return nil, nil, fmt.Errorf("bad --sub-probe %q: want N:SHARE", *f.subProbe) - } - c.SubProbe = &p - } } s, err := store.OpenWith(ctx, b, c) return s, b, err diff --git a/docs/artifact-formats.md b/docs/artifact-formats.md index 405d067..4ebff77 100644 --- a/docs/artifact-formats.md +++ b/docs/artifact-formats.md @@ -120,7 +120,8 @@ All four prefill variants (56.5 GB), compared with the HF source blobs: - The share of HF source bytes found is 0.0 % at both sizes. The variants hold the same bytes rearranged in runs of a few KiB, which finer chunks recover. Their bytes do not occur -in the HF source at these sizes. The store sub-chunks new ATOM chunks at 4 KiB (two-level chunking). +in the HF source at these sizes. The store sub-chunks new chunks at 4 KiB (two-level chunking; since #74 for every file of at least 1 MiB, see +[sub-chunk-policy.md](sub-chunk-policy.md)). **What remains after 4 KiB sub-chunking** (#9). A decode build shares 87 % of its bytes with a prefill build at 4 KiB, so the "second weight layout" seen at 64 KiB is mostly the same bytes in a different order. diff --git a/docs/data/sub-chunk-policy-20261030.json b/docs/data/sub-chunk-policy-20261030.json new file mode 100644 index 0000000..f85707c --- /dev/null +++ b/docs/data/sub-chunk-policy-20261030.json @@ -0,0 +1,3498 @@ +{ + "schema": "onggi.bench.subchunk-policy.v1", + "issue": 74, + "date": "2026-10-30", + "environment": { + "node": "lab node with the ATOM build set and RNGD artifacts (96 cores, shared; no NPU used)", + "os_kernel": "6.8.0-40-generic", + "go": "go1.24.13 linux/amd64", + "nice": "nice -n 19 ionice -c3, GOMAXPROCS=16", + "store": "local fs backend on the same disk; page cache not dropped", + "binaries": { + "onggi-new": "feat/74-vendor-neutral-chunking ec9bc0f (with the --sub-probe experiment)", + "onggi-main": "main 05c9760", + "onggi-a3": "feat/75-binding-contract 2d84ac3" + }, + "sha256": [ + "9faa746445147cbfe1332fd8efa0c5dfd7332e2a9c8b12604caf7fe8579bd99f onggi-new", + "8dc25cac59af6cf8f84a9f0b872433ca581e14d780feb89f1a44592d35c178b1 onggi-main", + "085f8f30f9ac4fb3231688e45e789168fe59345bee942dafe8e809aed8d40403 onggi-a3", + "" + ] + }, + "inputs": { + "atom-set": "Rebellions ATOM Llama-3.1-8B build set, 19 files, 263.9 GB (read-only)", + "hf-checkpoint": "meta-llama/Llama-3.1-8B-Instruct safetensors snapshot, 16.07 GB (hard links)", + "rngd": "four furiosa-llm 2026.3 artifacts pfbatch, preset (params + graphs), coarse, fine (graphs), pushed in that order", + "switch": "decode_S256_B2.rbln then decode_S256_B8_lm.rbln (two single-file artifacts)", + "switchlm": "decode_S256_B2_lm.rbln then decode_S256_B8_lm.rbln (the B2 -> B8 switch of docs/evaluation.md)" + }, + "policies": { + "kind": "--sub-kinds atom-prefill,atom-decode,atom-other (the default before #74), --workers 16", + "size1m": "--sub-min-file 1MiB (the new default), --workers 16", + "size64m": "--sub-min-file 64MiB, --workers 16", + "probe": "--sub-probe 64:0.1 (experiment, commit ec9bc0f; not in the final PR), --workers 1 (deterministic order)", + "probe16": "--sub-probe 64:0.1, --workers 16", + "main": "binary built from main 05c9760 with its defaults, --workers 16" + }, + "method": "Each workload pushes into a fresh store per policy (n=1; storage results are deterministic for a given push order). store.* = bytes on disk by object kind (packs, pack indexes, composite tables, lists incl. locators, manifests, config); push.stats.pack_bytes of the second artifact = bytes a node fetches for the switch; diff = onggi diff --json between the two refs. wall_s includes process start and the store size scan is excluded.", + "results": [ + { + "workload": "switch", + "policy": "kind", + "step": "b2", + "binary": "onggi-new", + "rc": "0", + "wall_s": "241.72", + "peak_heap": "", + "push": { + "manifest": "sha256:003ace8dcf1d5525ac9afaaa2571f7d642ce88d57c5d6e28f276a50201ada95c", + "ref": "b2", + "stats": { + "files": 1, + "bytes": 14175423892, + "new_bytes": 14093640343, + "packs": 138, + "pack_bytes": 9221814957, + "skipped": null, + "duration_ns": 241619427838, + "files_total": 1, + "bytes_total": 14175423892 + } + }, + "store": { + "packs": 9221814957, + "pack_index": 170369944, + "composite_tables": 116493300, + "lists": 203936286, + "manifests": 465, + "config": 350, + "n_packs": 138, + "n_pack_index": 138, + "n_composite_tables": 12, + "n_lists": 2, + "metadata": 490800345, + "total": 9712615302 + } + }, + { + "workload": "switch", + "policy": "kind", + "step": "b8", + "binary": "onggi-new", + "rc": "0", + "wall_s": "40.78", + "peak_heap": "", + "push": { + "manifest": "sha256:24833c16895ffcee7557a8682155a490956abeca21230c0b926b48f711a50390", + "ref": "b8", + "stats": { + "files": 1, + "bytes": 15228048882, + "new_bytes": 1199677342, + "packs": 11, + "pack_bytes": 693887547, + "skipped": null, + "duration_ns": 40684694878, + "files_total": 1, + "bytes_total": 15228048882 + } + }, + "store": { + "packs": 9915702504, + "pack_index": 184685036, + "composite_tables": 126667740, + "lists": 423595205, + "manifests": 933, + "config": 350, + "n_packs": 149, + "n_pack_index": 149, + "n_composite_tables": 13, + "n_lists": 4, + "metadata": 734949264, + "total": 10650651768 + } + }, + { + "workload": "switch", + "policy": "kind", + "step": "diff", + "diff": { + "files": [ + { + "path": "decode_S256_B8_lm.rbln", + "base": "decode_S256_B2.rbln", + "size": 15228048882, + "same": 0, + "moved": 13959162419, + "new": 1268886463, + "same_blocks": 0, + "changed_runs": 1, + "largest_run": 15228048882 + } + ], + "size": 15228048882, + "same": 0, + "moved": 13959162419, + "new": 1268886463, + "new_stored": 693760073, + "same_blocks": 0, + "changed_runs": 1, + "largest_run": 15228048882 + } + }, + { + "workload": "switch", + "policy": "size1m", + "step": "b2", + "binary": "onggi-new", + "rc": "0", + "wall_s": "231.97", + "peak_heap": "", + "push": { + "manifest": "sha256:9de03564d70423141257eff373c9b45c7480ab6bb041e59d9b13d0c450b3582f", + "ref": "b2", + "stats": { + "files": 1, + "bytes": 14175423892, + "new_bytes": 14093640343, + "packs": 138, + "pack_bytes": 9221814957, + "skipped": null, + "duration_ns": 231906166625, + "files_total": 1, + "bytes_total": 14175423892 + } + }, + "store": { + "packs": 9221814957, + "pack_index": 170369944, + "composite_tables": 116493300, + "lists": 203936286, + "manifests": 465, + "config": 299, + "n_packs": 138, + "n_pack_index": 138, + "n_composite_tables": 12, + "n_lists": 2, + "metadata": 490800294, + "total": 9712615251 + } + }, + { + "workload": "switch", + "policy": "size1m", + "step": "b8", + "binary": "onggi-new", + "rc": "0", + "wall_s": "40.45", + "peak_heap": "", + "push": { + "manifest": "sha256:e418a448b201576f21dcd31bd67188cf4d1912e4ef4d89e6164a636a85b893ad", + "ref": "b8", + "stats": { + "files": 1, + "bytes": 15228048882, + "new_bytes": 1199677342, + "packs": 11, + "pack_bytes": 693887547, + "skipped": null, + "duration_ns": 40359521269, + "files_total": 1, + "bytes_total": 15228048882 + } + }, + "store": { + "packs": 9915702504, + "pack_index": 184685036, + "composite_tables": 126667740, + "lists": 423595205, + "manifests": 933, + "config": 299, + "n_packs": 149, + "n_pack_index": 149, + "n_composite_tables": 13, + "n_lists": 4, + "metadata": 734949213, + "total": 10650651717 + } + }, + { + "workload": "switch", + "policy": "size1m", + "step": "diff", + "diff": { + "files": [ + { + "path": "decode_S256_B8_lm.rbln", + "base": "decode_S256_B2.rbln", + "size": 15228048882, + "same": 0, + "moved": 13959162419, + "new": 1268886463, + "same_blocks": 0, + "changed_runs": 1, + "largest_run": 15228048882 + } + ], + "size": 15228048882, + "same": 0, + "moved": 13959162419, + "new": 1268886463, + "new_stored": 693760073, + "same_blocks": 0, + "changed_runs": 1, + "largest_run": 15228048882 + } + }, + { + "workload": "switch", + "policy": "probe", + "step": "b2", + "binary": "onggi-new", + "rc": "0", + "wall_s": "97.74", + "peak_heap": "", + "push": { + "manifest": "sha256:7335d1d9f9b96ab6037361ce4b9dc2b85331039b36ab35fb903b3cc90b6c11da", + "ref": "b2", + "stats": { + "files": 1, + "bytes": 14175423892, + "new_bytes": 14119815362, + "packs": 138, + "pack_bytes": 9230786655, + "skipped": null, + "duration_ns": 97695027257, + "files_total": 1, + "bytes_total": 14175423892 + } + }, + "store": { + "packs": 9230786655, + "pack_index": 9138784, + "composite_tables": 37236, + "lists": 17058132, + "manifests": 465, + "config": 360, + "n_packs": 138, + "n_pack_index": 138, + "n_composite_tables": 1, + "n_lists": 2, + "metadata": 26234977, + "total": 9257021632 + } + }, + { + "workload": "switch", + "policy": "probe", + "step": "b8", + "binary": "onggi-new", + "rc": "0", + "wall_s": "22.14", + "peak_heap": "", + "push": { + "manifest": "sha256:90712ef18524b27f208a76cabf22d0aa6fe3286212e6da757f186b8acae4064b", + "ref": "b8", + "stats": { + "files": 1, + "bytes": 15228048882, + "new_bytes": 1256914338, + "packs": 11, + "pack_bytes": 696870797, + "skipped": null, + "duration_ns": 22101499920, + "files_total": 1, + "bytes_total": 15228048882 + } + }, + "store": { + "packs": 9927657452, + "pack_index": 9979292, + "composite_tables": 77892, + "lists": 35412175, + "manifests": 933, + "config": 360, + "n_packs": 149, + "n_pack_index": 149, + "n_composite_tables": 2, + "n_lists": 4, + "metadata": 45470652, + "total": 9973128104 + } + }, + { + "workload": "switch", + "policy": "probe", + "step": "diff", + "diff": { + "files": [ + { + "path": "decode_S256_B8_lm.rbln", + "base": "decode_S256_B2.rbln", + "size": 15228048882, + "same": 0, + "moved": 13959145400, + "new": 1268903482, + "same_blocks": 0, + "changed_runs": 1, + "largest_run": 15228048882 + } + ], + "size": 15228048882, + "same": 0, + "moved": 13959145400, + "new": 1268903482, + "new_stored": 696870291, + "same_blocks": 0, + "changed_runs": 1, + "largest_run": 15228048882 + } + }, + { + "workload": "rngd", + "policy": "kind", + "step": "pfbatch", + "binary": "onggi-new", + "rc": "0", + "wall_s": "37.79", + "peak_heap": "", + "push": { + "manifest": "sha256:f9b077ee06e4cb91a77f9cc775cc9bcd50682eb22206ffe7da0ff31ce91b5df1", + "ref": "pfbatch", + "stats": { + "files": 13, + "bytes": 16182992947, + "new_bytes": 16181295422, + "packs": 160, + "pack_bytes": 10734532809, + "skipped": null, + "duration_ns": 37754093211, + "files_total": 13, + "bytes_total": 16182992947 + } + }, + "store": { + "packs": 10734532809, + "pack_index": 10394960, + "composite_tables": 0, + "lists": 19483698, + "manifests": 3595, + "config": 350, + "n_packs": 160, + "n_pack_index": 160, + "n_composite_tables": 0, + "n_lists": 14, + "metadata": 29882603, + "total": 10764415412 + } + }, + { + "workload": "rngd", + "policy": "kind", + "step": "preset", + "binary": "onggi-new", + "rc": "0", + "wall_s": "5.51", + "peak_heap": "", + "push": { + "manifest": "sha256:4ce2c276844387a9c785b10a4650f7146448442c8848dc45523ef172219b0fc8", + "ref": "preset", + "stats": { + "files": 13, + "bytes": 16182266742, + "new_bytes": 20060051, + "packs": 1, + "pack_bytes": 4329095, + "skipped": null, + "duration_ns": 5467560119, + "files_total": 13, + "bytes_total": 16182266742 + } + }, + "store": { + "packs": 10738861904, + "pack_index": 10404748, + "composite_tables": 0, + "lists": 31190642, + "manifests": 7190, + "config": 350, + "n_packs": 161, + "n_pack_index": 161, + "n_composite_tables": 0, + "n_lists": 18, + "metadata": 41602930, + "total": 10780464834 + } + }, + { + "workload": "rngd", + "policy": "kind", + "step": "coarse", + "binary": "onggi-new", + "rc": "0", + "wall_s": "0.42", + "peak_heap": "", + "push": { + "manifest": "sha256:430d2ceb265bdcaf9e62679ce26162516f000b2605fa42f8092f1e500ececfa9", + "ref": "coarse", + "stats": { + "files": 7, + "bytes": 35623630, + "new_bytes": 14650233, + "packs": 1, + "pack_bytes": 2959933, + "skipped": null, + "duration_ns": 391573317, + "files_total": 7, + "bytes_total": 35623630 + } + }, + "store": { + "packs": 10741821837, + "pack_index": 10411848, + "composite_tables": 0, + "lists": 31221186, + "manifests": 8729, + "config": 350, + "n_packs": 162, + "n_pack_index": 162, + "n_composite_tables": 0, + "n_lists": 21, + "metadata": 41642113, + "total": 10783463950 + } + }, + { + "workload": "rngd", + "policy": "kind", + "step": "fine", + "binary": "onggi-new", + "rc": "0", + "wall_s": "0.41", + "peak_heap": "", + "push": { + "manifest": "sha256:8b177db7f5f17eeef60f26dc6ded6f185786da04fa3ba77de081757b8b9be08a", + "ref": "fine", + "stats": { + "files": 7, + "bytes": 39402719, + "new_bytes": 16531138, + "packs": 1, + "pack_bytes": 3504915, + "skipped": null, + "duration_ns": 382856097, + "files_total": 7, + "bytes_total": 39402719 + } + }, + "store": { + "packs": 10745326752, + "pack_index": 10419860, + "composite_tables": 0, + "lists": 31255502, + "manifests": 10268, + "config": 350, + "n_packs": 163, + "n_pack_index": 163, + "n_composite_tables": 0, + "n_lists": 24, + "metadata": 41685980, + "total": 10787012732 + } + }, + { + "workload": "rngd", + "policy": "kind", + "step": "diff", + "diff": { + "files": [ + { + "path": "artifact.json", + "base": "artifact.json", + "size": 7191899, + "same": 0, + "moved": 778410, + "new": 6413489, + "same_blocks": 0, + "changed_runs": 1, + "largest_run": 7191899 + }, + { + "path": "binary_bundle.zip", + "base": "binary_bundle.zip", + "size": 30170684, + "same": 285270, + "moved": 16170898, + "new": 13714516, + "same_blocks": 270336, + "changed_runs": 5, + "largest_run": 27953895 + }, + { + "path": "chat_template.jinja", + "base": "chat_template.jinja", + "size": 4614, + "same": 4614, + "moved": 0, + "new": 0, + "same_blocks": 4614, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "config.json", + "base": "config.json", + "size": 896, + "same": 896, + "moved": 0, + "new": 0, + "same_blocks": 896, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "generation_config.json", + "base": "generation_config.json", + "size": 183, + "same": 183, + "moved": 0, + "new": 0, + "same_blocks": 183, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/cache_access_log.txt", + "base": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/cache_access_log.txt", + "size": 19, + "same": 0, + "moved": 0, + "new": 19, + "same_blocks": 0, + "changed_runs": 1, + "largest_run": 19 + }, + { + "path": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00001-of-00004.safetensors", + "base": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00001-of-00004.safetensors", + "size": 4976698656, + "same": 4976698656, + "moved": 0, + "new": 0, + "same_blocks": 4976698656, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00002-of-00004.safetensors", + "base": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00002-of-00004.safetensors", + "size": 4999802704, + "same": 4999802704, + "moved": 0, + "new": 0, + "same_blocks": 4999802704, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00003-of-00004.safetensors", + "base": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00003-of-00004.safetensors", + "size": 4915916168, + "same": 4915916168, + "moved": 0, + "new": 0, + "same_blocks": 4915916168, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00004-of-00004.safetensors", + "base": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00004-of-00004.safetensors", + "size": 1235247792, + "same": 1235247792, + "moved": 0, + "new": 0, + "same_blocks": 1235247792, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model.safetensors.index.json", + "base": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model.safetensors.index.json", + "size": 22856, + "same": 22856, + "moved": 0, + "new": 0, + "same_blocks": 22856, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "tokenizer.json", + "base": "tokenizer.json", + "size": 17209920, + "same": 17209920, + "moved": 0, + "new": 0, + "same_blocks": 17209920, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "tokenizer_config.json", + "base": "tokenizer_config.json", + "size": 351, + "same": 351, + "moved": 0, + "new": 0, + "same_blocks": 351, + "changed_runs": 0, + "largest_run": 0 + } + ], + "size": 16182266742, + "same": 16145189410, + "moved": 16949308, + "new": 20128024, + "new_stored": 4329095, + "same_blocks": 16145174476, + "changed_runs": 7, + "largest_run": 27953895 + } + }, + { + "workload": "rngd", + "policy": "size1m", + "step": "pfbatch", + "binary": "onggi-new", + "rc": "0", + "wall_s": "112.9", + "peak_heap": "", + "push": { + "manifest": "sha256:28875e8c508edae73e0ef3a2bbb773bda6b91747729122490252322e01e925bf", + "ref": "pfbatch", + "stats": { + "files": 13, + "bytes": 16182992947, + "new_bytes": 16179665261, + "packs": 160, + "pack_bytes": 10740728486, + "skipped": null, + "duration_ns": 112789105650, + "files_total": 13, + "bytes_total": 16182992947 + } + }, + "store": { + "packs": 10740728486, + "pack_index": 196011632, + "composite_tables": 133805280, + "lists": 234314252, + "manifests": 3595, + "config": 298, + "n_packs": 160, + "n_pack_index": 160, + "n_composite_tables": 13, + "n_lists": 14, + "metadata": 564135057, + "total": 11304863543 + } + }, + { + "workload": "rngd", + "policy": "size1m", + "step": "preset", + "binary": "onggi-new", + "rc": "0", + "wall_s": "12.26", + "peak_heap": "", + "push": { + "manifest": "sha256:ec3a70c7ec33f12b2833590639880da44416964dc5d226404ecaa2f4b2384907", + "ref": "preset", + "stats": { + "files": 13, + "bytes": 16182266742, + "new_bytes": 9162268, + "packs": 1, + "pack_bytes": 646801, + "skipped": null, + "duration_ns": 12183606140, + "files_total": 13, + "bytes_total": 16182266742 + } + }, + "store": { + "packs": 10741375287, + "pack_index": 196101892, + "composite_tables": 133941516, + "lists": 460844143, + "manifests": 7190, + "config": 298, + "n_packs": 161, + "n_pack_index": 161, + "n_composite_tables": 14, + "n_lists": 18, + "metadata": 790895039, + "total": 11532270326 + } + }, + { + "workload": "rngd", + "policy": "size1m", + "step": "coarse", + "binary": "onggi-new", + "rc": "0", + "wall_s": "4.65", + "peak_heap": "", + "push": { + "manifest": "sha256:a9a00055de32a0cb38dceab0b6973816001cd47e0c2c0a6332be2070ff90f45d", + "ref": "coarse", + "stats": { + "files": 7, + "bytes": 35623630, + "new_bytes": 5862163, + "packs": 1, + "pack_bytes": 362050, + "skipped": null, + "duration_ns": 4575559085, + "files_total": 7, + "bytes_total": 35623630 + } + }, + "store": { + "packs": 10741737337, + "pack_index": 196159280, + "composite_tables": 134039772, + "lists": 461288895, + "manifests": 8729, + "config": 298, + "n_packs": 162, + "n_pack_index": 162, + "n_composite_tables": 15, + "n_lists": 21, + "metadata": 791496974, + "total": 11533234311 + } + }, + { + "workload": "rngd", + "policy": "size1m", + "step": "fine", + "binary": "onggi-new", + "rc": "0", + "wall_s": "4.83", + "peak_heap": "", + "push": { + "manifest": "sha256:fd5a71b348078b7e12bad4481bfff95625f40ee01fa843aa41c9bc1147b634ab", + "ref": "fine", + "stats": { + "files": 7, + "bytes": 39402719, + "new_bytes": 6707710, + "packs": 1, + "pack_bytes": 591031, + "skipped": null, + "duration_ns": 4738243941, + "files_total": 7, + "bytes_total": 39402719 + } + }, + "store": { + "packs": 10742328368, + "pack_index": 196227028, + "composite_tables": 134152680, + "lists": 461778974, + "manifests": 10268, + "config": 298, + "n_packs": 163, + "n_pack_index": 163, + "n_composite_tables": 16, + "n_lists": 24, + "metadata": 792169248, + "total": 11534497616 + } + }, + { + "workload": "rngd", + "policy": "size1m", + "step": "diff", + "diff": { + "files": [ + { + "path": "artifact.json", + "base": "artifact.json", + "size": 7191899, + "same": 0, + "moved": 967362, + "new": 6224537, + "same_blocks": 0, + "changed_runs": 1, + "largest_run": 7191899 + }, + { + "path": "binary_bundle.zip", + "base": "binary_bundle.zip", + "size": 30170684, + "same": 2159322, + "moved": 24925300, + "new": 3086062, + "same_blocks": 2068480, + "changed_runs": 22, + "largest_run": 27812561 + }, + { + "path": "chat_template.jinja", + "base": "chat_template.jinja", + "size": 4614, + "same": 4614, + "moved": 0, + "new": 0, + "same_blocks": 4614, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "config.json", + "base": "config.json", + "size": 896, + "same": 896, + "moved": 0, + "new": 0, + "same_blocks": 896, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "generation_config.json", + "base": "generation_config.json", + "size": 183, + "same": 183, + "moved": 0, + "new": 0, + "same_blocks": 183, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/cache_access_log.txt", + "base": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/cache_access_log.txt", + "size": 19, + "same": 0, + "moved": 0, + "new": 19, + "same_blocks": 0, + "changed_runs": 1, + "largest_run": 19 + }, + { + "path": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00001-of-00004.safetensors", + "base": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00001-of-00004.safetensors", + "size": 4976698656, + "same": 4976698656, + "moved": 0, + "new": 0, + "same_blocks": 4976698656, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00002-of-00004.safetensors", + "base": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00002-of-00004.safetensors", + "size": 4999802704, + "same": 4999802704, + "moved": 0, + "new": 0, + "same_blocks": 4999802704, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00003-of-00004.safetensors", + "base": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00003-of-00004.safetensors", + "size": 4915916168, + "same": 4915916168, + "moved": 0, + "new": 0, + "same_blocks": 4915916168, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00004-of-00004.safetensors", + "base": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00004-of-00004.safetensors", + "size": 1235247792, + "same": 1235247792, + "moved": 0, + "new": 0, + "same_blocks": 1235247792, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model.safetensors.index.json", + "base": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model.safetensors.index.json", + "size": 22856, + "same": 22856, + "moved": 0, + "new": 0, + "same_blocks": 22856, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "tokenizer.json", + "base": "tokenizer.json", + "size": 17209920, + "same": 17209920, + "moved": 0, + "new": 0, + "same_blocks": 17209920, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "tokenizer_config.json", + "base": "tokenizer_config.json", + "size": 351, + "same": 351, + "moved": 0, + "new": 0, + "same_blocks": 351, + "changed_runs": 0, + "largest_run": 0 + } + ], + "size": 16182266742, + "same": 16147063462, + "moved": 25892662, + "new": 9310618, + "new_stored": 646001, + "same_blocks": 16146972620, + "changed_runs": 24, + "largest_run": 27812561 + } + }, + { + "workload": "rngd", + "policy": "size64m", + "step": "pfbatch", + "binary": "onggi-new", + "rc": "0", + "wall_s": "122.46", + "peak_heap": "", + "push": { + "manifest": "sha256:efb6f0f4dd0144dffdc1f09cc03ac80034fc14b3be66c3a2ae06f2d9c061ffd2", + "ref": "pfbatch", + "stats": { + "files": 13, + "bytes": 16182992947, + "new_bytes": 16181287976, + "packs": 160, + "pack_bytes": 10743365262, + "skipped": null, + "duration_ns": 122336762353, + "files_total": 13, + "bytes_total": 16182992947 + } + }, + "store": { + "packs": 10743365262, + "pack_index": 195477000, + "composite_tables": 133404744, + "lists": 233702487, + "manifests": 3595, + "config": 300, + "n_packs": 160, + "n_pack_index": 160, + "n_composite_tables": 13, + "n_lists": 14, + "metadata": 562588126, + "total": 11305953388 + } + }, + { + "workload": "rngd", + "policy": "size64m", + "step": "preset", + "binary": "onggi-new", + "rc": "0", + "wall_s": "12.21", + "peak_heap": "", + "push": { + "manifest": "sha256:26aed391755823d0ca4db7285b40bdba103cb56b20515aa0c30351c4ab113806", + "ref": "preset", + "stats": { + "files": 13, + "bytes": 16182266742, + "new_bytes": 20060051, + "packs": 1, + "pack_bytes": 4329095, + "skipped": null, + "duration_ns": 12104338895, + "files_total": 13, + "bytes_total": 16182266742 + } + }, + "store": { + "packs": 10747694357, + "pack_index": 195486788, + "composite_tables": 133404744, + "lists": 459628220, + "manifests": 7189, + "config": 300, + "n_packs": 161, + "n_pack_index": 161, + "n_composite_tables": 13, + "n_lists": 18, + "metadata": 788527241, + "total": 11536221598 + } + }, + { + "workload": "rngd", + "policy": "size64m", + "step": "coarse", + "binary": "onggi-new", + "rc": "0", + "wall_s": "4.54", + "peak_heap": "", + "push": { + "manifest": "sha256:027238e680cce9c1ae671ddc66cbe5d7da3454e69d6a34de909e543cff598738", + "ref": "coarse", + "stats": { + "files": 7, + "bytes": 35623630, + "new_bytes": 14650233, + "packs": 1, + "pack_bytes": 2959933, + "skipped": null, + "duration_ns": 4449459089, + "files_total": 7, + "bytes_total": 35623630 + } + }, + "store": { + "packs": 10750654290, + "pack_index": 195493888, + "composite_tables": 133404744, + "lists": 459658668, + "manifests": 8728, + "config": 300, + "n_packs": 162, + "n_pack_index": 162, + "n_composite_tables": 13, + "n_lists": 21, + "metadata": 788566328, + "total": 11539220618 + } + }, + { + "workload": "rngd", + "policy": "size64m", + "step": "fine", + "binary": "onggi-new", + "rc": "0", + "wall_s": "4.23", + "peak_heap": "", + "push": { + "manifest": "sha256:f96fb8c158c986c0c06935a61f622ef3f1a5f61dd53e2ef2e7f05cc2bd10f0aa", + "ref": "fine", + "stats": { + "files": 7, + "bytes": 39402719, + "new_bytes": 16531138, + "packs": 1, + "pack_bytes": 3504915, + "skipped": null, + "duration_ns": 4143447567, + "files_total": 7, + "bytes_total": 39402719 + } + }, + "store": { + "packs": 10754159205, + "pack_index": 195501900, + "composite_tables": 133404744, + "lists": 459692888, + "manifests": 10266, + "config": 300, + "n_packs": 163, + "n_pack_index": 163, + "n_composite_tables": 13, + "n_lists": 24, + "metadata": 788610098, + "total": 11542769303 + } + }, + { + "workload": "rngd", + "policy": "size64m", + "step": "diff", + "diff": { + "files": [ + { + "path": "artifact.json", + "base": "artifact.json", + "size": 7191899, + "same": 0, + "moved": 778410, + "new": 6413489, + "same_blocks": 0, + "changed_runs": 1, + "largest_run": 7191899 + }, + { + "path": "binary_bundle.zip", + "base": "binary_bundle.zip", + "size": 30170684, + "same": 285270, + "moved": 16170898, + "new": 13714516, + "same_blocks": 270336, + "changed_runs": 5, + "largest_run": 27953895 + }, + { + "path": "chat_template.jinja", + "base": "chat_template.jinja", + "size": 4614, + "same": 4614, + "moved": 0, + "new": 0, + "same_blocks": 4614, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "config.json", + "base": "config.json", + "size": 896, + "same": 896, + "moved": 0, + "new": 0, + "same_blocks": 896, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "generation_config.json", + "base": "generation_config.json", + "size": 183, + "same": 183, + "moved": 0, + "new": 0, + "same_blocks": 183, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/cache_access_log.txt", + "base": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/cache_access_log.txt", + "size": 19, + "same": 0, + "moved": 0, + "new": 19, + "same_blocks": 0, + "changed_runs": 1, + "largest_run": 19 + }, + { + "path": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00001-of-00004.safetensors", + "base": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00001-of-00004.safetensors", + "size": 4976698656, + "same": 4976698656, + "moved": 0, + "new": 0, + "same_blocks": 4976698656, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00002-of-00004.safetensors", + "base": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00002-of-00004.safetensors", + "size": 4999802704, + "same": 4999802704, + "moved": 0, + "new": 0, + "same_blocks": 4999802704, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00003-of-00004.safetensors", + "base": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00003-of-00004.safetensors", + "size": 4915916168, + "same": 4915916168, + "moved": 0, + "new": 0, + "same_blocks": 4915916168, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00004-of-00004.safetensors", + "base": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00004-of-00004.safetensors", + "size": 1235247792, + "same": 1235247792, + "moved": 0, + "new": 0, + "same_blocks": 1235247792, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model.safetensors.index.json", + "base": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model.safetensors.index.json", + "size": 22856, + "same": 22856, + "moved": 0, + "new": 0, + "same_blocks": 22856, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "tokenizer.json", + "base": "tokenizer.json", + "size": 17209920, + "same": 17209920, + "moved": 0, + "new": 0, + "same_blocks": 17209920, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "tokenizer_config.json", + "base": "tokenizer_config.json", + "size": 351, + "same": 351, + "moved": 0, + "new": 0, + "same_blocks": 351, + "changed_runs": 0, + "largest_run": 0 + } + ], + "size": 16182266742, + "same": 16145189410, + "moved": 16949308, + "new": 20128024, + "new_stored": 4329095, + "same_blocks": 16145174476, + "changed_runs": 7, + "largest_run": 27953895 + } + }, + { + "workload": "rngd", + "policy": "probe", + "step": "pfbatch", + "binary": "onggi-new", + "rc": "0", + "wall_s": "113.51", + "peak_heap": "", + "push": { + "manifest": "sha256:62b5d09d77cabbc9bac3a90f46ef38f4c04fbeb1f2ce2d7f1f108e7be139c3d2", + "ref": "pfbatch", + "stats": { + "files": 13, + "bytes": 16182992947, + "new_bytes": 16181002765, + "packs": 160, + "pack_bytes": 10734023281, + "skipped": null, + "duration_ns": 113465450858, + "files_total": 13, + "bytes_total": 16182992947 + } + }, + "store": { + "packs": 10734023281, + "pack_index": 10831984, + "composite_tables": 279192, + "lists": 19926240, + "manifests": 3595, + "config": 360, + "n_packs": 160, + "n_pack_index": 160, + "n_composite_tables": 1, + "n_lists": 14, + "metadata": 31041371, + "total": 10765064652 + } + }, + { + "workload": "rngd", + "policy": "probe", + "step": "preset", + "binary": "onggi-new", + "rc": "0", + "wall_s": "16.76", + "peak_heap": "", + "push": { + "manifest": "sha256:34d76e26877ab320773363a134df6b19456cd0fb4da195ebb593a7106585b396", + "ref": "preset", + "stats": { + "files": 13, + "bytes": 16182266742, + "new_bytes": 16084800, + "packs": 1, + "pack_bytes": 2587053, + "skipped": null, + "duration_ns": 16719314956, + "files_total": 13, + "bytes_total": 16182266742 + } + }, + "store": { + "packs": 10736610334, + "pack_index": 10991628, + "composite_tables": 414960, + "lists": 32178670, + "manifests": 7189, + "config": 360, + "n_packs": 161, + "n_pack_index": 161, + "n_composite_tables": 2, + "n_lists": 18, + "metadata": 43592807, + "total": 10780203141 + } + }, + { + "workload": "rngd", + "policy": "probe", + "step": "coarse", + "binary": "onggi-new", + "rc": "0", + "wall_s": "0.55", + "peak_heap": "", + "push": { + "manifest": "sha256:35242e1e693742194f9c1e59f8003ba0745d5d9f1836f5d582f3b9ca2d9c7ec0", + "ref": "coarse", + "stats": { + "files": 7, + "bytes": 35623630, + "new_bytes": 6635647, + "packs": 1, + "pack_bytes": 576012, + "skipped": null, + "duration_ns": 515231310, + "files_total": 7, + "bytes_total": 35623630 + } + }, + "store": { + "packs": 10737186346, + "pack_index": 11057080, + "composite_tables": 513216, + "lists": 32437984, + "manifests": 8728, + "config": 360, + "n_packs": 162, + "n_pack_index": 162, + "n_composite_tables": 3, + "n_lists": 21, + "metadata": 44017368, + "total": 10781203714 + } + }, + { + "workload": "rngd", + "policy": "probe", + "step": "fine", + "binary": "onggi-new", + "rc": "0", + "wall_s": "0.55", + "peak_heap": "", + "push": { + "manifest": "sha256:063e53be881442ed2f4002e2cec5e317ff5bd632d60f510c83973f3c7682fd92", + "ref": "fine", + "stats": { + "files": 7, + "bytes": 39402719, + "new_bytes": 7133500, + "packs": 1, + "pack_bytes": 749868, + "skipped": null, + "duration_ns": 521278253, + "files_total": 7, + "bytes_total": 39402719 + } + }, + "store": { + "packs": 10737936214, + "pack_index": 11129252, + "composite_tables": 626124, + "lists": 32744635, + "manifests": 10266, + "config": 360, + "n_packs": 163, + "n_pack_index": 163, + "n_composite_tables": 4, + "n_lists": 24, + "metadata": 44510637, + "total": 10782446851 + } + }, + { + "workload": "rngd", + "policy": "probe", + "step": "diff", + "diff": { + "files": [ + { + "path": "artifact.json", + "base": "artifact.json", + "size": 7191899, + "same": 0, + "moved": 959157, + "new": 6232742, + "same_blocks": 0, + "changed_runs": 1, + "largest_run": 7191899 + }, + { + "path": "binary_bundle.zip", + "base": "binary_bundle.zip", + "size": 30170684, + "same": 2159322, + "moved": 17779836, + "new": 10231526, + "same_blocks": 2068480, + "changed_runs": 22, + "largest_run": 27812561 + }, + { + "path": "chat_template.jinja", + "base": "chat_template.jinja", + "size": 4614, + "same": 4614, + "moved": 0, + "new": 0, + "same_blocks": 4614, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "config.json", + "base": "config.json", + "size": 896, + "same": 896, + "moved": 0, + "new": 0, + "same_blocks": 896, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "generation_config.json", + "base": "generation_config.json", + "size": 183, + "same": 183, + "moved": 0, + "new": 0, + "same_blocks": 183, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/cache_access_log.txt", + "base": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/cache_access_log.txt", + "size": 19, + "same": 0, + "moved": 0, + "new": 19, + "same_blocks": 0, + "changed_runs": 1, + "largest_run": 19 + }, + { + "path": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00001-of-00004.safetensors", + "base": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00001-of-00004.safetensors", + "size": 4976698656, + "same": 4976698656, + "moved": 0, + "new": 0, + "same_blocks": 4976698656, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00002-of-00004.safetensors", + "base": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00002-of-00004.safetensors", + "size": 4999802704, + "same": 4999802704, + "moved": 0, + "new": 0, + "same_blocks": 4999802704, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00003-of-00004.safetensors", + "base": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00003-of-00004.safetensors", + "size": 4915916168, + "same": 4915916168, + "moved": 0, + "new": 0, + "same_blocks": 4915916168, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00004-of-00004.safetensors", + "base": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00004-of-00004.safetensors", + "size": 1235247792, + "same": 1235247792, + "moved": 0, + "new": 0, + "same_blocks": 1235247792, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model.safetensors.index.json", + "base": "params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model.safetensors.index.json", + "size": 22856, + "same": 22856, + "moved": 0, + "new": 0, + "same_blocks": 22856, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "tokenizer.json", + "base": "tokenizer.json", + "size": 17209920, + "same": 17209920, + "moved": 0, + "new": 0, + "same_blocks": 17209920, + "changed_runs": 0, + "largest_run": 0 + }, + { + "path": "tokenizer_config.json", + "base": "tokenizer_config.json", + "size": 351, + "same": 351, + "moved": 0, + "new": 0, + "same_blocks": 351, + "changed_runs": 0, + "largest_run": 0 + } + ], + "size": 16182266742, + "same": 16147063462, + "moved": 18738993, + "new": 16464287, + "new_stored": 2585701, + "same_blocks": 16146972620, + "changed_runs": 24, + "largest_run": 27812561 + } + }, + { + "workload": "hf", + "policy": "kind", + "step": "hf", + "binary": "onggi-new", + "rc": "0", + "wall_s": "38.33", + "peak_heap": "", + "push": { + "manifest": "sha256:b696d7fda1dc7f451a4003c239f8709c6f69b23e4057dfd32dc86d0817d2eeb9", + "ref": "hf", + "stats": { + "files": 13, + "bytes": 16069779031, + "new_bytes": 16069779031, + "packs": 160, + "pack_bytes": 10704552351, + "skipped": null, + "duration_ns": 38285414339, + "files_total": 13, + "bytes_total": 16069779031 + } + }, + "store": { + "packs": 10704552351, + "pack_index": 10330784, + "composite_tables": 0, + "lists": 19362368, + "manifests": 2668, + "config": 350, + "n_packs": 160, + "n_pack_index": 160, + "n_composite_tables": 0, + "n_lists": 14, + "metadata": 29696170, + "total": 10734248521 + } + }, + { + "workload": "hf", + "policy": "size1m", + "step": "hf", + "binary": "onggi-new", + "rc": "0", + "wall_s": "122.21", + "peak_heap": "", + "push": { + "manifest": "sha256:a4f473c36a59a8824d06196f6c65a901b8409a66f79d59594e0d556bba15152a", + "ref": "hf", + "stats": { + "files": 13, + "bytes": 16069779031, + "new_bytes": 16069771585, + "packs": 160, + "pack_bytes": 10716456241, + "skipped": null, + "duration_ns": 122126559480, + "files_total": 13, + "bytes_total": 16069779031 + } + }, + "store": { + "packs": 10716456241, + "pack_index": 194789040, + "composite_tables": 132960504, + "lists": 232865437, + "manifests": 2668, + "config": 299, + "n_packs": 160, + "n_pack_index": 160, + "n_composite_tables": 13, + "n_lists": 14, + "metadata": 560617948, + "total": 11277074189 + } + }, + { + "workload": "hf", + "policy": "probe", + "step": "hf", + "binary": "onggi-new", + "rc": "0", + "wall_s": "103.98", + "peak_heap": "", + "push": { + "manifest": "sha256:aeb05391b8b37eb8c12fb71b6956684ea9b407cb49fbad21bdd241536d0a118f", + "ref": "hf", + "stats": { + "files": 13, + "bytes": 16069779031, + "new_bytes": 16069779031, + "packs": 160, + "pack_bytes": 10704538330, + "skipped": null, + "duration_ns": 103943932282, + "files_total": 13, + "bytes_total": 16069779031 + } + }, + "store": { + "packs": 10704538330, + "pack_index": 10651416, + "composite_tables": 197904, + "lists": 19677646, + "manifests": 2668, + "config": 360, + "n_packs": 160, + "n_pack_index": 160, + "n_composite_tables": 1, + "n_lists": 14, + "metadata": 30529994, + "total": 10735068324 + } + }, + { + "workload": "atom", + "policy": "kind", + "step": "atom", + "binary": "onggi-new", + "rc": "0", + "wall_s": "110.1", + "peak_heap": "", + "push": { + "manifest": "sha256:ec61891fc60072041fe3c928c31f7ddd4dd8f56b9eac4c4f1405d775a62a6470", + "ref": "atom", + "stats": { + "files": 19, + "bytes": 263893021566, + "new_bytes": 24888631685, + "packs": 233, + "pack_bytes": 15619622182, + "skipped": null, + "duration_ns": 109968266103, + "files_total": 19, + "bytes_total": 263893021566 + } + }, + "store": { + "packs": 15619622182, + "pack_index": 265827012, + "composite_tables": 364977180, + "lists": 453697243, + "manifests": 3980, + "config": 350, + "n_packs": 233, + "n_pack_index": 233, + "n_composite_tables": 33, + "n_lists": 20, + "metadata": 1084505765, + "total": 16704127947 + } + }, + { + "workload": "atom", + "policy": "kind", + "step": "hf", + "binary": "onggi-new", + "rc": "0", + "wall_s": "37.81", + "peak_heap": "", + "push": { + "manifest": "sha256:9d8af12e82e61faf83b55faeb637ce7f83e89f7149dc5de9c12d9259f0a319b2", + "ref": "hf", + "stats": { + "files": 13, + "bytes": 16069779031, + "new_bytes": 16069779031, + "packs": 160, + "pack_bytes": 10704552351, + "skipped": null, + "duration_ns": 37698704296, + "files_total": 13, + "bytes_total": 16069779031 + } + }, + "store": { + "packs": 26324174533, + "pack_index": 276157796, + "composite_tables": 364977180, + "lists": 473059611, + "manifests": 6648, + "config": 350, + "n_packs": 393, + "n_pack_index": 393, + "n_composite_tables": 33, + "n_lists": 34, + "metadata": 1114201585, + "total": 27438376118 + } + }, + { + "workload": "atom", + "policy": "size1m", + "step": "atom", + "binary": "onggi-new", + "rc": "0", + "wall_s": "120.31", + "peak_heap": "", + "push": { + "manifest": "sha256:9f6e0c1336f8282b8a7f8e87ce1ad5efb0ab3ae6c47d464a623e5e62d3fb6f5a", + "ref": "atom", + "stats": { + "files": 19, + "bytes": 263893021566, + "new_bytes": 24888602368, + "packs": 233, + "pack_bytes": 15623049420, + "skipped": null, + "duration_ns": 120143476413, + "files_total": 19, + "bytes_total": 263893021566 + } + }, + "store": { + "packs": 15623049420, + "pack_index": 293739828, + "composite_tables": 382368372, + "lists": 481623294, + "manifests": 3980, + "config": 299, + "n_packs": 233, + "n_pack_index": 233, + "n_composite_tables": 35, + "n_lists": 20, + "metadata": 1157735773, + "total": 16780785193 + } + }, + { + "workload": "atom", + "policy": "size1m", + "step": "hf", + "binary": "onggi-new", + "rc": "0", + "wall_s": "117.52", + "peak_heap": "", + "push": { + "manifest": "sha256:8d54819d9bfb9ecf8398f6eabf80eefd6416ef1ccde37b27300e6ccc75a39bce", + "ref": "hf", + "stats": { + "files": 13, + "bytes": 16069779031, + "new_bytes": 16069771585, + "packs": 160, + "pack_bytes": 10716456241, + "skipped": null, + "duration_ns": 117334059517, + "files_total": 13, + "bytes_total": 16069779031 + } + }, + "store": { + "packs": 26339505661, + "pack_index": 488528868, + "composite_tables": 515328876, + "lists": 714488731, + "manifests": 6648, + "config": 299, + "n_packs": 393, + "n_pack_index": 393, + "n_composite_tables": 48, + "n_lists": 34, + "metadata": 1718353422, + "total": 28057859083 + } + }, + { + "workload": "atom", + "policy": "probe", + "step": "atom", + "binary": "onggi-new", + "rc": "0", + "wall_s": "598.93", + "peak_heap": "", + "push": { + "manifest": "sha256:107a363c578c8bb4f2a938427a8085df7115a6408329382c34c997c5f4e1040a", + "ref": "atom", + "stats": { + "files": 19, + "bytes": 263893021566, + "new_bytes": 41507734230, + "packs": 391, + "pack_bytes": 26238851934, + "skipped": null, + "duration_ns": 598801644495, + "files_total": 19, + "bytes_total": 263893021566 + } + }, + "store": { + "packs": 26238851934, + "pack_index": 181873444, + "composite_tables": 155503836, + "lists": 347736388, + "manifests": 3980, + "config": 360, + "n_packs": 391, + "n_pack_index": 391, + "n_composite_tables": 15, + "n_lists": 20, + "metadata": 685118008, + "total": 26923969942 + } + }, + { + "workload": "atom", + "policy": "probe", + "step": "hf", + "binary": "onggi-new", + "rc": "0", + "wall_s": "90.26", + "peak_heap": "", + "push": { + "manifest": "sha256:d3409fb339542d9bf1fd2f457d4cae747afcf9043fd45337bcdb3b3f1ba579a7", + "ref": "hf", + "stats": { + "files": 13, + "bytes": 16069779031, + "new_bytes": 16069779031, + "packs": 160, + "pack_bytes": 10704538330, + "skipped": null, + "duration_ns": 90148212541, + "files_total": 13, + "bytes_total": 16069779031 + } + }, + "store": { + "packs": 36943390264, + "pack_index": 192524860, + "composite_tables": 155701740, + "lists": 367414034, + "manifests": 6648, + "config": 360, + "n_packs": 551, + "n_pack_index": 551, + "n_composite_tables": 16, + "n_lists": 34, + "metadata": 715647642, + "total": 37659037906 + } + }, + { + "workload": "atom", + "policy": "main", + "step": "atom", + "binary": "onggi-main", + "rc": "0", + "wall_s": "112.83", + "peak_heap": "", + "push": { + "manifest": "sha256:58b87b04ea3066623ef70d49644e609cd77f62914148e88a4c21a75f1f40ee6b", + "ref": "atom", + "stats": { + "files": 19, + "bytes": 263893021566, + "new_bytes": 24888631685, + "packs": 233, + "pack_bytes": 15619344317, + "skipped": null, + "duration_ns": 112671085994, + "files_total": 19, + "bytes_total": 263893021566 + } + }, + "store": { + "packs": 15619344317, + "pack_index": 265826332, + "composite_tables": 364977180, + "lists": 453730843, + "manifests": 3980, + "config": 350, + "n_packs": 233, + "n_pack_index": 233, + "n_composite_tables": 33, + "n_lists": 20, + "metadata": 1084538685, + "total": 16703883002 + } + }, + { + "workload": "switchlm", + "policy": "kind", + "step": "la", + "wall_s": 268.42, + "push": { + "manifest": "sha256:70ff57bf0346d99b4929bee7ba91b9214e3dd166f244e081bdb3db285fbe22ac", + "ref": "la", + "stats": { + "files": 1, + "bytes": 15238278850, + "new_bytes": 15160648815, + "packs": 148, + "pack_bytes": 9912388355, + "skipped": null, + "duration_ns": 268322403851, + "files_total": 1, + "bytes_total": 15238278850 + } + }, + "store": { + "packs": 9912388355, + "pack_index": 183259344, + "composite_tables": 125311716, + "lists": 219358109, + "manifests": 468, + "config": 350, + "n_packs": 148, + "n_pack_index": 148, + "n_composite_tables": 12, + "n_lists": 2, + "metadata": 527929987, + "total": 10440318342 + } + }, + { + "workload": "switchlm", + "policy": "kind", + "step": "lb", + "wall_s": 23.48, + "push": { + "manifest": "sha256:4104405741df87245b5e66716ea5c29411f520c5580199cfd7f3e72ec136b97c", + "ref": "lb", + "stats": { + "files": 1, + "bytes": 15228048882, + "new_bytes": 149094548, + "packs": 1, + "pack_bytes": 3567290, + "skipped": null, + "duration_ns": 23376092096, + "files_total": 1, + "bytes_total": 15228048882 + } + }, + "store": { + "packs": 9915955645, + "pack_index": 184840212, + "composite_tables": 126793452, + "lists": 439017764, + "manifests": 936, + "config": 350, + "n_packs": 149, + "n_pack_index": 149, + "n_composite_tables": 13, + "n_lists": 4, + "metadata": 750652714, + "total": 10666608359 + } + }, + { + "workload": "switchlm", + "policy": "kind", + "step": "diff", + "diff": { + "files": [ + { + "path": "decode_S256_B8_lm.rbln", + "base": "decode_S256_B2_lm.rbln", + "size": 15228048882, + "same": 15009857021, + "moved": 0, + "new": 218191861, + "same_blocks": 15009845248, + "changed_runs": 3, + "largest_run": 218137453 + } + ], + "size": 15228048882, + "same": 15009857021, + "moved": 0, + "new": 218191861, + "new_stored": 3553089, + "same_blocks": 15009845248, + "changed_runs": 3, + "largest_run": 218137453 + } + }, + { + "workload": "switchlm", + "policy": "size1m", + "step": "la", + "wall_s": 257.69, + "push": { + "manifest": "sha256:72e487e8155dfd79d0a8a30c4b3ac9a17c691f9065232e0a647510c66e6bf011", + "ref": "la", + "stats": { + "files": 1, + "bytes": 15238278850, + "new_bytes": 15160648815, + "packs": 148, + "pack_bytes": 9912388355, + "skipped": null, + "duration_ns": 257586123171, + "files_total": 1, + "bytes_total": 15238278850 + } + }, + "store": { + "packs": 9912388355, + "pack_index": 183259344, + "composite_tables": 125311728, + "lists": 219358109, + "manifests": 468, + "config": 299, + "n_packs": 148, + "n_pack_index": 148, + "n_composite_tables": 13, + "n_lists": 2, + "metadata": 527929948, + "total": 10440318303 + } + }, + { + "workload": "switchlm", + "policy": "size1m", + "step": "lb", + "wall_s": 24.7, + "push": { + "manifest": "sha256:4c3823e6d0e0379fe30074912cb219759b56e6d3d52fa6dc59b05a22693ff725", + "ref": "lb", + "stats": { + "files": 1, + "bytes": 15228048882, + "new_bytes": 149094548, + "packs": 1, + "pack_bytes": 3567290, + "skipped": null, + "duration_ns": 24588950354, + "files_total": 1, + "bytes_total": 15228048882 + } + }, + "store": { + "packs": 9915955645, + "pack_index": 184840212, + "composite_tables": 126793464, + "lists": 439017764, + "manifests": 936, + "config": 299, + "n_packs": 149, + "n_pack_index": 149, + "n_composite_tables": 14, + "n_lists": 4, + "metadata": 750652675, + "total": 10666608320 + } + }, + { + "workload": "switchlm", + "policy": "size1m", + "step": "diff", + "diff": { + "files": [ + { + "path": "decode_S256_B8_lm.rbln", + "base": "decode_S256_B2_lm.rbln", + "size": 15228048882, + "same": 15009857021, + "moved": 0, + "new": 218191861, + "same_blocks": 15009845248, + "changed_runs": 3, + "largest_run": 218137453 + } + ], + "size": 15228048882, + "same": 15009857021, + "moved": 0, + "new": 218191861, + "new_stored": 3553089, + "same_blocks": 15009845248, + "changed_runs": 3, + "largest_run": 218137453 + } + }, + { + "workload": "switchlm", + "policy": "probe", + "step": "la", + "wall_s": 104.66, + "push": { + "manifest": "sha256:3f99e3aaf828b92f0bae384d6b131c261a1a945c34fd138b857399d3f54dd8bb", + "ref": "la", + "stats": { + "files": 1, + "bytes": 15238278850, + "new_bytes": 15189950012, + "packs": 148, + "pack_bytes": 9922051518, + "skipped": null, + "duration_ns": 104612347432, + "files_total": 1, + "bytes_total": 15238278850 + } + }, + "store": { + "packs": 9922051518, + "pack_index": 9827808, + "composite_tables": 37344, + "lists": 18337924, + "manifests": 468, + "config": 360, + "n_packs": 148, + "n_pack_index": 148, + "n_composite_tables": 1, + "n_lists": 2, + "metadata": 28203904, + "total": 9950255422 + } + }, + { + "workload": "switchlm", + "policy": "probe", + "step": "lb", + "wall_s": 18.79, + "push": { + "manifest": "sha256:2620ae64bdaa289a077ee5ad882f0b855a77d9376c6775d99ba1fc75bed33585", + "ref": "lb", + "stats": { + "files": 1, + "bytes": 15228048882, + "new_bytes": 149161111, + "packs": 1, + "pack_bytes": 3581677, + "skipped": null, + "duration_ns": 18748141804, + "files_total": 1, + "bytes_total": 15228048882 + } + }, + "store": { + "packs": 9925633195, + "pack_index": 11409404, + "composite_tables": 1519080, + "lists": 38408151, + "manifests": 936, + "config": 360, + "n_packs": 149, + "n_pack_index": 149, + "n_composite_tables": 2, + "n_lists": 4, + "metadata": 51337931, + "total": 9976971126 + } + }, + { + "workload": "switchlm", + "policy": "probe", + "step": "diff", + "diff": { + "files": [ + { + "path": "decode_S256_B8_lm.rbln", + "base": "decode_S256_B2_lm.rbln", + "size": 15228048882, + "same": 15009790458, + "moved": 0, + "new": 218258424, + "same_blocks": 15009775616, + "changed_runs": 3, + "largest_run": 218204016 + } + ], + "size": 15228048882, + "same": 15009790458, + "moved": 0, + "new": 218258424, + "new_stored": 3567740, + "same_blocks": 15009775616, + "changed_runs": 3, + "largest_run": 218204016 + } + }, + { + "workload": "atom", + "policy": "probe16", + "step": "atom", + "rc": 0, + "wall_s": 172.19, + "push": { + "manifest": "sha256:f97b82dfddf8d294082b8324af71f8f838a8fbf9ce6164ac098f7d9fa3246e8f", + "ref": "atom", + "stats": { + "files": 19, + "bytes": 263893021566, + "new_bytes": 41878923241, + "packs": 393, + "pack_bytes": 26351656485, + "skipped": null, + "duration_ns": 172044477282, + "files_total": 19, + "bytes_total": 263893021566 + } + }, + "store": { + "packs": 26351656485, + "pack_index": 174913820, + "composite_tables": 147090984, + "lists": 329591360, + "manifests": 3980, + "config": 360, + "n_packs": 393, + "n_pack_index": 393, + "n_composite_tables": 14, + "n_lists": 20, + "metadata": 651600504, + "total": 27003256989 + } + }, + { + "workload": "atom", + "policy": "probe16", + "step": "hf", + "rc": 0, + "wall_s": 31.77, + "push": { + "manifest": "sha256:ef9a02e91e1f94177494f2ed14709799628893ddc9994a1addc8008286d18b58", + "ref": "hf", + "stats": { + "files": 13, + "bytes": 16069779031, + "new_bytes": 16069779031, + "packs": 160, + "pack_bytes": 10704538330, + "skipped": null, + "duration_ns": 31680785672, + "files_total": 13, + "bytes_total": 16069779031 + } + }, + "store": { + "packs": 37056194815, + "pack_index": 185565340, + "composite_tables": 147288888, + "lists": 349269006, + "manifests": 6648, + "config": 360, + "n_packs": 553, + "n_pack_index": 553, + "n_composite_tables": 15, + "n_lists": 34, + "metadata": 682130242, + "total": 37738325057 + } + }, + { + "workload": "rngd", + "policy": "probe16", + "step": "pfbatch", + "rc": 0, + "wall_s": 36.84, + "push": { + "manifest": "sha256:dbfd412c98426362fcbdb92067206b968f130bb4b3625af9258c4ac85be319b5", + "ref": "pfbatch", + "stats": { + "files": 13, + "bytes": 16182992947, + "new_bytes": 16181002765, + "packs": 160, + "pack_bytes": 10734023281, + "skipped": null, + "duration_ns": 36790793677, + "files_total": 13, + "bytes_total": 16182992947 + } + }, + "store": { + "packs": 10734023281, + "pack_index": 10842560, + "composite_tables": 279192, + "lists": 19926240, + "manifests": 3595, + "config": 360, + "n_packs": 160, + "n_pack_index": 160, + "n_composite_tables": 1, + "n_lists": 14, + "metadata": 31051947, + "total": 10765075228 + } + }, + { + "workload": "rngd", + "policy": "probe16", + "step": "preset", + "rc": 0, + "wall_s": 5.56, + "push": { + "manifest": "sha256:d14f57ef0f9d250b338043c55b9f777367e6cdbd7cf50d1cc15d5ee06ecb80fd", + "ref": "preset", + "stats": { + "files": 13, + "bytes": 16182266742, + "new_bytes": 16084800, + "packs": 1, + "pack_bytes": 2587053, + "skipped": null, + "duration_ns": 5508510146, + "files_total": 13, + "bytes_total": 16182266742 + } + }, + "store": { + "packs": 10736610334, + "pack_index": 11002204, + "composite_tables": 414960, + "lists": 32178670, + "manifests": 7190, + "config": 360, + "n_packs": 161, + "n_pack_index": 161, + "n_composite_tables": 2, + "n_lists": 18, + "metadata": 43603384, + "total": 10780213718 + } + }, + { + "workload": "rngd", + "policy": "probe16", + "step": "coarse", + "rc": 0, + "wall_s": 0.41, + "push": { + "manifest": "sha256:bf3c4d987bac1f9dec319c31cbad90d1db777b33a16bcca48425fa716bb40f12", + "ref": "coarse", + "stats": { + "files": 7, + "bytes": 35623630, + "new_bytes": 6635647, + "packs": 1, + "pack_bytes": 576012, + "skipped": null, + "duration_ns": 387361432, + "files_total": 7, + "bytes_total": 35623630 + } + }, + "store": { + "packs": 10737186346, + "pack_index": 11067656, + "composite_tables": 513216, + "lists": 32438048, + "manifests": 8729, + "config": 360, + "n_packs": 162, + "n_pack_index": 162, + "n_composite_tables": 3, + "n_lists": 21, + "metadata": 44028009, + "total": 10781214355 + } + }, + { + "workload": "rngd", + "policy": "probe16", + "step": "fine", + "rc": 0, + "wall_s": 0.43, + "push": { + "manifest": "sha256:6766426350b8d7fdc03aaecddb0fe77480b6e97a054606ddc081e42b2a34a970", + "ref": "fine", + "stats": { + "files": 7, + "bytes": 39402719, + "new_bytes": 7133500, + "packs": 1, + "pack_bytes": 749868, + "skipped": null, + "duration_ns": 408093230, + "files_total": 7, + "bytes_total": 39402719 + } + }, + "store": { + "packs": 10737936214, + "pack_index": 11139828, + "composite_tables": 626124, + "lists": 32744763, + "manifests": 10268, + "config": 360, + "n_packs": 163, + "n_pack_index": 163, + "n_composite_tables": 4, + "n_lists": 24, + "metadata": 44521343, + "total": 10782457557 + } + } + ], + "placement_reports": { + "atom_with_hf_reference": { + "chunk_params": { + "Min": 16384, + "Avg": 65536, + "Max": 262144 + }, + "bytes": 263893021566, + "unique_bytes": 46775134846, + "ratio": 5.6417372699154695, + "chunks": 3517972, + "unique_chunks": 610743, + "groups": [ + { + "kind": "atom-decode", + "files": 13, + "bytes": 191152006863, + "new_bytes": 16768736171, + "unique_bytes": 16768736171 + }, + { + "kind": "atom-prefill", + "files": 4, + "bytes": 56521901037, + "new_bytes": 27791852124, + "unique_bytes": 40672984441 + }, + { + "kind": "atom-other", + "files": 1, + "bytes": 14117765368, + "new_bytes": 113198253, + "unique_bytes": 14072331549 + }, + { + "kind": "other", + "files": 1, + "bytes": 2101348298, + "new_bytes": 2101348298, + "unique_bytes": 2101348298 + } + ], + "files": [ + { + "path": "atom-set/decode_S256_B1.rbln", + "kind": "atom-decode", + "bytes": 14118071282, + "new_bytes": 14069173778, + "chunks": 187819, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 0.9889304589218747, + "fine": 0.9949680963793848 + } + }, + { + "path": "atom-set/decode_S256_B1_lm.rbln", + "kind": "atom-decode", + "bytes": 15177735226, + "new_bytes": 1159986479, + "chunks": 201919, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 0.9891072675509065, + "fine": 0.990313885582339 + } + }, + { + "path": "atom-set/decode_S256_B2.rbln", + "kind": "atom-decode", + "bytes": 14175423892, + "new_bytes": 160909157, + "chunks": 189136, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 0.9855955357980345, + "fine": 0.9909991199577455 + } + }, + { + "path": "atom-set/decode_S256_B2_lm.rbln", + "kind": "atom-decode", + "bytes": 15238278850, + "new_bytes": 168150225, + "chunks": 203315, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 0.9876847443305581, + "fine": 0.9920115155918675 + } + }, + { + "path": "atom-set/decode_S256_B4.rbln", + "kind": "atom-decode", + "bytes": 14152245933, + "new_bytes": 175376839, + "chunks": 188691, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 0.9900808900817171, + "fine": 0.9936993127859602 + } + }, + { + "path": "atom-set/decode_S256_B4_lm.rbln", + "kind": "atom-decode", + "bytes": 15214978703, + "new_bytes": 178626793, + "chunks": 202962, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 0.9901050017263373, + "fine": 0.9940192951448523 + } + }, + { + "path": "atom-set/decode_S256_B8_lm.rbln", + "kind": "atom-decode", + "bytes": 15228048882, + "new_bytes": 206308937, + "chunks": 202585, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 0.9856647373086624, + "fine": 0.9856717135142697 + } + }, + { + "path": "atom-set/decode_S512_B1.rbln", + "kind": "atom-decode", + "bytes": 14002676629, + "new_bytes": 43501172, + "chunks": 187039, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 0.9977549025923449, + "fine": 0.9995112341603444 + } + }, + { + "path": "atom-set/decode_S512_B1_lm.rbln", + "kind": "atom-decode", + "bytes": 15061413691, + "new_bytes": 39556930, + "chunks": 201192, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 0.997373624361461, + "fine": 0.9990573825079592 + } + }, + { + "path": "atom-set/decode_S512_B2.rbln", + "kind": "atom-decode", + "bytes": 14176417264, + "new_bytes": 162117526, + "chunks": 189127, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 0.9862878851278146, + "fine": 0.9906461871479518 + } + }, + { + "path": "atom-set/decode_S512_B2_lm.rbln", + "kind": "atom-decode", + "bytes": 15239296537, + "new_bytes": 149647866, + "chunks": 203227, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 0.9871159447207232, + "fine": 0.9896259774448144 + } + }, + { + "path": "atom-set/decode_S512_B4.rbln", + "kind": "atom-decode", + "bytes": 14152348510, + "new_bytes": 134162298, + "chunks": 188798, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 0.9921266131256402, + "fine": 0.9950553444221252 + } + }, + { + "path": "atom-set/decode_S512_B4_lm.rbln", + "kind": "atom-decode", + "bytes": 15215071464, + "new_bytes": 121218171, + "chunks": 203040, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 0.991596341344664, + "fine": 0.9956085234198148 + } + }, + { + "path": "atom-set/prefill_S256_C128.rbln", + "kind": "atom-prefill", + "bytes": 14017135688, + "new_bytes": 9828954890, + "chunks": 187201, + "class": "embedded-rearranged", + "shares": { + "ref": 0, + "coarse": 0.44525966766114106, + "fine": 0.8965216613233087 + } + }, + { + "path": "atom-set/prefill_S256_C16.rbln", + "kind": "atom-prefill", + "bytes": 14154808354, + "new_bytes": 196359084, + "chunks": 188162, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 0.985982481003077, + "fine": 0.9861830471943669 + } + }, + { + "path": "atom-set/prefill_S256_C256.rbln", + "kind": "atom-prefill", + "bytes": 14113353000, + "new_bytes": 9125719032, + "chunks": 188209, + "class": "embedded-rearranged", + "shares": { + "ref": 0, + "coarse": 0.4155616143803673, + "fine": 0.8677158882797023 + } + }, + { + "path": "atom-set/prefill_S512_C512.rbln", + "kind": "atom-prefill", + "bytes": 14236603995, + "new_bytes": 8640819118, + "chunks": 189687, + "class": "embedded-rearranged", + "shares": { + "ref": 0, + "coarse": 0.3930561585449227, + "fine": 0.8924220907220648 + } + }, + { + "path": "atom-set/llama8b_S256_fp16.rbln", + "kind": "atom-other", + "bytes": 14117765368, + "new_bytes": 113198253, + "chunks": 187830, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 0.9887636557298535, + "fine": 0.9938685088791596 + } + }, + { + "path": "atom-set/tables.npz", + "kind": "other", + "bytes": 2101348298, + "new_bytes": 2101348298, + "chunks": 28033, + "class": "new", + "shares": { + "ref": 0, + "coarse": 0, + "fine": 9.517698717073888e-10 + } + } + ], + "classify": { + "fine_params": { + "Min": 1024, + "Avg": 4096, + "Max": 16384 + }, + "references": [ + "hf-checkpoint" + ], + "ref_share": 0.95, + "rearranged_gap": 0.25, + "shifted_share": 0.5, + "reference_files": 13, + "reference_bytes": 16069779031 + } + }, + "rngd_with_hf_reference": { + "chunk_params": { + "Min": 16384, + "Avg": 65536, + "Max": 262144 + }, + "bytes": 32440286038, + "unique_bytes": 16232536844, + "ratio": 1.9984729651170228, + "chunks": 433776, + "unique_chunks": 216931, + "groups": [ + { + "kind": "rngd-params", + "files": 8, + "bytes": 32255330640, + "new_bytes": 16127665320, + "unique_bytes": 16127665320 + }, + { + "kind": "rngd-bundle", + "files": 8, + "bytes": 116045792, + "new_bytes": 87632666, + "unique_bytes": 87632666 + }, + { + "kind": "other", + "files": 24, + "bytes": 68909606, + "new_bytes": 17238858, + "unique_bytes": 17238858 + } + ], + "files": [ + { + "path": "rngd/pfbatch/params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00001-of-00004.safetensors", + "kind": "rngd-params", + "bytes": 4976698656, + "new_bytes": 4976698656, + "chunks": 66642, + "class": "referenced", + "shares": { + "ref": 0.9999849725681281, + "coarse": 1, + "fine": 1 + } + }, + { + "path": "rngd/pfbatch/params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00002-of-00004.safetensors", + "kind": "rngd-params", + "bytes": 4999802704, + "new_bytes": 4999802704, + "chunks": 66801, + "class": "referenced", + "shares": { + "ref": 0.999986890482709, + "coarse": 1, + "fine": 1 + } + }, + { + "path": "rngd/pfbatch/params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00003-of-00004.safetensors", + "kind": "rngd-params", + "bytes": 4915916168, + "new_bytes": 4915916168, + "chunks": 65892, + "class": "referenced", + "shares": { + "ref": 0.9999859118834347, + "coarse": 1, + "fine": 1 + } + }, + { + "path": "rngd/pfbatch/params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00004-of-00004.safetensors", + "kind": "rngd-params", + "bytes": 1235247792, + "new_bytes": 1235247792, + "chunks": 16430, + "class": "embedded-shifted", + "shares": { + "ref": 0.9454986736782607, + "coarse": 1, + "fine": 1 + } + }, + { + "path": "rngd/preset/params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00001-of-00004.safetensors", + "kind": "rngd-params", + "bytes": 4976698656, + "new_bytes": 0, + "chunks": 66642, + "class": "referenced", + "shares": { + "ref": 0.9999849725681281, + "coarse": 1, + "fine": 1 + } + }, + { + "path": "rngd/preset/params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00002-of-00004.safetensors", + "kind": "rngd-params", + "bytes": 4999802704, + "new_bytes": 0, + "chunks": 66801, + "class": "referenced", + "shares": { + "ref": 0.999986890482709, + "coarse": 1, + "fine": 1 + } + }, + { + "path": "rngd/preset/params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00003-of-00004.safetensors", + "kind": "rngd-params", + "bytes": 4915916168, + "new_bytes": 0, + "chunks": 65892, + "class": "referenced", + "shares": { + "ref": 0.9999859118834347, + "coarse": 1, + "fine": 1 + } + }, + { + "path": "rngd/preset/params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model-00004-of-00004.safetensors", + "kind": "rngd-params", + "bytes": 1235247792, + "new_bytes": 0, + "chunks": 16430, + "class": "embedded-shifted", + "shares": { + "ref": 0.9454986736782607, + "coarse": 1, + "fine": 1 + } + }, + { + "path": "rngd/coarse/artifact.json", + "kind": "rngd-bundle", + "bytes": 4911304, + "new_bytes": 4911304, + "chunks": 51, + "class": "new", + "shares": { + "ref": 0, + "coarse": 0, + "fine": 0.00033616326743365915 + } + }, + { + "path": "rngd/coarse/binary_bundle.zip", + "kind": "rngd-bundle", + "bytes": 13496362, + "new_bytes": 13428389, + "chunks": 145, + "class": "embedded-rearranged", + "shares": { + "ref": 0, + "coarse": 0.36724155739154, + "fine": 0.9556762037058579 + } + }, + { + "path": "rngd/fine/artifact.json", + "kind": "rngd-bundle", + "bytes": 6108280, + "new_bytes": 6108280, + "chunks": 59, + "class": "new", + "shares": { + "ref": 0, + "coarse": 0.12743521907967545, + "fine": 0.1923469454576411 + } + }, + { + "path": "rngd/fine/binary_bundle.zip", + "kind": "rngd-bundle", + "bytes": 16078475, + "new_bytes": 11726186, + "chunks": 178, + "class": "embedded-rearranged", + "shares": { + "ref": 0, + "coarse": 0.3033376610654928, + "fine": 0.8862514013300391 + } + }, + { + "path": "rngd/pfbatch/artifact.json", + "kind": "rngd-bundle", + "bytes": 7415165, + "new_bytes": 6636755, + "chunks": 76, + "class": "new", + "shares": { + "ref": 0, + "coarse": 0.10497541187552806, + "fine": 0.13045724538833592 + } + }, + { + "path": "rngd/pfbatch/binary_bundle.zip", + "kind": "rngd-bundle", + "bytes": 30673623, + "new_bytes": 25686802, + "chunks": 362, + "class": "embedded-rearranged", + "shares": { + "ref": 0, + "coarse": 0.5364924775922296, + "fine": 0.8834996439774981 + } + }, + { + "path": "rngd/preset/artifact.json", + "kind": "rngd-bundle", + "bytes": 7191899, + "new_bytes": 6413489, + "chunks": 73, + "class": "new", + "shares": { + "ref": 0, + "coarse": 0.10823427859595915, + "fine": 0.1655470968098968 + } + }, + { + "path": "rngd/preset/binary_bundle.zip", + "kind": "rngd-bundle", + "bytes": 30170684, + "new_bytes": 12721461, + "chunks": 358, + "class": "embedded-rearranged", + "shares": { + "ref": 0, + "coarse": 0.578350262128628, + "fine": 0.9579323093901352 + } + }, + { + "path": "rngd/coarse/chat_template.jinja", + "kind": "other", + "bytes": 4614, + "new_bytes": 4614, + "chunks": 1, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 1, + "fine": 1 + } + }, + { + "path": "rngd/coarse/config.json", + "kind": "other", + "bytes": 896, + "new_bytes": 896, + "chunks": 1, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 1, + "fine": 1 + } + }, + { + "path": "rngd/coarse/generation_config.json", + "kind": "other", + "bytes": 183, + "new_bytes": 183, + "chunks": 1, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 1, + "fine": 1 + } + }, + { + "path": "rngd/coarse/tokenizer.json", + "kind": "other", + "bytes": 17209920, + "new_bytes": 17209920, + "chunks": 231, + "class": "embedded-shifted", + "shares": { + "ref": 0.18962121846005095, + "coarse": 1, + "fine": 1 + } + }, + { + "path": "rngd/coarse/tokenizer_config.json", + "kind": "other", + "bytes": 351, + "new_bytes": 351, + "chunks": 1, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 1, + "fine": 1 + } + }, + { + "path": "rngd/fine/chat_template.jinja", + "kind": "other", + "bytes": 4614, + "new_bytes": 0, + "chunks": 1, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 1, + "fine": 1 + } + }, + { + "path": "rngd/fine/config.json", + "kind": "other", + "bytes": 896, + "new_bytes": 0, + "chunks": 1, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 1, + "fine": 1 + } + }, + { + "path": "rngd/fine/generation_config.json", + "kind": "other", + "bytes": 183, + "new_bytes": 0, + "chunks": 1, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 1, + "fine": 1 + } + }, + { + "path": "rngd/fine/tokenizer.json", + "kind": "other", + "bytes": 17209920, + "new_bytes": 0, + "chunks": 231, + "class": "embedded-shifted", + "shares": { + "ref": 0.18962121846005095, + "coarse": 1, + "fine": 1 + } + }, + { + "path": "rngd/fine/tokenizer_config.json", + "kind": "other", + "bytes": 351, + "new_bytes": 0, + "chunks": 1, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 1, + "fine": 1 + } + }, + { + "path": "rngd/pfbatch/chat_template.jinja", + "kind": "other", + "bytes": 4614, + "new_bytes": 0, + "chunks": 1, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 1, + "fine": 1 + } + }, + { + "path": "rngd/pfbatch/config.json", + "kind": "other", + "bytes": 896, + "new_bytes": 0, + "chunks": 1, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 1, + "fine": 1 + } + }, + { + "path": "rngd/pfbatch/generation_config.json", + "kind": "other", + "bytes": 183, + "new_bytes": 0, + "chunks": 1, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 1, + "fine": 1 + } + }, + { + "path": "rngd/pfbatch/params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/cache_access_log.txt", + "kind": "other", + "bytes": 19, + "new_bytes": 19, + "chunks": 1, + "class": "new", + "shares": { + "ref": 0, + "coarse": 0, + "fine": 0 + } + }, + { + "path": "rngd/pfbatch/params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model.safetensors.index.json", + "kind": "other", + "bytes": 22856, + "new_bytes": 22856, + "chunks": 1, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 1, + "fine": 1 + } + }, + { + "path": "rngd/pfbatch/tokenizer.json", + "kind": "other", + "bytes": 17209920, + "new_bytes": 0, + "chunks": 231, + "class": "embedded-shifted", + "shares": { + "ref": 0.18962121846005095, + "coarse": 1, + "fine": 1 + } + }, + { + "path": "rngd/pfbatch/tokenizer_config.json", + "kind": "other", + "bytes": 351, + "new_bytes": 0, + "chunks": 1, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 1, + "fine": 1 + } + }, + { + "path": "rngd/preset/chat_template.jinja", + "kind": "other", + "bytes": 4614, + "new_bytes": 0, + "chunks": 1, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 1, + "fine": 1 + } + }, + { + "path": "rngd/preset/config.json", + "kind": "other", + "bytes": 896, + "new_bytes": 0, + "chunks": 1, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 1, + "fine": 1 + } + }, + { + "path": "rngd/preset/generation_config.json", + "kind": "other", + "bytes": 183, + "new_bytes": 0, + "chunks": 1, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 1, + "fine": 1 + } + }, + { + "path": "rngd/preset/params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/cache_access_log.txt", + "kind": "other", + "bytes": 19, + "new_bytes": 19, + "chunks": 1, + "class": "new", + "shares": { + "ref": 0, + "coarse": 0, + "fine": 0 + } + }, + { + "path": "rngd/preset/params-Llama-3.1-8B-Instruct-llama-32L-W16A16KV16-shard_size=5000000000-500336bba7f6353fa0b3e71a8614ee0f7643eb4399e41451475599c4c28a191c.safetensors/model.safetensors.index.json", + "kind": "other", + "bytes": 22856, + "new_bytes": 0, + "chunks": 1, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 1, + "fine": 1 + } + }, + { + "path": "rngd/preset/tokenizer.json", + "kind": "other", + "bytes": 17209920, + "new_bytes": 0, + "chunks": 231, + "class": "embedded-shifted", + "shares": { + "ref": 0.18962121846005095, + "coarse": 1, + "fine": 1 + } + }, + { + "path": "rngd/preset/tokenizer_config.json", + "kind": "other", + "bytes": 351, + "new_bytes": 0, + "chunks": 1, + "class": "embedded-shifted", + "shares": { + "ref": 0, + "coarse": 1, + "fine": 1 + } + } + ], + "classify": { + "fine_params": { + "Min": 1024, + "Avg": 4096, + "Max": 16384 + }, + "references": [ + "hf-checkpoint" + ], + "ref_share": 0.95, + "rearranged_gap": 0.25, + "shifted_share": 0.5, + "reference_files": 13, + "reference_bytes": 16069779031 + } + } + }, + "placement_report_cost": { + "atom": [ + "Command being timed: \"nice -n 19 ionice -c3 ./onggi-new dedup-report --quiet --json --classify --reference hf-checkpoint --workers 16 atom-set\"", + "\tUser time (seconds): 549.31", + "\tSystem time (seconds): 51.23", + "\tPercent of CPU this job got: 777%", + "\tElapsed (wall clock) time (h:mm:ss or m:ss): 1:17.24", + "\tAverage shared text size (kbytes): 0", + "\tAverage unshared data size (kbytes): 0", + "\tAverage stack size (kbytes): 0", + "\tAverage total size (kbytes): 0", + "\tMaximum resident set size (kbytes): 1841864" + ], + "rngd": [ + "Command being timed: \"nice -n 19 ionice -c3 ./onggi-new dedup-report --quiet --json --classify --reference hf-checkpoint --workers 16 rngd/pfbatch rngd/preset rngd/coarse rngd/fine\"", + "\tUser time (seconds): 92.76", + "\tSystem time (seconds): 9.77", + "\tPercent of CPU this job got: 449%", + "\tElapsed (wall clock) time (h:mm:ss or m:ss): 0:22.81", + "\tAverage shared text size (kbytes): 0", + "\tAverage unshared data size (kbytes): 0", + "\tAverage stack size (kbytes): 0", + "\tAverage total size (kbytes): 0", + "\tMaximum resident set size (kbytes): 449236" + ] + }, + "switch_plans": { + "a3-plan-la-lb.json": { + "schema": "onggi.switchplan.v1", + "bytes_json": 1429, + "size": 15228048882, + "same": 15009857021, + "moved": 0, + "new": 218191861, + "new_stored": 3553089, + "files": [ + { + "path": "decode_S256_B8_lm.rbln", + "base": "decode_S256_B2_lm.rbln", + "size": 15228048882, + "same": 15009857021, + "moved": 0, + "new": 218191861, + "changed_runs": 3, + "ranges_by_kind": { + "new": [ + 3, + 218191861 + ], + "same": [ + 2, + 15009857021 + ] + }, + "n_ranges": 5 + } + ] + }, + "a3-plan-va-vb.json": { + "schema": "onggi.switchplan.v1", + "bytes_json": 17671, + "size": 15228048882, + "same": 0, + "moved": 13959162419, + "new": 1268886463, + "new_stored": 693760073, + "files": [ + { + "path": "decode_S256_B8_lm.rbln", + "base": "decode_S256_B2.rbln", + "size": 15228048882, + "same": 0, + "moved": 13959162419, + "new": 1268886463, + "changed_runs": 1, + "ranges_by_kind": { + "new": [ + 7, + 1268886463 + ], + "elsewhere": [ + 82, + 13959162419 + ] + }, + "n_ranges": 89 + } + ] + }, + "a3-plan-va-vc.json": { + "schema": "onggi.switchplan.v1", + "bytes_json": 68247501, + "size": 14017135688, + "same": 0, + "moved": 11125321034, + "new": 2891814654, + "new_stored": 1921135926, + "files": [ + { + "path": "prefill_S256_C128.rbln", + "base": "decode_S256_B2.rbln", + "size": 14017135688, + "same": 0, + "moved": 11125321034, + "new": 2891814654, + "changed_runs": 1, + "ranges_by_kind": { + "new": [ + 226845, + 2891814654 + ], + "elsewhere": [ + 226894, + 11125321034 + ] + }, + "n_ranges": 453739 + } + ] + }, + "a3-plan-va-vb.time": "\tCommand being timed: \"./onggi-a3 diff --store stores/a3 --json --ranges va vb\"\n\tUser time (seconds): 10.02\n\tSystem time (seconds): 2.74\n\tPercent of CPU this job got: 114%\n\tElapsed (wall clock) time (h:mm:ss or m:ss): 0:11.12\n\tAverage shared text size (kbytes): 0\n\tAverage unshared data size (kbytes): 0\n\tAverage stack size (kbytes): 0\n\tAverage total size (kbytes): 0\n\tMaximum resident set size (kbytes): 2114644\n\tAverage resident set size (kbytes): 0\n\tMajor (requiring I/O) page faults: 0\n\tMinor (reclaiming a frame) page faults: 826018\n\tVoluntary context switches: 3038\n\tInvoluntary context switches: 98\n\tSwaps: 0\n\tFile system inputs: 0\n\tFile system outputs: 40\n\tSocket messages sent: 0\n\tSocket messages received: 0\n\tSignals delivered: 0\n\tPage size (bytes): 4096\n\tExit status: 0\n", + "a3-plan-va-vc.time": "\tCommand being timed: \"./onggi-a3 diff --store stores/a3 --json --ranges va vc\"\n\tUser time (seconds): 9.87\n\tSystem time (seconds): 2.50\n\tPercent of CPU this job got: 112%\n\tElapsed (wall clock) time (h:mm:ss or m:ss): 0:11.00\n\tAverage shared text size (kbytes): 0\n\tAverage unshared data size (kbytes): 0\n\tAverage stack size (kbytes): 0\n\tAverage total size (kbytes): 0\n\tMaximum resident set size (kbytes): 1953432\n\tAverage resident set size (kbytes): 0\n\tMajor (requiring I/O) page faults: 0\n\tMinor (reclaiming a frame) page faults: 860133\n\tVoluntary context switches: 2965\n\tInvoluntary context switches: 33\n\tSwaps: 0\n\tFile system inputs: 0\n\tFile system outputs: 133296\n\tSocket messages sent: 0\n\tSocket messages received: 0\n\tSignals delivered: 0\n\tPage size (bytes): 4096\n\tExit status: 0\n" + }, + "compatibility": { + "main_binary_pulls_store_created_with_size_policy": "manifest sha256:fd5a71b348078b7e12bad4481bfff95625f40ee01fa843aa41c9bc1147b634ab\nfiles 7\nwritten 39.4 MB (read 7.6 MB in 148 ranges)\ntook 0s (688 MB/s)\nrc=0\nIDENTICAL\nREF MANIFEST BYTES\ncoarse sha256:a9a00055de32a0cb38dceab0b6973816001cd47e0c2c0a6332be2070ff90f45d 35.6 MB\nfine sha256:fd5a71b348078b7e12bad4481bfff95625f40ee01fa843aa41c9bc1147b634ab 39.4 MB\npfbatch sha256:28875e8c508edae73e0ef3a2bbb773bda6b91747729122490252322e01e925bf 16.2 GB\npreset sha256:ec3a70c7ec33f12b2833590639880da44416964dc5d226404ecaa2f4b2384907 16.2 GB\n\nstore: 3503926 chunks in 163 packs, 10.7 GB; chunker 16.4 kB/65.5 kB/262.1 kB; durability unknown\ntwo-level: 216922 chunks held as sub-chunks of 1.0 kB/4.1 kB/16.4 kB (all kinds)\n", + "new_binary_reads_store_created_by_main": "REF MANIFEST BYTES\natom sha256:58b87b04ea3066623ef70d49644e609cd77f62914148e88a4c21a75f1f40ee6b 263.9 GB\n\nstore: 4842079 chunks in 233 packs, 15.6 GB; chunker 16.4 kB/65.5 kB/262.1 kB; durability unknown\ntwo-level: 582709 chunks held as sub-chunks of 1.0 kB/4.1 kB/16.4 kB (kinds [atom-prefill atom-decode atom-other])\n{'schema': 'onggi.store.v1', 'chunker': {'Min': 16384, 'Avg': 65536, 'Max': 262144}, 'pack_size': 67108864, 'sub_chunker': {'Min': 1024, 'Avg': 4096, 'Max': 16384}, 'sub_kinds': ['atom-prefill', 'atom-decode', 'atom-other'], 'frame': 262144, 'created': '2026-10-30T07:36:15.209415807Z'}\n" + } +} \ No newline at end of file diff --git a/docs/evaluation.md b/docs/evaluation.md index 2cd6134..9616328 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -89,8 +89,8 @@ reads 10.8 GB. Since #58 a pull reads such bytes from the base's files where the the ATOM set, prefill C128 next to a decode B2 copy (0 % at the same offsets, 79 % elsewhere) read 1.93 GB from the store instead of 9.47 GB and 11.1 GB from the base, byte-identical. -Unlike ATOM, sub-chunks are not used here (TensorRT engines are kind `other`); 64 KiB chunks already find the shifted -weights. +These engines were stored before #74, without sub-chunks (TensorRT engines were kind `other`); 64 KiB chunks already +find the shifted weights. Since #74 new stores sub-chunk them like any file of at least 1 MiB. ## RNGD: furiosa-llm artifacts (#54) diff --git a/docs/sub-chunk-policy.md b/docs/sub-chunk-policy.md new file mode 100644 index 0000000..b12cb70 --- /dev/null +++ b/docs/sub-chunk-policy.md @@ -0,0 +1,107 @@ +# Which files get sub-chunks (#74) + +Two-level chunking (#15) re-splits the chunks a push finds new into ~4 KiB sub-chunks, so files that hold the same +bytes in a different order still dedup. Until #74 a new store chose those files **by name**: `--sub-kinds +atom-prefill,atom-decode,atom-other`, which `pkg/layout` derives from the `.rbln` extension and the words `prefill` +and `decode`. Every other format, such as TensorRT-LLM engines or furiosa-llm bundles, got one level only, and a +renamed ATOM file lost the second level. + +Since #74 a new store decides **by size alone**: new chunks of every file of at least `--sub-min-file` (default +**1 MiB**) get sub-chunks. No name, extension or format is read (ADR-0009). `pkg/layout` is used only by reports +(`dedup-report` groups, `inspect`) and as a pairing hint in `diff`. + +## Candidates and measurements + +Measured 2026-10-30 on the lab node (ATOM build set, HF checkpoint and RNGD artifacts on local disk; `nice -n 19`, +16 threads, shared node; n = 1 per configuration, since stored sizes are deterministic for a given push order). Raw data: +[`data/sub-chunk-policy-20261030.json`](data/sub-chunk-policy-20261030.json). + +- **kind**: the old default, file kinds by name. +- **size ≥ 1 MiB** (chosen) and **size ≥ 64 MiB**. +- **probe** (experiment, commit ec9bc0f, not merged): sub-chunk a file's first 64 new chunks; continue only if at + least 10 % of their sub-chunk bytes were already stored. + +| store after pushing | kind | size ≥ 1 MiB | probe | +|---|---|---|---| +| ATOM set (263.9 GB, 19 files) | **16.70 GB** (metadata 1.08 GB), 110 s | 16.78 GB (+0.5 %; 1.16 GB), 120 s | 26.92 GB (+61 %; 0.69 GB), 172 s | +| ATOM set, then HF checkpoint | 27.44 GB | 28.06 GB | 37.66 GB | +| HF checkpoint alone (16.07 GB) | 10.73 GB (metadata 0.03 GB), 38 s | 11.28 GB (+5.1 %; 0.56 GB), 122 s | 10.74 GB, 104 s¹ | +| RNGD pfbatch, preset, coarse, fine (32.4 GB) | 10.79 GB, pfbatch push 38 s | 11.53 GB (+6.9 %), 113 s | 10.78 GB, 37 s | + +| switch: bytes a node fetches | kind | size ≥ 1 MiB | size ≥ 64 MiB | probe | +|---|---|---|---|---| +| ATOM decode B2 → B8 (15.2 GB, `_lm` variants) | 3.57 MB | 3.57 MB | — | 3.58 MB | +| ATOM decode B2 (no lm_head) → B8 | 694 MB | 694 MB | — | 697 MB | +| RNGD pfbatch → preset | 4.33 MB | **0.65 MB** | 4.33 MB | 2.59 MB | + +Probe sizes are from a push with one worker (deterministic order); with 16 workers the ATOM set stores 27.00 GB and +RNGD 10.78 GB, and the push times shown for the ATOM set and RNGD are from that run. ¹ one worker. The binary from +`main` with its defaults stores the ATOM set in 16.70 GB, the same as **kind**. + +What the numbers say: + +- **Size** never loses dedup. On the ATOM set it equals the kind policy (it also sub-chunks `tables.npz`, +77 MB), + and it finds the furiosa-llm graph bundles' rearranged bytes: the RNGD switch fetches 0.65 MB instead of 4.33 MB. + Its cost is metadata where content is new and not rearranged, as in a checkpoint pushed for the first time: + +0.53 GB (3.3 % of 16 GB) of indexes and composite tables, about 3× push time for that new content, and a locator of + ~230 MB instead of ~12 MB for every artifact that references the sub-chunked checkpoint (RNGD preset). +- **Probe** keeps checkpoints cheap but loses 10.2 GB on the ATOM set. The first file of a family finds nothing + stored, so it is not sub-chunked, and later files cannot match its bytes at 4 KiB. Which file comes first depends + on push order and concurrency, so the loss does too. +- **64 MiB** costs as much as 1 MiB (checkpoint shards are GBs) and misses the 13–30 MB bundles. + +Bytes alone cannot tell the first copy of rearranged weights from a checkpoint. A policy that skips sub-chunks for +new content (probe) therefore trades a bounded metadata cost for an unbounded dedup loss. The size policy is the +vendor-neutral default: the worst case is a few percent of metadata on new checkpoints. Operators who know a store +holds only checkpoints can create it with `--sub 0`; `--sub-kinds` still gives the old behaviour. + +Follow-up: locators list the sub-chunks of every composite an artifact references, so referenced checkpoints make +every artifact's locator large. Referencing composites instead would remove most of the size policy's cost. + +## Compatibility + +- **Existing stores** keep what their config records. A store with `sub_kinds` keeps deciding by kind + (`TestSubPolicyIsStoredWithTheStore`), and the binary from this branch reads and pushes to stores made by `main`. +- **New stores** record `"sub_min_file": 1048576` in `config/store`; `onggi ls` shows the policy. +- **Formats** are unchanged: manifests (including `files[].kind`), chunk lists, locators, indexes, composite tables. + The binary from `main` pulled an artifact from a store created by this branch, byte-identical. +- **Old binaries pushing to a new store** ignore `sub_min_file` (unknown JSON field) and, since the store records no + `sub_kinds`, sub-chunk every file, small ones included. The result is valid and readable by every version; it only + costs more metadata for files under 1 MiB. +- **CLI:** `--sub-min-file SIZE` (default 1MiB); `--sub-kinds LIST|all` overrides it (the behaviour before #74); + both only when creating a store. `--sub 0` still turns the second level off. + +## Placement classes (`onggi dedup-report --classify`) + +`dedup-report --classify [--reference PATH]...` also reports, per file, how the bytes it shares can be found, from +chunk hashes alone. "Elsewhere" means another file of the set or a reference file. Each coarse 64 KiB chunk is split +into fine chunks (`--fine 4KiB`), as the store's second level does. The rules apply in this order: + +| class | rule (default threshold) | meaning | +|---|---|---| +| `referenced` | ≥ `--ref-share` 0.95 of the bytes are in the references at coarse chunks | a copy of the checkpoint | +| `embedded-rearranged` | fine share − coarse share ≥ `--rearranged-gap` 0.25 | shared bytes in runs shorter than a chunk: needs the second level | +| `embedded-shifted` | coarse share ≥ `--shifted-share` 0.5 | shared bytes copied whole, at any offset | +| `new` | otherwise | | + +The JSON gains `files[].class`, `files[].shares` (`ref`, `coarse`, `fine`) and a `classify` block with the thresholds. +Fine-chunk keys are the first 8 bytes of SHA-256; a collision only moves bytes between shares. + +On the lab node, with the HF checkpoint as reference: + +- **ATOM set** (77 s, 1.8 GB RSS): + - 13 decode graphs, `llama8b_S256_fp16` and prefill C16 are `embedded-shifted` (coarse 98.6–99.8 %). + - Prefill C128, C256 and S512_C512 are `embedded-rearranged` (coarse 39–45 %, fine 87–90 %). + - `tables.npz` is `new`. + - No ATOM file shares bytes with the bf16 checkpoint (ref 0.0). +- **RNGD artifacts** (23 s, 0.45 GB RSS): + - Param shards 1–3 are `referenced` (ref 1.000). + - Shard 4 holds 5.5 % that is not in the checkpoint (ref 0.9455), so it is `embedded-shifted`. + - The four `binary_bundle.zip` are `embedded-rearranged` (coarse 30–58 %, fine 88–96 %). This is why the size + policy, which also sub-chunks them, cuts the pfbatch → preset switch from 4.33 to 0.65 MB. + - `artifact.json` is `new`. + +Identical small files (configs, tokenizer) show as `embedded-shifted`. The classes describe where shared bytes lie, +not what a file is, and they mean most for weight-bearing files. A TensorRT-LLM engine, 99.8 % of whose bytes are in +its checkpoint (#56), would be `referenced` against the checkpoint and `embedded-shifted` against sibling engines +without one. The engines live on the lab A100 host, not on this node, and were not re-measured here. diff --git a/pkg/store/format.go b/pkg/store/format.go index 5c3632d..f89b233 100644 --- a/pkg/store/format.go +++ b/pkg/store/format.go @@ -237,21 +237,12 @@ type Config struct { // format (#74). With neither SubKinds nor SubMinFile, every file gets sub-chunks; that is also // what binaries from before #74 do on a store that records only SubMinFile. SubMinFile int64 `json:"sub_min_file,omitempty"` - // SubProbe is an experiment (#74); binaries from before it ignore it like SubMinFile. - SubProbe *SubProbe `json:"sub_probe,omitempty"` // Frame, when set, makes push compress consecutive new chunks of a sub-chunked file // together, up to this many raw bytes (#52). Stores created before it have none. Frame int64 `json:"frame,omitempty"` Created time.Time `json:"created"` } -// SubProbe stops sub-chunking a file when the first Chunks new chunks it sub-chunks find less -// than MinShare of their bytes already stored (#74 experiment). -type SubProbe struct { - Chunks int `json:"chunks"` - MinShare float64 `json:"min_share"` -} - // File is one file of an artifact. type File struct { Path string `json:"path"` // slash-separated, relative to the artifact root diff --git a/pkg/store/push.go b/pkg/store/push.go index 22853a9..e914322 100644 --- a/pkg/store/push.go +++ b/pkg/store/push.go @@ -411,9 +411,6 @@ func (s *Store) chunkFile(ctx context.Context, path string, twoLevel bool, mode if twoLevel && s.Config.Frame > 0 { fb = &frameBuilder{limit: int(s.Config.Frame), mode: mode} } - probe := s.Config.SubProbe // experiment (#74) - var probed int - var probeBytes, probeFound int64 for { ch, err := c.Next() if errors.Is(err, io.EOF) { @@ -445,11 +442,7 @@ func (s *Store) chunkFile(ctx context.Context, path string, twoLevel bool, mode err = s.Config.Sub.Split(ch, func(sub []byte) error { sd := digest.Of(sub) subs = append(subs, ChunkRef{Digest: sd, Length: uint32(len(sub))}) - fresh := sd == d || claim(sd) - if !fresh { - probeFound += int64(len(sub)) - } - if fresh { // sd == d: the whole chunk is one sub-chunk; this push owns d already + if sd == d || claim(sd) { // sd == d: the whole chunk is one sub-chunk; this push owns d already if fb != nil { n, err := fb.add(ctx, sub, sd, out) nb += n @@ -466,13 +459,6 @@ func (s *Store) chunkFile(ctx context.Context, path string, twoLevel bool, mode if err != nil { return nil, 0, err } - if probe != nil && probed < probe.Chunks { - probed++ - probeBytes += int64(len(ch)) - if probed == probe.Chunks && float64(probeFound) < probe.MinShare*float64(probeBytes) { - twoLevel = false // the rest of the file goes out as whole chunks - } - } if len(subs) == 1 && subs[0].Digest == d { continue // one sub-chunk equal to the chunk: it is stored as bytes under d } diff --git a/pkg/store/store.go b/pkg/store/store.go index b5533e0..f7d2bdf 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -54,7 +54,6 @@ type Create struct { // SubMinFile sub-chunks the new chunks of every file of at least this many bytes (#74); used // only with Sub. With neither SubKinds nor SubMinFile, every file gets sub-chunks. SubMinFile int64 - SubProbe *SubProbe // experiment (#74): stop sub-chunking a file whose first new chunks find nothing PackSize int64 Frame int64 // compression frames for sub-chunked files (#52); used only with Sub } @@ -64,8 +63,7 @@ type Create struct { var DefaultSubKinds = []layout.Kind{layout.ATOMPrefill, layout.ATOMDecode, layout.ATOMOther} // DefaultSubMinFile is the file size from which new stores sub-chunk new chunks (#74). It looks -// at no name or format: weight-bearing files are large, and below it the second level only adds -// metadata. +// at no name or format; docs/sub-chunk-policy.md has the measurements behind it. const DefaultSubMinFile = 1 << 20 // Open reads the store's config, creating it with p and packSize (one level of chunks) if the @@ -98,7 +96,7 @@ func OpenWith(ctx context.Context, b backend.Backend, c Create) (*Store, error) } cfg = Config{Schema: "onggi.store.v1", Chunker: p, PackSize: packSize, Sub: c.Sub, Created: time.Now().UTC()} if c.Sub != nil { - cfg.SubKinds, cfg.SubMinFile, cfg.SubProbe, cfg.Frame = c.SubKinds, c.SubMinFile, c.SubProbe, c.Frame + cfg.SubKinds, cfg.SubMinFile, cfg.Frame = c.SubKinds, c.SubMinFile, c.Frame } data, _ := json.MarshalIndent(cfg, "", " ") if err := b.Put(ctx, configKey, bytes.NewReader(data), int64(len(data))); err != nil {