Skip to content

Paper baselines: tensor-level dedup, rsync/xdelta3, repeated runs and ablation (#73) #77

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
gspark-etri wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Paper baselines: tensor-level dedup, rsync/xdelta3, repeated runs and ablation (#73) #77

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
131 changes: 131 additions & 0 deletions bench/repeat.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
#!/usr/bin/env bash
# Repeated onggi measurements and an ablation for the paper (#73), on one build set and one variant switch.
#
# bench/repeat.sh repeat <onggi binary> <build dir> <work dir> <variant A file> <variant B file> [n]
# bench/repeat.sh ablation <onggi binary> <build dir> <work dir> <variant A file> <variant B file>
#
# repeat, n times (default 5), interleaved in each round:
# push-set push the whole build dir into a new store: stored bytes (du, with indexes), time, peak RSS
# transfer-xet push A, then B, into a new store with --sub 0 (Xet-style): B's pack bytes and push time
# transfer-onggi the same with onggi's defaults
# pull-plain pull B into an empty dir from the second store (no cache): bytes read, time
# pull-delta pull A, then B with --base A --base-dir: bytes read, bytes left in place, time
# plain and delta pulls alternate which goes first. ablation runs push-set and the A → B transfer once for each of:
# default, --sub 0, --frame 0, --compress none.
#
# Prints one JSON object per measurement (run, method, flags, bytes, seconds, peak RSS KiB, and the onggi JSON).
# Pulled files are checked against the original's sha256. Needs GNU time at /usr/bin/time and about
# 2 × |A| + 3 × |B| + one store of the set free in the work dir. Everything runs under nice; nothing outside the
# work dir is written. The page cache cannot be dropped without root: the build set is read in every round.
set -euo pipefail
mode=$1 onggi=$2 src=$3 work=$4 va=$5 vb=$6 n=${7:-5}
N="nice -n 19"
mkdir -p "$work/tmp"
t0() { /usr/bin/time -v -o "$work/time.txt" $N "$@"; }
emit() { # emit <run> <method> <flags> <bytes> <json file|-> [extra json]
python3 - "$work/time.txt" "$@" << 'EOF'
import json, sys
tf, run, method, flags, nbytes, jf = sys.argv[1:7]
extra = json.loads(sys.argv[7]) if len(sys.argv) > 7 else {}
wall = rss = None
for l in open(tf):
if "Elapsed (wall clock)" in l:
wall = 0.0
for p in l.rsplit(" ", 1)[1].strip().split(":"):
wall = wall * 60 + float(p)
if "Maximum resident set size" in l:
rss = int(l.split(":")[1])
d = {"run": int(run), "method": method, "flags": flags, "bytes": int(nbytes), "seconds": round(wall, 2), "peak_rss_kib": rss}
if jf != "-":
d["onggi"] = json.load(open(jf))
d.update(extra)
print(json.dumps(d), flush=True)
EOF
}
jget() { python3 -c "import json,sys;d=json.load(open(sys.argv[1]));d=d.get('stats',d);print(d[sys.argv[2]])" "$@"; }
want=$(sha256sum "$src/$vb" | cut -c1-64)
same() { [ "$(sha256sum "$1" | cut -c1-64)" = "$want" ] && echo true || echo false; }

# the two variants, each alone in a directory (copies: the build dir is only read)
for v in a b; do
f=$va
[ $v = b ] && f=$vb
if [ ! -f "$work/v$v/$f" ]; then
rm -rf "$work/v$v" && mkdir -p "$work/v$v" && cp "$src/$f" "$work/v$v/"
fi
done

push_set() { # push_set <run> <name> [flags]
local run=$1 name=$2
shift 2
rm -rf "$work/set"
t0 "$onggi" push --store "$work/set" "$@" --quiet --json --tmp "$work/tmp" "$src" > "$work/push-set.json"
emit "$run" push-set "$name" "$(du -sb "$work/set" | cut -f1)" "$work/push-set.json"
rm -rf "$work/set"
}

transfer() { # transfer <run> <name> <store> [flags]: push A, then B; B's packs are what crosses the network
local run=$1 name=$2 st=$3
shift 3
rm -rf "$st"
$N "$onggi" push --store "$st" "$@" --ref a --quiet --json --tmp "$work/tmp" "$work/va" > /dev/null
t0 "$onggi" push --store "$st" "$@" --ref b --quiet --json --tmp "$work/tmp" "$work/vb" > "$work/push-b.json"
emit "$run" "transfer-$name" "$name" "$(jget "$work/push-b.json" pack_bytes)" "$work/push-b.json"
}

pull_plain() {
local run=$1 st=$2
rm -rf "$work/out"
t0 "$onggi" pull --store "$st" --json b "$work/out" > "$work/pull.json"
emit "$run" pull-plain default "$(jget "$work/pull.json" read_bytes)" "$work/pull.json" \
"{\"identical\": $(same "$work/out/$vb")}"
rm -rf "$work/out"
}

pull_delta() {
local run=$1 st=$2
rm -rf "$work/base" "$work/out"
$N "$onggi" pull --store "$st" a "$work/base" > /dev/null
t0 "$onggi" pull --store "$st" --json --base a --base-dir "$work/base" b "$work/out" > "$work/pull.json"
emit "$run" pull-delta default "$(jget "$work/pull.json" read_bytes)" "$work/pull.json" \
"{\"identical\": $(same "$work/out/$vb")}"
rm -rf "$work/base" "$work/out"
}

case $mode in
repeat)
for i in $(seq 1 "$n"); do
push_set "$i" default
transfer "$i" xet "$work/t-xet" --sub 0
rm -rf "$work/t-xet"
transfer "$i" onggi "$work/t-onggi"
if [ "$i" = 1 ]; then
$N "$onggi" diff --store "$work/t-onggi" --json --ranges a b > "$work/diff.json"
python3 -c "import json,sys;d=json.load(open(sys.argv[1]));print(json.dumps({'run':0,'method':'diff','diff':d}))" "$work/diff.json"
fi
if [ $((i % 2)) = 1 ]; then
pull_plain "$i" "$work/t-onggi"
pull_delta "$i" "$work/t-onggi"
else
pull_delta "$i" "$work/t-onggi"
pull_plain "$i" "$work/t-onggi"
fi
rm -rf "$work/t-onggi"
done
;;
ablation)
for m in default: sub0:--sub=0 frame0:--frame=0 compress-none:--compress=none; do
name=${m%%:*} flag=${m#*:}
# shellcheck disable=SC2086
push_set 1 "$name" $flag
# shellcheck disable=SC2086
transfer 1 "$name" "$work/t-$name" $flag
rm -rf "$work/t-$name"
done
;;
*)
echo "usage: bench/repeat.sh repeat|ablation <onggi> <build dir> <work dir> <A> <B> [n]" >&2
exit 2
;;
esac
rm -rf "$work/va" "$work/vb" "$work/tmp"
59 changes: 59 additions & 0 deletions bench/switch_baselines.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
#!/usr/bin/env bash
# Switch baselines for the paper (#73): turning a node's copy of variant A into variant B with rsync and xdelta3.
#
# bench/switch_baselines.sh <build dir> <work dir> <variant A file> <variant B file> [xdelta3 binary] [window]
#
# rsync: the node's copy of A is placed under B's name and rsync --no-whole-file brings it to B (local transfer,
# so sender and receiver talk through a pipe; "Total bytes sent" is what would cross the network), once without
# and once with -z (zstd level 3). xdelta3: encode B against A with a source window of [window] bytes
# (default 2 GiB), decode it, compare. Every result is checked against B's sha256. Needs GNU time at
# /usr/bin/time and about 3 × the size of B free in the work dir. Prints TSV rows:
# tool, variant, bytes (crossing the network), seconds, peak RSS KiB, identical
# and keeps the logs in the work dir. Everything runs under nice; nothing outside the work dir is written.
set -euo pipefail
src=$1 work=$2 va=$3 vb=$4 xd=${5:-xdelta3} win=${6:-2147483648}
mkdir -p "$work"
N="nice -n 19"
row() { printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$@"; }
tv() { # tv <log> cmd...: run under GNU time; the log gets time's report
local log=$1
shift
/usr/bin/time -v -o "$log.time" $N "$@" > "$log" 2>&1
}
wall() { python3 - "$1" << 'EOF'
import sys
for l in open(sys.argv[1]):
if "Elapsed (wall clock)" in l:
t = l.rsplit(" ", 1)[1].strip().split(":")
s = 0.0
for p in t:
s = s * 60 + float(p)
print(round(s, 2))
EOF
}
rss() { awk -F': ' '/Maximum resident set size/ {print $2}' "$1"; }
want=$(sha256sum "$src/$vb" | cut -c1-64)
same() { [ "$(sha256sum "$1" | cut -c1-64)" = "$want" ] && echo yes || echo NO; }

# rsync, without and with compression
for z in plain:"" zstd:"-z --compress-choice=zstd --compress-level=3"; do
name=${z%%:*} flags=${z#*:}
rm -rf "$work/rs" && mkdir -p "$work/rs"
cp "$src/$va" "$work/rs/$vb"
log=$work/rsync-$name.log
# shellcheck disable=SC2086
tv "$log" rsync --no-whole-file --ignore-times --stats $flags "$src/$vb" "$work/rs/"
sent=$(awk -F': ' '/^Total bytes sent/ {gsub(/,/, "", $2); print $2}' "$log")
row "rsync-$name" "$vb" "$sent" "$(wall "$log.time")" "$(rss "$log.time")" "$(same "$work/rs/$vb")"
grep -E '^(Literal data|Matched data|Total bytes (sent|received))' "$log" | sed "s/^/# rsync-$name: /"
rm -rf "$work/rs"
done

# xdelta3: encode against A with a large source window, then decode
log=$work/xdelta3-encode.log
tv "$log" "$xd" -e -f -B "$win" -s "$src/$va" "$src/$vb" "$work/delta.vcdiff"
row xdelta3-encode "$vb" "$(stat -c %s "$work/delta.vcdiff")" "$(wall "$log.time")" "$(rss "$log.time")" -
log=$work/xdelta3-decode.log
tv "$log" "$xd" -d -f -B "$win" -s "$src/$va" "$work/delta.vcdiff" "$work/out"
row xdelta3-decode "$vb" "$(stat -c %s "$work/delta.vcdiff")" "$(wall "$log.time")" "$(rss "$log.time")" "$(same "$work/out")"
rm -f "$work/delta.vcdiff" "$work/out"
Loading
Loading