diff --git a/.dockerignore b/.dockerignore index 5e798fa0..f4dee858 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,2 @@ -images \ No newline at end of file +images +data \ No newline at end of file diff --git a/.gitignore b/.gitignore index 4ee35b07..b258253c 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,4 @@ hs_err_pid* # log /log/ +data \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 33050bf0..aa3cf61e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM maven:3.8-openjdk-11-slim AS builder +FROM maven:3.8.7-openjdk-18-slim AS builder WORKDIR /app COPY . /app diff --git a/deps/com/moilioncircle/redis-replicator/3.8.1-modif/_remote.repositories b/deps/com/moilioncircle/redis-replicator/3.8.1-modif/_remote.repositories new file mode 100644 index 00000000..69733271 --- /dev/null +++ b/deps/com/moilioncircle/redis-replicator/3.8.1-modif/_remote.repositories @@ -0,0 +1,4 @@ +#NOTE: This is a Maven Resolver internal implementation file, its format can be changed without prior notice. +#Tue Apr 08 19:42:06 IDT 2025 +redis-replicator-3.8.1-modif.jar>= +redis-replicator-3.8.1-modif.pom>= diff --git a/deps/com/moilioncircle/redis-replicator/3.8.1-modif/redis-replicator-3.8.1-modif.jar b/deps/com/moilioncircle/redis-replicator/3.8.1-modif/redis-replicator-3.8.1-modif.jar new file mode 100644 index 00000000..c1b0fdbe Binary files /dev/null and b/deps/com/moilioncircle/redis-replicator/3.8.1-modif/redis-replicator-3.8.1-modif.jar differ diff --git a/deps/com/moilioncircle/redis-replicator/3.8.1-modif/redis-replicator-3.8.1-modif.pom b/deps/com/moilioncircle/redis-replicator/3.8.1-modif/redis-replicator-3.8.1-modif.pom new file mode 100644 index 00000000..a8804cee --- /dev/null +++ b/deps/com/moilioncircle/redis-replicator/3.8.1-modif/redis-replicator-3.8.1-modif.pom @@ -0,0 +1,457 @@ + + + + + 4.0.0 + + com.moilioncircle + redis-replicator + 3.8.1-modif + + redis-replicator + + Redis Replicator implement Redis Replication protocol written in java. + It can parse,filter,broadcast the RDB and AOF events in a real time manner. + It also can synchronize redis data to your local cache or to database. + + https://github.com/leonchen83/redis-replicator + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + + + + + + Leon Chen + chen.bao.yi@gmail.com + https://github.com/leonchen83 + moilioncircle + http://www.moilioncircle.com/ + + Developer + + +8 + + + + Adrian Yao + unstudy + adrianyaofly@gmail.com + https://github.com/adrianyao89 + +8 + + + + Trydofor + trydofor@gmail.com + https://github.com/trydofor + moilioncircle + http://www.moilioncircle.com/ + + Developer + + +8 + + + + + scm:git:git@github.com:leonchen83/redis-replicator.git + scm:git:git@github.com:leonchen83/redis-replicator.git + scm:git:git@github.com:leonchen83/redis-replicator.git + 3.x + + + + github + https://github.com/leonchen83/redis-replicator/issues + + + + UTF-8 + 1.8 + 1.8 + + 2.0.7 + + + + + org.slf4j + slf4j-api + ${slf4j.api.version} + + + + junit + junit + 4.13.2 + test + + + org.apache.logging.log4j + log4j-slf4j2-impl + 2.20.0 + test + + + org.slf4j + slf4j-api + + + + + org.apache.logging.log4j + log4j-core + 2.20.0 + test + + + redis.clients + jedis + 5.1.0 + test + + + + + + release + + + + org.apache.maven.plugins + maven-jar-plugin + 3.3.0 + + + + com.moilioncircle.redis.replicator + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.11.0 + + + base-compile + + compile + + + + module-info.java + + 8 + + + + default-compile + + 9 + + + + + + -parameters + + ${project.build.sourceEncoding} + + + + org.apache.maven.plugins + maven-resources-plugin + 3.3.1 + + ${project.build.sourceEncoding} + + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.5.0 + + none + ${maven.compiler.source} + ${project.build.sourceEncoding} + + **/module-info.java + + + + + package + + jar + + + + + + org.apache.maven.plugins + maven-source-plugin + 3.2.1 + + + attach-sources + + jar + + + + + + org.apache.maven.plugins + maven-gpg-plugin + 3.0.1 + + + verify + + sign + + + + + + --pinentry-mode + loopback + + + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.13 + true + + ossrh + https://oss.sonatype.org/ + true + + + + + + + oss + https://oss.sonatype.org/content/repositories/snapshots/ + + + oss + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + + + + CI + + + + org.apache.maven.plugins + maven-jar-plugin + 3.3.0 + + + + com.moilioncircle.redis.replicator + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.11.0 + + + base-compile + + compile + + + + module-info.java + + 8 + + + + default-compile + + 9 + + + + + + -parameters + + ${project.build.sourceEncoding} + + + + org.apache.maven.plugins + maven-resources-plugin + 3.3.1 + + ${project.build.sourceEncoding} + + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.5.0 + + none + ${maven.compiler.source} + ${project.build.sourceEncoding} + + **/module-info.java + + + + + package + + jar + + + + + + org.apache.maven.plugins + maven-source-plugin + 3.2.1 + + + attach-sources + + jar + + + + + + + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.3.0 + + + + com.moilioncircle.redis.replicator + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.11.0 + + + base-compile + + compile + + + + module-info.java + + 8 + + + + default-compile + + 9 + + + + + + -parameters + + ${project.build.sourceEncoding} + + + + org.eluder.coveralls + coveralls-maven-plugin + 4.3.0 + + IgJ5nELUEO3RFdZl8BYnkeNjOxXMFQIkT + + + + javax.xml.bind + jaxb-api + 2.3.1 + + + + + org.jacoco + jacoco-maven-plugin + 0.8.10 + + + prepare-agent + + prepare-agent + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.4.0 + + + add-test-source + generate-test-sources + + add-test-source + + + + examples + + + + + + + + diff --git a/deps/com/moilioncircle/redis-replicator/maven-metadata-local.xml b/deps/com/moilioncircle/redis-replicator/maven-metadata-local.xml new file mode 100644 index 00000000..ff71b8a7 --- /dev/null +++ b/deps/com/moilioncircle/redis-replicator/maven-metadata-local.xml @@ -0,0 +1,12 @@ + + + com.moilioncircle + redis-replicator + + 3.8.1 + + 3.8.1 + + 20250408160242 + + diff --git a/dump.rdb b/dump.rdb new file mode 100644 index 00000000..45dba5f5 Binary files /dev/null and b/dump.rdb differ diff --git a/pom.xml b/pom.xml index 869556c2..f3e8afc6 100644 --- a/pom.xml +++ b/pom.xml @@ -16,8 +16,8 @@ --> + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 com.moilioncircle @@ -66,11 +66,19 @@ 1.8 + + + project.local + project + file:${project.basedir}/deps + + + com.moilioncircle redis-replicator - 3.11.0 + 3.8.1-modif org.slf4j @@ -78,6 +86,7 @@ + com.moilioncircle redis-rdb-cli-api @@ -215,4 +224,4 @@ - + \ No newline at end of file diff --git a/src/main/java/com/moilioncircle/redis/rdb/cli/glossary/Action.java b/src/main/java/com/moilioncircle/redis/rdb/cli/glossary/Action.java index eb8c33c3..5f7b039b 100644 --- a/src/main/java/com/moilioncircle/redis/rdb/cli/glossary/Action.java +++ b/src/main/java/com/moilioncircle/redis/rdb/cli/glossary/Action.java @@ -23,6 +23,8 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.ArrayList; import java.util.List; @@ -42,6 +44,9 @@ import com.moilioncircle.redis.replicator.Replicator; import com.moilioncircle.redis.replicator.io.CRCOutputStream; import com.moilioncircle.redis.replicator.io.RedisInputStream; +import com.moilioncircle.redis.replicator.rdb.BaseRdbEncoder; +import com.moilioncircle.redis.replicator.rdb.BaseRdbParser; +import com.moilioncircle.redis.replicator.util.ByteArray; import com.moilioncircle.redis.replicator.util.Tuples; import com.moilioncircle.redis.replicator.util.type.Tuple2; @@ -61,6 +66,7 @@ public List> dress(Configure configure, Args.RdtArgs if (isEmpty(arg.merge)) return list; CRCOutputStream out = Outputs.newCRCOutput(arg.output, configure.getOutputBufferSize()); int version = 0; + long totalUsedMem = 0; for (File file : arg.merge) { RedisURI uri = XUris.fromFile(file); if (uri.getFileType() == null || uri.getFileType() != RDB) { @@ -68,6 +74,7 @@ public List> dress(Configure configure, Args.RdtArgs } version = maxVersion(version, file); + totalUsedMem += scanUsedMem(file); Replicator r = new XRedisReplicator(uri, configure, DefaultReplFilter.RDB); r.setRdbVisitor(new MergeRdbVisitor(r, configure, arg, () -> out)); @@ -84,6 +91,9 @@ public List> dress(Configure configure, Args.RdtArgs // header & version out.write("REDIS".getBytes()); out.write(Strings.lappend(version, 4, '0').getBytes()); + if (totalUsedMem > 0) { + writeUsedMemAux(out, totalUsedMem); + } return list; case SPLIT: Replicator r = new XRedisReplicator(arg.split, configure, DefaultReplFilter.RDB); @@ -103,6 +113,33 @@ public List> dress(Configure configure, Args.RdtArgs } } + private long scanUsedMem(File file) { + try (RedisInputStream in = new RedisInputStream(new FileInputStream(file))) { + in.skip(9); // skip "REDIS" magic (5 bytes) + version (4 bytes) + BaseRdbParser parser = new BaseRdbParser(in); + while (true) { + int opcode = in.read(); + if (opcode != 0xFA) break; // not an aux field opcode + byte[] key = parser.rdbLoadEncodedStringObject().first(); + byte[] value = parser.rdbLoadEncodedStringObject().first(); + if ("used-mem".equals(new String(key, StandardCharsets.UTF_8))) { + return Long.parseLong(new String(value, StandardCharsets.UTF_8)); + } + } + } catch (Exception ignore) { + } + return 0L; + } + + private void writeUsedMemAux(OutputStream out, long usedMem) throws IOException { + BaseRdbEncoder encoder = new BaseRdbEncoder(); + byte[] keyBytes = "used-mem".getBytes(); + byte[] valueBytes = Long.toString(usedMem).getBytes(); + out.write(0xFA); // RDB_OPCODE_AUX + encoder.rdbSavePlainStringObject(new ByteArray(keyBytes), out); + encoder.rdbSavePlainStringObject(new ByteArray(valueBytes), out); + } + private int maxVersion(int version, File file) throws IOException { try (RedisInputStream in = new RedisInputStream(new FileInputStream(file))) { in.skip(5); // skip REDIS