From e85e28b6651b49de45e87f0f713011103d266f37 Mon Sep 17 00:00:00 2001 From: AntiO2 Date: Tue, 21 Oct 2025 20:03:12 +0800 Subject: [PATCH 1/9] retina ts test --- index/rocksdb/CMakeLists.txt | 51 ++ index/rocksdb/config.h | 45 ++ index/rocksdb/cpp.gitignore | 1047 +++++++++++++++++++++++++++++ index/rocksdb/rocksdb_factory.cpp | 56 ++ index/rocksdb/rocksdb_factory.h | 17 + index/rocksdb/rocksdb_load.cpp | 82 +++ index/rocksdb/rocksdb_test.cc | 109 +++ 7 files changed, 1407 insertions(+) create mode 100644 index/rocksdb/CMakeLists.txt create mode 100644 index/rocksdb/config.h create mode 100644 index/rocksdb/cpp.gitignore create mode 100644 index/rocksdb/rocksdb_factory.cpp create mode 100644 index/rocksdb/rocksdb_factory.h create mode 100644 index/rocksdb/rocksdb_load.cpp create mode 100644 index/rocksdb/rocksdb_test.cc diff --git a/index/rocksdb/CMakeLists.txt b/index/rocksdb/CMakeLists.txt new file mode 100644 index 0000000..37f42ff --- /dev/null +++ b/index/rocksdb/CMakeLists.txt @@ -0,0 +1,51 @@ +cmake_minimum_required(VERSION 3.10) +project(rocksdb_bench LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_FLAGS "-O2 -pthread") + +# RocksDB +include_directories(/usr/local/include) +link_directories(/usr/local/lib) + +include(FetchContent) + +FetchContent_Declare( + googletest + URL https://github.com/google/googletest/archive/refs/tags/v1.15.2.zip + DOWNLOAD_EXTRACT_TIMESTAMP TRUE +) + +FetchContent_MakeAvailable(googletest) + +enable_testing() + +add_executable(rocksdb_old_ts_test + rocksdb_test.cc + rocksdb_factory.cpp +) + + +target_link_libraries(rocksdb_old_ts_test + gtest gtest_main + rocksdb + pthread +) + + +add_executable(rocksdb_load + rocksdb_load.cpp + rocksdb_factory.cpp +) + +target_link_libraries(rocksdb_load + gtest gtest_main + rocksdb + pthread +) + + +include(GoogleTest) +gtest_discover_tests(rocksdb_old_ts_test) +add_test(NAME rocksdb_old_ts_test_TESTS COMMAND rocksdb_old_ts_test) + diff --git a/index/rocksdb/config.h b/index/rocksdb/config.h new file mode 100644 index 0000000..29d144d --- /dev/null +++ b/index/rocksdb/config.h @@ -0,0 +1,45 @@ +// +// Created by antio2 on 2025/10/21. +// + +#ifndef ROCKSDB_BENCH_CONFIG_H +#define ROCKSDB_BENCH_CONFIG_H + +#pragma once +#include + +enum class ts_typt_t { + retina, + udt +}; + +struct Config1000 { + std::string db_path = "/tmp/test_rocksdb_perf"; + int thread_num = 16; + int id_range = 10000000; + int ops_per_thread = 1000000; + bool destroy_before_start = true; + ts_typt_t ts_type = ts_typt_t::retina; +}; + +struct Config1000_udt { + std::string db_path = "/tmp/test_rocksdb_udt_1000"; + int thread_num = 16; + int id_range = 10000000; + int ops_per_thread = 1000000; + bool destroy_before_start = true; + ts_typt_t ts_type = ts_typt_t::udt; +}; + +struct Config10 { + std::string db_path = "/tmp/test_rocksdb_perf_10"; + int thread_num = 16; + int id_range = 100000; + int ops_per_thread = 1000000; + bool destroy_before_start = true; + ts_typt_t ts_type = ts_typt_t::retina; +}; + +using Config = Config1000_udt; + +#endif //ROCKSDB_BENCH_CONFIG_H diff --git a/index/rocksdb/cpp.gitignore b/index/rocksdb/cpp.gitignore new file mode 100644 index 0000000..a5cccb3 --- /dev/null +++ b/index/rocksdb/cpp.gitignore @@ -0,0 +1,1047 @@ +##### Windows +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +##### Linux +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +##### MacOS +# General +.DS_Store +.AppleDouble +.LSOverride + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +##### Android +# Built application files +*.apk +*.ap_ +*.aab + +# Files for the ART/Dalvik VM +*.dex + +# Java class files +*.class + +# Generated files +bin/ +gen/ +out/ +# Uncomment the following line in case you need and you don't have the release build type files in your app +# release/ + +# Gradle files +.gradle/ +build/ + +# Local configuration file (sdk path, etc) +local.properties + +# Proguard folder generated by Eclipse +proguard/ + +# Log Files +*.log + +# Android Studio Navigation editor temp files +.navigation/ + +# Android Studio captures folder +captures/ + +# IntelliJ +*.iml +.idea/workspace.xml +.idea/tasks.xml +.idea/gradle.xml +.idea/assetWizardSettings.xml +.idea/dictionaries +.idea/libraries +# Android Studio 3 in .gitignore file. +.idea/caches +.idea/modules.xml +# Comment next line if keeping position of elements in Navigation Editor is relevant for you +.idea/navEditor.xml + +# Keystore files +# Uncomment the following lines if you do not want to check your keystore files in. +#*.jks +#*.keystore + +# External native build folder generated in Android Studio 2.2 and later +.externalNativeBuild + +# Google Services (e.g. APIs or Firebase) +# google-services.json + +# Freeline +freeline.py +freeline/ +freeline_project_description.json + +# fastlane +fastlane/report.xml +fastlane/Preview.html +fastlane/screenshots +fastlane/test_output +fastlane/readme.md + +# Version control +vcs.xml + +# lint +lint/intermediates/ +lint/generated/ +lint/outputs/ +lint/tmp/ +# lint/reports/ + +##### Backup +*.bak +*.gho +*.ori +*.orig +*.tmp + +##### GPG +secring.* + +##### Dropbox +# Dropbox settings and caches +.dropbox +.dropbox.attr +.dropbox.cache + +##### SynopsysVCS +# Waveform formats +*.vcd +*.vpd +*.evcd +*.fsdb + +# Default name of the simulation executable. A different name can be +# specified with this switch (the associated daidir database name is +# also taken from here): -o / +simv + +# Generated for Verilog and VHDL top configs +simv.daidir/ +simv.db.dir/ + +# Infrastructure necessary to co-simulate SystemC models with +# Verilog/VHDL models. An alternate directory may be specified with this +# switch: -Mdir= +csrc/ + +# Log file - the following switch allows to specify the file that will be +# used to write all messages from simulation: -l +*.log + +# Coverage results (generated with urg) and database location. The +# following switch can also be used: urg -dir .vdb +simv.vdb/ +urgReport/ + +# DVE and UCLI related files. +DVEfiles/ +ucli.key + +# When the design is elaborated for DirectC, the following file is created +# with declarations for C/C++ functions. +vc_hdrs.h + +##### SVN +.svn/ + +##### Mercurial +.hg/ +.hgignore +.hgsigs +.hgsub +.hgsubstate +.hgtags + +##### Bazaar +.bzr/ +.bzrignore + +##### CVS +/CVS/* +**/CVS/* +.cvsignore +*/.cvsignore + +##### TortoiseGit +# Project-level settings +/.tgitconfig + +##### PuTTY +# Private key +*.ppk + +##### Vim +# Swap +[._]*.s[a-v][a-z] +!*.svg # comment out if you don't need vector files +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist +*~ +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + +##### Emacs +# -*- mode: gitignore; -*- +*~ +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +*.elc +auto-save-list +tramp +.\#* + +# Org-mode +.org-id-locations +*_archive + +# flymake-mode +*_flymake.* + +# eshell files +/eshell/history +/eshell/lastdir + +# elpa packages +/elpa/ + +# reftex files +*.rel + +# AUCTeX auto folder +/auto/ + +# cask packages +.cask/ +dist/ + +# Flycheck +flycheck_*.el + +# server auth directory +/server/ + +# projectiles files +.projectile + +# directory configuration +.dir-locals.el + +# network security +/network-security.data + +##### SublimeText +# Cache files for Sublime Text +*.tmlanguage.cache +*.tmPreferences.cache +*.stTheme.cache + +# Workspace files are user-specific +*.sublime-workspace + +# Project files should be checked into the repository, unless a significant +# proportion of contributors will probably not be using Sublime Text +# *.sublime-project + +# SFTP configuration file +sftp-config.json +sftp-config-alt*.json + +# Package control specific files +Package Control.last-run +Package Control.ca-list +Package Control.ca-bundle +Package Control.system-ca-bundle +Package Control.cache/ +Package Control.ca-certs/ +Package Control.merged-ca-bundle +Package Control.user-ca-bundle +oscrypto-ca-bundle.crt +bh_unicode_properties.cache + +# Sublime-github package stores a github token in this file +# https://packagecontrol.io/packages/sublime-github +GitHub.sublime-settings + +##### Notepad++ +# Notepad++ backups # +*.bak + +##### TextMate +*.tmproj +*.tmproject +tmtags + +##### VisualStudioCode +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +# Local History for Visual Studio Code +.history/ + +##### NetBeans +**/nbproject/private/ +**/nbproject/Makefile-*.mk +**/nbproject/Package-*.bash +build/ +nbbuild/ +dist/ +nbdist/ +.nb-gradle/ + +##### JetBrains +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +##### Eclipse +.metadata +bin/ +tmp/ +*.tmp +*.bak +*.swp +*~.nib +local.properties +.settings/ +.loadpath +.recommenders + +# External tool builders +.externalToolBuilders/ + +# Locally stored "Eclipse launch configurations" +*.launch + +# PyDev specific (Python IDE for Eclipse) +*.pydevproject + +# CDT-specific (C/C++ Development Tooling) +.cproject + +# CDT- autotools +.autotools + +# Java annotation processor (APT) +.factorypath + +# PDT-specific (PHP Development Tools) +.buildpath + +# sbteclipse plugin +.target + +# Tern plugin +.tern-project + +# TeXlipse plugin +.texlipse + +# STS (Spring Tool Suite) +.springBeans + +# Code Recommenders +.recommenders/ + +# Annotation Processing +.apt_generated/ +.apt_generated_test/ + +# Scala IDE specific (Scala & Java development for Eclipse) +.cache-main +.scala_dependencies +.worksheet + +# Uncomment this line if you wish to ignore the project description file. +# Typically, this file would be tracked if it contains build/dependency configurations: +#.project + +##### Qt +# C++ objects and libs +*.slo +*.lo +*.o +*.a +*.la +*.lai +*.so +*.so.* +*.dll +*.dylib + +# Qt-es +object_script.*.Release +object_script.*.Debug +*_plugin_import.cpp +/.qmake.cache +/.qmake.stash +*.pro.user +*.pro.user.* +*.qbs.user +*.qbs.user.* +*.moc +moc_*.cpp +moc_*.h +qrc_*.cpp +ui_*.h +*.qmlc +*.jsc +Makefile* +*build-* +*.qm +*.prl + +# Qt unit tests +target_wrapper.* + +# QtCreator +*.autosave + +# QtCreator Qml +*.qmlproject.user +*.qmlproject.user.* + +# QtCreator CMake +CMakeLists.txt.user* + +# QtCreator 4.8< compilation database +compile_commands.json + +# QtCreator local machine specific files for imported projects +*creator.user* + +##### VisualStudio +##### VisualStudio +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. +## +## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore + +# User-specific files +*.rsuser +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Mono auto generated files +mono_crash.* + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +[Ww][Ii][Nn]32/ +[Aa][Rr][Mm]/ +[Aa][Rr][Mm]64/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ +[Ll]ogs/ + +# Visual Studio 2015/2017 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# Visual Studio 2017 auto generated files +Generated\ Files/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUnit +*.VisualState.xml +TestResult.xml +nunit-*.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# Benchmark Results +BenchmarkDotNet.Artifacts/ + +# .NET Core +project.lock.json +project.fragment.lock.json +artifacts/ + +# ASP.NET Scaffolding +ScaffoldingReadMe.txt + +# StyleCop +StyleCopReport.xml + +# Files built by Visual Studio +*_i.c +*_p.c +*_h.h +*.ilk +*.meta +*.obj +*.iobj +*.pch +*.pdb +*.ipdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*_wpftmp.csproj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# Visual Studio Trace Files +*.e2e + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# AxoCover is a Code Coverage Tool +.axoCover/* +!.axoCover/settings.json + +# Coverlet is a free, cross platform Code Coverage Tool +coverage*[.json, .xml, .info] + +# Visual Studio code coverage results +*.coverage +*.coveragexml + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# Note: Comment the next line if you want to checkin your web deploy settings, +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# NuGet Symbol Packages +*.snupkg +# The packages folder can be ignored because of Package Restore +**/[Pp]ackages/* +# except build/, which is used as an MSBuild target. +!**/[Pp]ackages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/[Pp]ackages/repositories.config +# NuGet v3's project.json files produces more ignorable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt +*.appx +*.appxbundle +*.appxupload + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!?*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +orleans.codegen.cs + +# Including strong name files can present a security risk +# (https://github.com/github/gitignore/pull/2483#issue-259490424) +#*.snk + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm +ServiceFabricBackup/ +*.rptproj.bak + +# SQL Server files +*.mdf +*.ldf +*.ndf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings +*.rptproj.rsuser +*- [Bb]ackup.rdl +*- [Bb]ackup ([0-9]).rdl +*- [Bb]ackup ([0-9][0-9]).rdl + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat +node_modules/ + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) +*.vbw + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# CodeRush personal settings +.cr/personal + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Cake - Uncomment if you are using it +# tools/** +# !tools/packages.config + +# Tabs Studio +*.tss + +# Telerik's JustMock configuration file +*.jmconfig + +# BizTalk build output +*.btp.cs +*.btm.cs +*.odx.cs +*.xsd.cs + +# OpenCover UI analysis results +OpenCover/ + +# Azure Stream Analytics local run output +ASALocalRun/ + +# MSBuild Binary and Structured Log +*.binlog + +# NVidia Nsight GPU debugger configuration file +*.nvuser + +# MFractors (Xamarin productivity tool) working folder +.mfractor/ + +# Local History for Visual Studio +.localhistory/ + +# BeatPulse healthcheck temp database +healthchecksdb + +# Backup folder for Package Reference Convert tool in Visual Studio 2017 +MigrationBackup/ + +# Ionide (cross platform F# VS Code tools) working folder +.ionide/ + +# Fody - auto-generated XML schema +FodyWeavers.xsd + +##### Gradle +.gradle +**/build/ +!src/**/build/ + +# Ignore Gradle GUI config +gradle-app.setting + +# Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) +!gradle-wrapper.jar + +# Cache of project +.gradletasknamecache + +# # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 +# gradle/wrapper/gradle-wrapper.properties + +##### CMake +CMakeLists.txt.user +CMakeCache.txt +CMakeFiles +CMakeScripts +Testing +Makefile +cmake_install.cmake +install_manifest.txt +compile_commands.json +CTestTestfile.cmake +_deps + +##### C++ +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +# C/C++ binary extension file +*.bin + +##### C +# Prerequisites +*.d + +# Object files +*.o +*.ko +*.obj +*.elf + +# Linker output +*.ilk +*.map +*.exp + +# Precompiled Headers +*.gch +*.pch + +# Libraries +*.lib +*.a +*.la +*.lo + +# Shared objects (inc. Windows DLLs) +*.dll +*.so +*.so.* +*.dylib + +# Executables +*.exe +*.out +*.app +*.i*86 +*.x86_64 +*.hex + +# Debug files +*.dSYM/ +*.su +*.idb +*.pdb + +# Kernel Module Compile Results +*.mod* +*.cmd +.tmp_versions/ +modules.order +Module.symvers +Mkfile.old +dkms.conf + +# Raspberry Pi Pico Object file +*.uf2 +# Raspberry Pi Pico disassembler file +*.dis +cmake-build-debug/** +cmake-build-release/** \ No newline at end of file diff --git a/index/rocksdb/rocksdb_factory.cpp b/index/rocksdb/rocksdb_factory.cpp new file mode 100644 index 0000000..56a0736 --- /dev/null +++ b/index/rocksdb/rocksdb_factory.cpp @@ -0,0 +1,56 @@ +// +// Created by antio2 on 2025/10/21. +// + +#include "rocksdb_factory.h" +#pragma once +#include +#include +#include +#include +#include +#include +#include + +using namespace rocksdb; + +std::unique_ptr createRocksDB( + const std::string& dbPath, + std::vector>& handles) +{ + std::vector existingColumnFamilies; + Status s = DB::ListColumnFamilies(DBOptions(), dbPath, &existingColumnFamilies); + + if (!s.ok()) { + existingColumnFamilies = {kDefaultColumnFamilyName}; + } + + bool hasDefault = false; + for (const auto& n : existingColumnFamilies) + if (n == kDefaultColumnFamilyName) hasDefault = true; + if (!hasDefault) + existingColumnFamilies.push_back(kDefaultColumnFamilyName); + + std::vector descriptors; + for (const auto& name : existingColumnFamilies) { + descriptors.emplace_back(name, ColumnFamilyOptions()); + } + + DBOptions dbOptions; + dbOptions.create_if_missing = true; + dbOptions.create_missing_column_families = true; + + DB* db = nullptr; + std::vector rawHandles; + s = DB::Open(dbOptions, dbPath, descriptors, &rawHandles, &db); + if (!s.ok()) { + throw std::runtime_error("Failed to open RocksDB: " + s.ToString()); + } + + handles.clear(); + for (auto* h : rawHandles) + handles.emplace_back(h); + + std::cout << "Opened RocksDB: " << dbPath << ", CF=" << handles.size() << std::endl; + return std::unique_ptr(db); +} diff --git a/index/rocksdb/rocksdb_factory.h b/index/rocksdb/rocksdb_factory.h new file mode 100644 index 0000000..3bfda43 --- /dev/null +++ b/index/rocksdb/rocksdb_factory.h @@ -0,0 +1,17 @@ +// +// Created by antio2 on 2025/10/21. +// + +#ifndef ROCKSDB_BENCH_ROCKSDB_FACTORY_H +#define ROCKSDB_BENCH_ROCKSDB_FACTORY_H + + +#include +#include +#include + +std::unique_ptr createRocksDB( + const std::string& dbPath, + std::vector>& handles); + +#endif //ROCKSDB_BENCH_ROCKSDB_FACTORY_H diff --git a/index/rocksdb/rocksdb_load.cpp b/index/rocksdb/rocksdb_load.cpp new file mode 100644 index 0000000..6213652 --- /dev/null +++ b/index/rocksdb/rocksdb_load.cpp @@ -0,0 +1,82 @@ +// +// Created by antio2 on 2025/10/21. +// +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "config.h" +#include "rocksdb_factory.h" + +using namespace rocksdb; + +class RocksDBSequentialLoadTest : public ::testing::Test { +protected: + Config cfg; + std::unique_ptr db; + std::vector> handles; + + void SetUp() override { + Options options; + options.create_if_missing = true; + + if (cfg.destroy_before_start) { + DestroyDB(cfg.db_path, options); + } + + db = createRocksDB(cfg.db_path, handles); + } + + void TearDown() override { + handles.clear(); + db.reset(); + } + + static std::string encodeKey(int indexId, int key, long long timestamp) { + std::string s(4 + 4 + 8, '\0'); + memcpy(&s[0], &indexId, 4); + memcpy(&s[4], &key, 4); + memcpy(&s[8], ×tamp, 8); + return s; + } + + static std::string encodeValue(long long rowId) { + std::string s(8, '\0'); + memcpy(&s[0], &rowId, 8); + return s; + } + + void fillSequentialData() { + std::vector threads; + std::atomic counter{0}; + long long timestamp = 1000000; + + for (int t = 0; t < cfg.thread_num; ++t) { + threads.emplace_back([this, t, &counter, timestamp]() { + WriteOptions wopt; + for (int key = 0; key < cfg.id_range; ++key) { + std::string k = encodeKey(t, key, timestamp); + std::string v = encodeValue(key); + db->Put(wopt, k, v); + counter.fetch_add(1, std::memory_order_relaxed); + } + }); + } + + for (auto& th : threads) th.join(); + std::cout << "Total loaded: " << counter.load() << " entries" << std::endl; + } +}; + +TEST_F(RocksDBSequentialLoadTest, SequentialLoad) { + fillSequentialData(); +} \ No newline at end of file diff --git a/index/rocksdb/rocksdb_test.cc b/index/rocksdb/rocksdb_test.cc new file mode 100644 index 0000000..7c48f17 --- /dev/null +++ b/index/rocksdb/rocksdb_test.cc @@ -0,0 +1,109 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "config.h" +#include "rocksdb_factory.h" + +using namespace rocksdb; + +class RocksDBPerfTest : public ::testing::Test { +protected: + Config cfg; + std::vector> handles; + std::unique_ptr db; + void SetUp() override { + Options options; + options.create_if_missing = true; + + if (cfg.destroy_before_start) + DestroyDB(cfg.db_path, options); + + db = createRocksDB(cfg.db_path, handles); + } + + void TearDown() override { + handles.clear(); + db.reset(); + } + + static std::string encodeKey(int indexId, int key, long long timestamp) { + std::string s; + s.resize(4 + 4 + 8); + memcpy(&s[0], &indexId, 4); + memcpy(&s[4], &key, 4); + memcpy(&s[8], ×tamp, 8); + return s; + } + + static std::string encodeValue(long long rowId) { + std::string s(8, '\0'); + memcpy(&s[0], &rowId, 8); + return s; + } + + long long getUniqueRowId(int indexId, int key, long long timestamp) { + ReadOptions ropt; + std::unique_ptr it(db->NewIterator(ropt)); + std::string seekKey = encodeKey(indexId, key, timestamp); + it->SeekForPrev(seekKey); + + if (!it->Valid()) { + return -1; + } + + const Slice val = it->value(); + long long rowId; + memcpy(&rowId, val.data(), sizeof(rowId)); + return rowId; + } +public: + void threadWorker(int indexId, std::atomic& counter) { + std::mt19937_64 rng(std::random_device{}()); + WriteOptions wopt; + + for (int i = 0; i < cfg.ops_per_thread; ++i) { + int key = i % cfg.id_range; + long long ts = std::chrono::system_clock::now().time_since_epoch().count(); + long long oldRow = getUniqueRowId(indexId, key, ts); + long long newRow = (oldRow == -1) ? rng() : oldRow + 1; + + std::string k = encodeKey(indexId, key, ts); + std::string v = encodeValue(newRow); + db->Put(wopt, k, v); + + counter.fetch_add(1, std::memory_order_relaxed); + } + } +}; + +TEST_F(RocksDBPerfTest, MultiThreadPerf) +{ + std::cerr << "Start update test" << std::endl; + std::atomic totalOps = 0; + auto start = std::chrono::high_resolution_clock::now(); + + std::vector threads; + for (int i = 0; i < cfg.thread_num; ++i) { + threads.emplace_back(&RocksDBPerfTest::threadWorker, this, i, std::ref(totalOps)); + } + + for (auto& t : threads) t.join(); + + auto end = std::chrono::high_resolution_clock::now(); + double sec = std::chrono::duration(end - start).count(); + + std::cout << "Total ops: " << totalOps + << ", time: " << sec << "s, throughput: " + << (totalOps / sec) << " ops/s" << std::endl; + + ASSERT_GT(totalOps, 0); +} From dcf687e3a2e1b635f34eb68defea5284217aebc6 Mon Sep 17 00:00:00 2001 From: AntiO2 Date: Tue, 21 Oct 2025 21:07:54 +0800 Subject: [PATCH 2/9] UDT Test --- index/rocksdb/CMakeLists.txt | 1 + index/rocksdb/config.h | 25 +++++++++++------ index/rocksdb/rocksdb_factory.cpp | 28 +++++++++++++------ index/rocksdb/rocksdb_factory.h | 4 +-- index/rocksdb/rocksdb_load.cpp | 45 ++++++++++++++++++++----------- index/rocksdb/rocksdb_test.cc | 45 ++++++++++++++++++++----------- index/rocksdb/util.h | 16 +++++++++++ 7 files changed, 116 insertions(+), 48 deletions(-) create mode 100644 index/rocksdb/util.h diff --git a/index/rocksdb/CMakeLists.txt b/index/rocksdb/CMakeLists.txt index 37f42ff..b01c5ed 100644 --- a/index/rocksdb/CMakeLists.txt +++ b/index/rocksdb/CMakeLists.txt @@ -3,6 +3,7 @@ project(rocksdb_bench LANGUAGES CXX) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_FLAGS "-O2 -pthread") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -frtti") # RocksDB include_directories(/usr/local/include) diff --git a/index/rocksdb/config.h b/index/rocksdb/config.h index 29d144d..19996fa 100644 --- a/index/rocksdb/config.h +++ b/index/rocksdb/config.h @@ -8,7 +8,7 @@ #pragma once #include -enum class ts_typt_t { +enum class ts_type_t { retina, udt }; @@ -19,7 +19,7 @@ struct Config1000 { int id_range = 10000000; int ops_per_thread = 1000000; bool destroy_before_start = true; - ts_typt_t ts_type = ts_typt_t::retina; + ts_type_t ts_type = ts_type_t::retina; }; struct Config1000_udt { @@ -28,18 +28,27 @@ struct Config1000_udt { int id_range = 10000000; int ops_per_thread = 1000000; bool destroy_before_start = true; - ts_typt_t ts_type = ts_typt_t::udt; + ts_type_t ts_type = ts_type_t::udt; }; -struct Config10 { - std::string db_path = "/tmp/test_rocksdb_perf_10"; +struct Config1_retina { + std::string db_path = "/tmp/test_rocksdb_perf_retina_0.1"; int thread_num = 16; - int id_range = 100000; + int id_range = 1000; int ops_per_thread = 1000000; bool destroy_before_start = true; - ts_typt_t ts_type = ts_typt_t::retina; + ts_type_t ts_type = ts_type_t::retina; }; -using Config = Config1000_udt; +struct Config1_udt { + std::string db_path = "/tmp/test_rocksdb_perf_udt_0.1"; + int thread_num = 16; + int id_range = 1000; + int ops_per_thread = 1000000; + bool destroy_before_start = true; + ts_type_t ts_type = ts_type_t::udt; +}; + +using Config = Config1_udt; #endif //ROCKSDB_BENCH_CONFIG_H diff --git a/index/rocksdb/rocksdb_factory.cpp b/index/rocksdb/rocksdb_factory.cpp index 56a0736..2492908 100644 --- a/index/rocksdb/rocksdb_factory.cpp +++ b/index/rocksdb/rocksdb_factory.cpp @@ -2,7 +2,8 @@ // Created by antio2 on 2025/10/21. // -#include "rocksdb_factory.h" + + #pragma once #include #include @@ -11,15 +12,17 @@ #include #include #include - +#include "rocksdb_factory.h" +#include "config.h" +#include using namespace rocksdb; std::unique_ptr createRocksDB( - const std::string& dbPath, + const Config& cfg, std::vector>& handles) { std::vector existingColumnFamilies; - Status s = DB::ListColumnFamilies(DBOptions(), dbPath, &existingColumnFamilies); + Status s = DB::ListColumnFamilies(DBOptions(), cfg.db_path, &existingColumnFamilies); if (!s.ok()) { existingColumnFamilies = {kDefaultColumnFamilyName}; @@ -31,9 +34,15 @@ std::unique_ptr createRocksDB( if (!hasDefault) existingColumnFamilies.push_back(kDefaultColumnFamilyName); + // 根据 ts_type 设置 column family comparator std::vector descriptors; for (const auto& name : existingColumnFamilies) { - descriptors.emplace_back(name, ColumnFamilyOptions()); + ColumnFamilyOptions cfOptions; + if (cfg.ts_type == ts_type_t::udt) { + cfOptions.comparator = rocksdb::BytewiseComparatorWithU64Ts(); + } else { + } + descriptors.emplace_back(name, cfOptions); } DBOptions dbOptions; @@ -42,7 +51,7 @@ std::unique_ptr createRocksDB( DB* db = nullptr; std::vector rawHandles; - s = DB::Open(dbOptions, dbPath, descriptors, &rawHandles, &db); + s = DB::Open(dbOptions, cfg.db_path, descriptors, &rawHandles, &db); if (!s.ok()) { throw std::runtime_error("Failed to open RocksDB: " + s.ToString()); } @@ -51,6 +60,9 @@ std::unique_ptr createRocksDB( for (auto* h : rawHandles) handles.emplace_back(h); - std::cout << "Opened RocksDB: " << dbPath << ", CF=" << handles.size() << std::endl; + std::cout << "Opened RocksDB: " << cfg.db_path + << ", CF=" << handles.size() + << ", ts_type=" << (cfg.ts_type == ts_type_t::udt ? "UDT" : "Retina") << std::endl; + return std::unique_ptr(db); -} +} \ No newline at end of file diff --git a/index/rocksdb/rocksdb_factory.h b/index/rocksdb/rocksdb_factory.h index 3bfda43..6535795 100644 --- a/index/rocksdb/rocksdb_factory.h +++ b/index/rocksdb/rocksdb_factory.h @@ -9,9 +9,9 @@ #include #include #include - +#include "config.h" std::unique_ptr createRocksDB( - const std::string& dbPath, + const Config& cfg, std::vector>& handles); #endif //ROCKSDB_BENCH_ROCKSDB_FACTORY_H diff --git a/index/rocksdb/rocksdb_load.cpp b/index/rocksdb/rocksdb_load.cpp index 6213652..8da1eec 100644 --- a/index/rocksdb/rocksdb_load.cpp +++ b/index/rocksdb/rocksdb_load.cpp @@ -1,6 +1,3 @@ -// -// Created by antio2 on 2025/10/21. -// #include #include #include @@ -16,6 +13,7 @@ #include "config.h" #include "rocksdb_factory.h" +#include "util.h" using namespace rocksdb; @@ -33,7 +31,8 @@ class RocksDBSequentialLoadTest : public ::testing::Test { DestroyDB(cfg.db_path, options); } - db = createRocksDB(cfg.db_path, handles); + // Open DB with ts_type-aware factory + db = createRocksDB(cfg, handles); } void TearDown() override { @@ -41,12 +40,22 @@ class RocksDBSequentialLoadTest : public ::testing::Test { db.reset(); } - static std::string encodeKey(int indexId, int key, long long timestamp) { - std::string s(4 + 4 + 8, '\0'); - memcpy(&s[0], &indexId, 4); - memcpy(&s[4], &key, 4); - memcpy(&s[8], ×tamp, 8); - return s; + // Encode key based on timestamp type + static std::string encodeKey(int indexId, int key, long long timestamp, ts_type_t ts_type) { + if (ts_type == ts_type_t::retina) { + std::string s(4 + 4 + 8, '\0'); + memcpy(&s[0], &indexId, 4); + memcpy(&s[4], &key, 4); + memcpy(&s[8], ×tamp, 8); + return s; + } else if (ts_type == ts_type_t::udt) { + // UDT key format: indexId(4) + key(4) + timestamp_udt(8) + std::string s(4 + 4, '\0'); + memcpy(&s[0], &indexId, 4); + memcpy(&s[4], &key, 4); + return s; + } + return {}; } static std::string encodeValue(long long rowId) { @@ -55,18 +64,24 @@ class RocksDBSequentialLoadTest : public ::testing::Test { return s; } + // Sequentially fill DB from 0 to max-1 void fillSequentialData() { std::vector threads; std::atomic counter{0}; - long long timestamp = 1000000; - + long long timestamp = 0; + std::string tsBuffer; + const Slice& tsSlice = longToSlice(timestamp, tsBuffer); for (int t = 0; t < cfg.thread_num; ++t) { - threads.emplace_back([this, t, &counter, timestamp]() { + threads.emplace_back([this, t, &counter, timestamp, &tsSlice]() { WriteOptions wopt; for (int key = 0; key < cfg.id_range; ++key) { - std::string k = encodeKey(t, key, timestamp); + std::string k = encodeKey(t, key, timestamp, cfg.ts_type); std::string v = encodeValue(key); - db->Put(wopt, k, v); + if (cfg.ts_type == ts_type_t::retina) { + db->Put(wopt, k, v); + } else { + db->Put(wopt, k, tsSlice, v); + } counter.fetch_add(1, std::memory_order_relaxed); } }); diff --git a/index/rocksdb/rocksdb_test.cc b/index/rocksdb/rocksdb_test.cc index 7c48f17..52b4cc6 100644 --- a/index/rocksdb/rocksdb_test.cc +++ b/index/rocksdb/rocksdb_test.cc @@ -26,8 +26,7 @@ class RocksDBPerfTest : public ::testing::Test { if (cfg.destroy_before_start) DestroyDB(cfg.db_path, options); - - db = createRocksDB(cfg.db_path, handles); + db = createRocksDB(cfg, handles); } void TearDown() override { @@ -52,23 +51,30 @@ class RocksDBPerfTest : public ::testing::Test { long long getUniqueRowId(int indexId, int key, long long timestamp) { ReadOptions ropt; - std::unique_ptr it(db->NewIterator(ropt)); - std::string seekKey = encodeKey(indexId, key, timestamp); - it->SeekForPrev(seekKey); - - if (!it->Valid()) { - return -1; + std::string val_str; + + if (cfg.ts_type == ts_type_t::udt) { + // Convert timestamp to Slice + std::string ts_buf; + Slice tsSlice = EncodeU64Ts(timestamp, &ts_buf); // helper function + ropt.timestamp = &tsSlice; + std::string k = encodeKey(indexId, key, 0); // key part only + Status s = db->Get(ropt, k, &val_str); + if (!s.ok()) return -1; + } else { + std::string k = encodeKey(indexId, key, timestamp); + Status s = db->Get(ropt, k, &val_str); + if (!s.ok()) return -1; } - const Slice val = it->value(); long long rowId; - memcpy(&rowId, val.data(), sizeof(rowId)); + memcpy(&rowId, val_str.data(), sizeof(rowId)); return rowId; } + public: void threadWorker(int indexId, std::atomic& counter) { std::mt19937_64 rng(std::random_device{}()); - WriteOptions wopt; for (int i = 0; i < cfg.ops_per_thread; ++i) { int key = i % cfg.id_range; @@ -76,9 +82,18 @@ class RocksDBPerfTest : public ::testing::Test { long long oldRow = getUniqueRowId(indexId, key, ts); long long newRow = (oldRow == -1) ? rng() : oldRow + 1; - std::string k = encodeKey(indexId, key, ts); - std::string v = encodeValue(newRow); - db->Put(wopt, k, v); + WriteOptions wopt; + if (cfg.ts_type == ts_type_t::udt) { + std::string ts_buf; + Slice tsSlice = EncodeU64Ts(ts, &ts_buf); + std::string k = encodeKey(indexId, key, 0); // key without timestamp + std::string v = encodeValue(newRow); + db->Put(wopt, k, tsSlice, v); + } else { + std::string k = encodeKey(indexId, key, ts); + std::string v = encodeValue(newRow); + db->Put(wopt, k, v); + } counter.fetch_add(1, std::memory_order_relaxed); } @@ -103,7 +118,7 @@ TEST_F(RocksDBPerfTest, MultiThreadPerf) std::cout << "Total ops: " << totalOps << ", time: " << sec << "s, throughput: " - << (totalOps / sec) << " ops/s" << std::endl; + << (long long)(totalOps / sec) << " ops/s" << std::endl; ASSERT_GT(totalOps, 0); } diff --git a/index/rocksdb/util.h b/index/rocksdb/util.h new file mode 100644 index 0000000..f9a36aa --- /dev/null +++ b/index/rocksdb/util.h @@ -0,0 +1,16 @@ +// +// Created by antio2 on 2025/10/21. +// + +#ifndef ROCKSDB_BENCH_UTIL_H +#define ROCKSDB_BENCH_UTIL_H +#include +#include + +inline rocksdb::Slice longToSlice(long long value, std::string& buf) { + buf.resize(sizeof(long long)); // make sure buffer has 8 bytes + memcpy(&buf[0], &value, sizeof(long long)); // serialize value into buffer + return rocksdb::Slice(buf); +} + +#endif //ROCKSDB_BENCH_UTIL_H From b7c01c94968168cf5d484386609e62aa387a72a6 Mon Sep 17 00:00:00 2001 From: AntiO2 Date: Tue, 21 Oct 2025 14:07:44 +0000 Subject: [PATCH 3/9] fix cmake --- index/rocksdb/CMakeLists.txt | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/index/rocksdb/CMakeLists.txt b/index/rocksdb/CMakeLists.txt index b01c5ed..2801d47 100644 --- a/index/rocksdb/CMakeLists.txt +++ b/index/rocksdb/CMakeLists.txt @@ -21,32 +21,41 @@ FetchContent_MakeAvailable(googletest) enable_testing() -add_executable(rocksdb_old_ts_test +add_executable(rocksdb_ts_test rocksdb_test.cc rocksdb_factory.cpp ) +add_executable(rocksdb_load + rocksdb_load.cpp + rocksdb_factory.cpp +) -target_link_libraries(rocksdb_old_ts_test - gtest gtest_main +target_link_libraries(rocksdb_ts_test + gtest + gtest_main rocksdb pthread -) - + z + snappy + lz4 + zstd -add_executable(rocksdb_load - rocksdb_load.cpp - rocksdb_factory.cpp ) target_link_libraries(rocksdb_load - gtest gtest_main + gtest + gtest_main rocksdb pthread -) + z + snappy + lz4 + zstd +) include(GoogleTest) -gtest_discover_tests(rocksdb_old_ts_test) -add_test(NAME rocksdb_old_ts_test_TESTS COMMAND rocksdb_old_ts_test) +gtest_discover_tests(rocksdb_ts_test) +add_test(NAME rocksdb_ts_test_TESTS COMMAND rocksdb_old_ts_test) From ee9c31fe0c50a813d175a38fa8b590261772d57c Mon Sep 17 00:00:00 2001 From: AntiO2 Date: Wed, 22 Oct 2025 17:26:38 +0800 Subject: [PATCH 4/9] Config Preset --- index/rocksdb/config.h | 92 +++++++++++------- index/rocksdb/rocksdb_factory.cpp | 22 ++++- index/rocksdb/rocksdb_factory.h | 3 + index/rocksdb/rocksdb_load.cpp | 90 +++++++++--------- index/rocksdb/rocksdb_test.cc | 153 +++++++++++++++++------------- index/rocksdb/util.h | 32 +++++++ 6 files changed, 244 insertions(+), 148 deletions(-) diff --git a/index/rocksdb/config.h b/index/rocksdb/config.h index 19996fa..b38ff32 100644 --- a/index/rocksdb/config.h +++ b/index/rocksdb/config.h @@ -9,46 +9,72 @@ #include enum class ts_type_t { - retina, + embed_asc, + embed_desc, udt }; +inline std::ostream& operator<<(std::ostream& os, ts_type_t t) { + switch (t) { + case ts_type_t::embed_asc: os << "embed_asc"; break; + case ts_type_t::embed_desc: os << "embed_desc"; break; + case ts_type_t::udt: os << "udt"; break; + default: os << "unknown"; break; + } + return os; +} -struct Config1000 { - std::string db_path = "/tmp/test_rocksdb_perf"; - int thread_num = 16; - int id_range = 10000000; - int ops_per_thread = 1000000; - bool destroy_before_start = true; - ts_type_t ts_type = ts_type_t::retina; +enum class Preset { + Udt10m, + Asc10m, + Desc10m, + Udt1k, + Asc1k, + Desc1k }; -struct Config1000_udt { - std::string db_path = "/tmp/test_rocksdb_udt_1000"; - int thread_num = 16; - int id_range = 10000000; - int ops_per_thread = 1000000; - bool destroy_before_start = true; - ts_type_t ts_type = ts_type_t::udt; -}; +struct Config { +public: + using scale_t = int; -struct Config1_retina { - std::string db_path = "/tmp/test_rocksdb_perf_retina_0.1"; - int thread_num = 16; - int id_range = 1000; - int ops_per_thread = 1000000; - bool destroy_before_start = true; - ts_type_t ts_type = ts_type_t::retina; -}; + const std::string db_path_prefix = "/tmp/rocksdb"; // hard code + std::string db_path_; + int thread_num_; + int id_range_; + int ops_per_thread_; + bool destroy_before_start_; + ts_type_t ts_type_; -struct Config1_udt { - std::string db_path = "/tmp/test_rocksdb_perf_udt_0.1"; - int thread_num = 16; - int id_range = 1000; - int ops_per_thread = 1000000; - bool destroy_before_start = true; - ts_type_t ts_type = ts_type_t::udt; -}; + Config() = delete; -using Config = Config1_udt; + explicit Config(Preset preset) + : Config(getScaleAndType(preset)) + {} + Config(scale_t scale, ts_type_t ts_type) { + thread_num_ = 16; + ops_per_thread_ = 1000000; + id_range_ = scale; + destroy_before_start_ = true; + ts_type_ = ts_type; + std::ostringstream oss; + oss << "/tmp/rocksdb/" << "test_" << id_range_ << "_" << ts_type_; + db_path_ = oss.str(); + } +private: + static std::pair getScaleAndType(Preset preset) { + switch (preset) { + case Preset::Udt10m: return {10000000, ts_type_t::udt}; + case Preset::Asc10m: return {10000000, ts_type_t::embed_asc}; + case Preset::Desc10m: return {10000000, ts_type_t::embed_desc}; + case Preset::Udt1k: return {1000, ts_type_t::udt}; + case Preset::Asc1k: return {1000, ts_type_t::embed_asc}; + case Preset::Desc1k: return {1000, ts_type_t::embed_desc}; + } + return {0, ts_type_t::embed_asc}; + } + + explicit Config(const std::pair& p) + : Config(p.first, p.second) + {} +}; #endif //ROCKSDB_BENCH_CONFIG_H diff --git a/index/rocksdb/rocksdb_factory.cpp b/index/rocksdb/rocksdb_factory.cpp index 2492908..579ff45 100644 --- a/index/rocksdb/rocksdb_factory.cpp +++ b/index/rocksdb/rocksdb_factory.cpp @@ -17,12 +17,24 @@ #include using namespace rocksdb; +std::unique_ptr initRocksDB( + const Config& cfg, + std::vector>& handles) +{ + Options options; + options.create_if_missing = true; + + if (cfg.destroy_before_start_) + DestroyDB(cfg.db_path_, options); + return createRocksDB(cfg, handles); +} + std::unique_ptr createRocksDB( const Config& cfg, std::vector>& handles) { std::vector existingColumnFamilies; - Status s = DB::ListColumnFamilies(DBOptions(), cfg.db_path, &existingColumnFamilies); + Status s = DB::ListColumnFamilies(DBOptions(), cfg.db_path_, &existingColumnFamilies); if (!s.ok()) { existingColumnFamilies = {kDefaultColumnFamilyName}; @@ -38,7 +50,7 @@ std::unique_ptr createRocksDB( std::vector descriptors; for (const auto& name : existingColumnFamilies) { ColumnFamilyOptions cfOptions; - if (cfg.ts_type == ts_type_t::udt) { + if (cfg.ts_type_ == ts_type_t::udt) { cfOptions.comparator = rocksdb::BytewiseComparatorWithU64Ts(); } else { } @@ -51,7 +63,7 @@ std::unique_ptr createRocksDB( DB* db = nullptr; std::vector rawHandles; - s = DB::Open(dbOptions, cfg.db_path, descriptors, &rawHandles, &db); + s = DB::Open(dbOptions, cfg.db_path_, descriptors, &rawHandles, &db); if (!s.ok()) { throw std::runtime_error("Failed to open RocksDB: " + s.ToString()); } @@ -60,9 +72,9 @@ std::unique_ptr createRocksDB( for (auto* h : rawHandles) handles.emplace_back(h); - std::cout << "Opened RocksDB: " << cfg.db_path + std::cout << "Opened RocksDB: " << cfg.db_path_ << ", CF=" << handles.size() - << ", ts_type=" << (cfg.ts_type == ts_type_t::udt ? "UDT" : "Retina") << std::endl; + << ", ts_type=" << (cfg.ts_type_ == ts_type_t::udt ? "UDT" : "Retina") << std::endl; return std::unique_ptr(db); } \ No newline at end of file diff --git a/index/rocksdb/rocksdb_factory.h b/index/rocksdb/rocksdb_factory.h index 6535795..a1d0d72 100644 --- a/index/rocksdb/rocksdb_factory.h +++ b/index/rocksdb/rocksdb_factory.h @@ -10,6 +10,9 @@ #include #include #include "config.h" +std::unique_ptr initRocksDB( + const Config& cfg, + std::vector>& handles); std::unique_ptr createRocksDB( const Config& cfg, std::vector>& handles); diff --git a/index/rocksdb/rocksdb_load.cpp b/index/rocksdb/rocksdb_load.cpp index 8da1eec..d52b985 100644 --- a/index/rocksdb/rocksdb_load.cpp +++ b/index/rocksdb/rocksdb_load.cpp @@ -5,11 +5,8 @@ #include #include #include -#include -#include #include #include -#include #include "config.h" #include "rocksdb_factory.h" @@ -19,20 +16,15 @@ using namespace rocksdb; class RocksDBSequentialLoadTest : public ::testing::Test { protected: - Config cfg; + std::unique_ptr cfg; std::unique_ptr db; std::vector> handles; - void SetUp() override { - Options options; - options.create_if_missing = true; - - if (cfg.destroy_before_start) { - DestroyDB(cfg.db_path, options); - } - - // Open DB with ts_type-aware factory - db = createRocksDB(cfg, handles); + void test(Preset preset) + { + cfg = std::make_unique(preset); + OpenDb(); + fillSequentialData(); } void TearDown() override { @@ -40,30 +32,18 @@ class RocksDBSequentialLoadTest : public ::testing::Test { db.reset(); } - // Encode key based on timestamp type - static std::string encodeKey(int indexId, int key, long long timestamp, ts_type_t ts_type) { - if (ts_type == ts_type_t::retina) { - std::string s(4 + 4 + 8, '\0'); - memcpy(&s[0], &indexId, 4); - memcpy(&s[4], &key, 4); - memcpy(&s[8], ×tamp, 8); - return s; - } else if (ts_type == ts_type_t::udt) { - // UDT key format: indexId(4) + key(4) + timestamp_udt(8) - std::string s(4 + 4, '\0'); - memcpy(&s[0], &indexId, 4); - memcpy(&s[4], &key, 4); - return s; +private: + void OpenDb() { + Options options; + options.create_if_missing = true; + + if (cfg->destroy_before_start_) { + DestroyDB(cfg->db_path_, options); } - return {}; - } - static std::string encodeValue(long long rowId) { - std::string s(8, '\0'); - memcpy(&s[0], &rowId, 8); - return s; + // Open DB with ts_type-aware factory + db = createRocksDB(*cfg, handles); } - // Sequentially fill DB from 0 to max-1 void fillSequentialData() { std::vector threads; @@ -71,27 +51,47 @@ class RocksDBSequentialLoadTest : public ::testing::Test { long long timestamp = 0; std::string tsBuffer; const Slice& tsSlice = longToSlice(timestamp, tsBuffer); - for (int t = 0; t < cfg.thread_num; ++t) { + for (int t = 0; t < cfg->thread_num_; ++t) { threads.emplace_back([this, t, &counter, timestamp, &tsSlice]() { WriteOptions wopt; - for (int key = 0; key < cfg.id_range; ++key) { - std::string k = encodeKey(t, key, timestamp, cfg.ts_type); + for (int key = 0; key < cfg->id_range_; ++key) { + std::string k = encodeKey(*cfg, t, key, timestamp); std::string v = encodeValue(key); - if (cfg.ts_type == ts_type_t::retina) { - db->Put(wopt, k, v); - } else { + if (cfg->ts_type_ == ts_type_t::udt) { db->Put(wopt, k, tsSlice, v); + } else { + db->Put(wopt, k, v); } counter.fetch_add(1, std::memory_order_relaxed); } }); } - for (auto& th : threads) th.join(); std::cout << "Total loaded: " << counter.load() << " entries" << std::endl; } }; -TEST_F(RocksDBSequentialLoadTest, SequentialLoad) { - fillSequentialData(); -} \ No newline at end of file + +TEST_F(RocksDBSequentialLoadTest, SequentialLoadUdt10m) { + test(Preset::Udt10m); +} + +TEST_F(RocksDBSequentialLoadTest, SequentialLoadAsc10m) { + test(Preset::Asc10m); +} + +TEST_F(RocksDBSequentialLoadTest, SequentialLoadDesc10m) { + test(Preset::Desc10m); +} + +TEST_F(RocksDBSequentialLoadTest, SequentialLoadUdt1k) { + test(Preset::Udt1k); +} + +TEST_F(RocksDBSequentialLoadTest, SequentialLoadAsc1k) { + test(Preset::Asc1k); +} + +TEST_F(RocksDBSequentialLoadTest, SequentialLoadDesc1k) { + test(Preset::Desc1k); +} diff --git a/index/rocksdb/rocksdb_test.cc b/index/rocksdb/rocksdb_test.cc index 52b4cc6..c4e76f5 100644 --- a/index/rocksdb/rocksdb_test.cc +++ b/index/rocksdb/rocksdb_test.cc @@ -5,120 +5,143 @@ #include #include #include -#include #include #include #include #include "config.h" #include "rocksdb_factory.h" +#include "util.h" using namespace rocksdb; class RocksDBPerfTest : public ::testing::Test { protected: - Config cfg; + std::unique_ptr cfg; std::vector> handles; std::unique_ptr db; - void SetUp() override { - Options options; - options.create_if_missing = true; + void test(Preset preset) + { + cfg = std::make_unique(preset); + db = initRocksDB(*cfg, handles); + + std::cerr << "Start update test" << std::endl; + std::atomic totalOps = 0; + auto start = std::chrono::high_resolution_clock::now(); + + std::vector threads; + for (int i = 0; i < cfg->thread_num_; ++i) { + threads.emplace_back(&RocksDBPerfTest::threadWorker, this, i, std::ref(totalOps)); + } + + for (auto& t : threads) t.join(); + + auto end = std::chrono::high_resolution_clock::now(); + double sec = std::chrono::duration(end - start).count(); - if (cfg.destroy_before_start) - DestroyDB(cfg.db_path, options); - db = createRocksDB(cfg, handles); + std::cout << "Total ops: " << totalOps + << ", time: " << sec << "s, throughput: " + << (long long)(totalOps / sec) << " ops/s" << std::endl; + + ASSERT_GT(totalOps, 0); } void TearDown() override { handles.clear(); db.reset(); } - - static std::string encodeKey(int indexId, int key, long long timestamp) { - std::string s; - s.resize(4 + 4 + 8); - memcpy(&s[0], &indexId, 4); - memcpy(&s[4], &key, 4); - memcpy(&s[8], ×tamp, 8); - return s; - } - - static std::string encodeValue(long long rowId) { - std::string s(8, '\0'); - memcpy(&s[0], &rowId, 8); - return s; - } - + long long getUniqueRowId(int indexId, int key, long long timestamp) { ReadOptions ropt; std::string val_str; + std::string k = encodeKey(*cfg, indexId, key, timestamp); + switch (cfg->ts_type_) { + + case ts_type_t::embed_asc: + { + std::unique_ptr it(db->NewIterator(ropt)); + it->SeekForPrev(k); + if (!it->Valid()) return -1; + + const Slice val = it->value(); + long long rowId; + memcpy(&rowId, val.data(), sizeof(rowId)); + return rowId; + } + case ts_type_t::embed_desc: + { + std::unique_ptr it(db->NewIterator(ropt)); + it->Seek(k); + if (!it->Valid()) return -1; + const Slice val = it->value(); + long long rowId; + memcpy(&rowId, val.data(), sizeof(rowId)); + return rowId; + } - if (cfg.ts_type == ts_type_t::udt) { - // Convert timestamp to Slice - std::string ts_buf; - Slice tsSlice = EncodeU64Ts(timestamp, &ts_buf); // helper function - ropt.timestamp = &tsSlice; - std::string k = encodeKey(indexId, key, 0); // key part only - Status s = db->Get(ropt, k, &val_str); - if (!s.ok()) return -1; - } else { - std::string k = encodeKey(indexId, key, timestamp); - Status s = db->Get(ropt, k, &val_str); - if (!s.ok()) return -1; + case ts_type_t::udt: + { + std::string ts_buf; + Slice tsSlice = EncodeU64Ts(timestamp, &ts_buf); + ropt.timestamp = &tsSlice; + + Status s = db->Get(ropt, k, &val_str, &ts_buf); + if (!s.ok()) { + if (s.IsNotFound()) return -1; + std::cerr << "DB Get failed: " << s.ToString() << std::endl; + return -1; + } + long long rowId; + memcpy(&rowId, val_str.data(), sizeof(rowId)); + return rowId; + } } - - long long rowId; - memcpy(&rowId, val_str.data(), sizeof(rowId)); - return rowId; } public: void threadWorker(int indexId, std::atomic& counter) { std::mt19937_64 rng(std::random_device{}()); - for (int i = 0; i < cfg.ops_per_thread; ++i) { - int key = i % cfg.id_range; + for (int i = 0; i < cfg->ops_per_thread_; ++i) { + int key = i % cfg->id_range_; long long ts = std::chrono::system_clock::now().time_since_epoch().count(); long long oldRow = getUniqueRowId(indexId, key, ts); long long newRow = (oldRow == -1) ? rng() : oldRow + 1; - + std::string k = encodeKey(*cfg, indexId, key, ts); + std::string v = encodeValue(newRow); WriteOptions wopt; - if (cfg.ts_type == ts_type_t::udt) { + if (cfg->ts_type_ == ts_type_t::udt) { std::string ts_buf; Slice tsSlice = EncodeU64Ts(ts, &ts_buf); - std::string k = encodeKey(indexId, key, 0); // key without timestamp - std::string v = encodeValue(newRow); db->Put(wopt, k, tsSlice, v); } else { - std::string k = encodeKey(indexId, key, ts); - std::string v = encodeValue(newRow); db->Put(wopt, k, v); } - counter.fetch_add(1, std::memory_order_relaxed); } } }; -TEST_F(RocksDBPerfTest, MultiThreadPerf) -{ - std::cerr << "Start update test" << std::endl; - std::atomic totalOps = 0; - auto start = std::chrono::high_resolution_clock::now(); +TEST_F(RocksDBPerfTest, MultiThreadPerf_Udt10m) { + test(Preset::Udt10m); +} - std::vector threads; - for (int i = 0; i < cfg.thread_num; ++i) { - threads.emplace_back(&RocksDBPerfTest::threadWorker, this, i, std::ref(totalOps)); - } +TEST_F(RocksDBPerfTest, MultiThreadPerf_Asc10m) { + test(Preset::Asc10m); +} - for (auto& t : threads) t.join(); +TEST_F(RocksDBPerfTest, MultiThreadPerf_Desc10m) { + test(Preset::Desc10m); +} - auto end = std::chrono::high_resolution_clock::now(); - double sec = std::chrono::duration(end - start).count(); +TEST_F(RocksDBPerfTest, MultiThreadPerf_Udt1k) { + test(Preset::Udt1k); +} - std::cout << "Total ops: " << totalOps - << ", time: " << sec << "s, throughput: " - << (long long)(totalOps / sec) << " ops/s" << std::endl; +TEST_F(RocksDBPerfTest, MultiThreadPerf_Asc1k) { + test(Preset::Asc1k); +} - ASSERT_GT(totalOps, 0); +TEST_F(RocksDBPerfTest, MultiThreadPerf_Desc1k) { + test(Preset::Desc1k); } diff --git a/index/rocksdb/util.h b/index/rocksdb/util.h index f9a36aa..dd9d3f0 100644 --- a/index/rocksdb/util.h +++ b/index/rocksdb/util.h @@ -13,4 +13,36 @@ inline rocksdb::Slice longToSlice(long long value, std::string& buf) { return rocksdb::Slice(buf); } +std::string encodeKey(Config cfg, int indexId, int key, long long timestamp) { + ts_type_t ts_type = cfg.ts_type_; + if (ts_type == ts_type_t::embed_asc) { + std::string s(4 + 4 + 8, '\0'); + memcpy(&s[0], &indexId, 4); + memcpy(&s[4], &key, 4); + memcpy(&s[8], ×tamp, 8); + return s; + } else if (ts_type == ts_type_t::udt) { + // UDT key format: indexId(4) + key(4) + timestamp_udt(8) + std::string s(4 + 4, '\0'); + memcpy(&s[0], &indexId, 4); + memcpy(&s[4], &key, 4); + return s; + } else if (ts_type == ts_type_t::embed_desc) + { + long long new_ts = LONG_LONG_MAX - timestamp; + std::string s(4 + 4 + 8, '\0'); + memcpy(&s[0], &indexId, 4); + memcpy(&s[4], &key, 4); + memcpy(&s[8], &new_ts, 8); + return s; + } + return {}; +} + +std::string encodeValue(long long rowId) { + std::string s(8, '\0'); + memcpy(&s[0], &rowId, 8); + return s; +} + #endif //ROCKSDB_BENCH_UTIL_H From 678679c5e100884bec224125d4dd382caabbe248 Mon Sep 17 00:00:00 2001 From: AntiO2 Date: Wed, 22 Oct 2025 22:06:02 +0800 Subject: [PATCH 5/9] Use Bloom Filter --- index/rocksdb/CMakeLists.txt | 1 - index/rocksdb/rocksdb_factory.cpp | 18 +++++++++--- index/rocksdb/rocksdb_load.cpp | 46 ++++++++++++++++++++++++++++--- index/rocksdb/rocksdb_test.cc | 42 +++++++++++++++++++--------- index/rocksdb/util.h | 6 ---- 5 files changed, 85 insertions(+), 28 deletions(-) diff --git a/index/rocksdb/CMakeLists.txt b/index/rocksdb/CMakeLists.txt index 2801d47..c1bdf5a 100644 --- a/index/rocksdb/CMakeLists.txt +++ b/index/rocksdb/CMakeLists.txt @@ -2,7 +2,6 @@ cmake_minimum_required(VERSION 3.10) project(rocksdb_bench LANGUAGES CXX) set(CMAKE_CXX_STANDARD 20) -set(CMAKE_CXX_FLAGS "-O2 -pthread") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -frtti") # RocksDB diff --git a/index/rocksdb/rocksdb_factory.cpp b/index/rocksdb/rocksdb_factory.cpp index 579ff45..70023e3 100644 --- a/index/rocksdb/rocksdb_factory.cpp +++ b/index/rocksdb/rocksdb_factory.cpp @@ -15,6 +15,10 @@ #include "rocksdb_factory.h" #include "config.h" #include +#include +#include +#include + using namespace rocksdb; std::unique_ptr initRocksDB( @@ -23,9 +27,6 @@ std::unique_ptr initRocksDB( { Options options; options.create_if_missing = true; - - if (cfg.destroy_before_start_) - DestroyDB(cfg.db_path_, options); return createRocksDB(cfg, handles); } @@ -46,7 +47,6 @@ std::unique_ptr createRocksDB( if (!hasDefault) existingColumnFamilies.push_back(kDefaultColumnFamilyName); - // 根据 ts_type 设置 column family comparator std::vector descriptors; for (const auto& name : existingColumnFamilies) { ColumnFamilyOptions cfOptions; @@ -55,6 +55,16 @@ std::unique_ptr createRocksDB( } else { } descriptors.emplace_back(name, cfOptions); + + if(cfg.ts_type_ == ts_type_t::embed_desc) + { + rocksdb::BlockBasedTableOptions table_options; + table_options.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false)); + table_options.whole_key_filtering = false; + + cfOptions.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(8)); + cfOptions.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options)); + } } DBOptions dbOptions; diff --git a/index/rocksdb/rocksdb_load.cpp b/index/rocksdb/rocksdb_load.cpp index d52b985..8ff31b0 100644 --- a/index/rocksdb/rocksdb_load.cpp +++ b/index/rocksdb/rocksdb_load.cpp @@ -49,10 +49,11 @@ class RocksDBSequentialLoadTest : public ::testing::Test { std::vector threads; std::atomic counter{0}; long long timestamp = 0; - std::string tsBuffer; - const Slice& tsSlice = longToSlice(timestamp, tsBuffer); + std::string tsBuffer(8, '\0'); + const Slice& tsSlice = EncodeU64Ts(timestamp, &tsBuffer); for (int t = 0; t < cfg->thread_num_; ++t) { - threads.emplace_back([this, t, &counter, timestamp, &tsSlice]() { +// threads.emplace_back([this, t, &counter, timestamp, &tsSlice]() + { WriteOptions wopt; for (int key = 0; key < cfg->id_range_; ++key) { std::string k = encodeKey(*cfg, t, key, timestamp); @@ -64,7 +65,9 @@ class RocksDBSequentialLoadTest : public ::testing::Test { } counter.fetch_add(1, std::memory_order_relaxed); } - }); + } + +// ); } for (auto& th : threads) th.join(); std::cout << "Total loaded: " << counter.load() << " entries" << std::endl; @@ -95,3 +98,38 @@ TEST_F(RocksDBSequentialLoadTest, SequentialLoadAsc1k) { TEST_F(RocksDBSequentialLoadTest, SequentialLoadDesc1k) { test(Preset::Desc1k); } + +TEST(RocksDBSmallTest, UDT) { + Preset preset = Preset::Udt1k; + std::vector> handles; + auto cfg = std::make_unique(preset); + { + Options options; + options.create_if_missing = true; + if (cfg->destroy_before_start_) { + DestroyDB(cfg->db_path_, options); + } + } + + auto db = createRocksDB(*cfg, handles); + + std::string tsBuf = std::string(8, '\0'); + long long timestamp = 1761139362806988526; + auto ts = EncodeU64Ts(timestamp, &tsBuf); + + WriteOptions wopt; + std::string key = "abc"; + std::string value = "ray"; + db->Put(wopt, key, ts, value); + + ReadOptions ropt; + long long timestamp2 = 1761139362806988525; + auto ts2 = EncodeU64Ts(timestamp2, &tsBuf); + ropt.timestamp = &ts2; + std::string rvalue = std::string(8, '\0'); + auto s = db->Get(ropt, key, &rvalue); + s.AssertOK(); + std::cout << rvalue << std::endl; + handles.clear(); + db = nullptr; +} diff --git a/index/rocksdb/rocksdb_test.cc b/index/rocksdb/rocksdb_test.cc index c4e76f5..337df60 100644 --- a/index/rocksdb/rocksdb_test.cc +++ b/index/rocksdb/rocksdb_test.cc @@ -53,10 +53,10 @@ class RocksDBPerfTest : public ::testing::Test { long long getUniqueRowId(int indexId, int key, long long timestamp) { ReadOptions ropt; - std::string val_str; + std::string val_str(8, '\0'); std::string k = encodeKey(*cfg, indexId, key, timestamp); + std::string ts_buf(8, '\0'); switch (cfg->ts_type_) { - case ts_type_t::embed_asc: { std::unique_ptr it(db->NewIterator(ropt)); @@ -71,23 +71,38 @@ class RocksDBPerfTest : public ::testing::Test { case ts_type_t::embed_desc: { std::unique_ptr it(db->NewIterator(ropt)); - it->Seek(k); - if (!it->Valid()) return -1; - const Slice val = it->value(); - long long rowId; - memcpy(&rowId, val.data(), sizeof(rowId)); + Slice prefix(k.data(), 8); + it->Seek(prefix); + long long rowId = -1; + uint64_t bestTs = 0; + for (; it->Valid() && it->key().starts_with(prefix); it->Next()) { + const Slice curKey = it->key(); + if(curKey.size() != 16) + { + return -1; + } + uint64_t ts; + Slice tsSlice(curKey.data() + 8, 8); + DecodeU64Ts(tsSlice, &ts); + auto real_ts = LONG_LONG_MAX - ts; + if (real_ts < timestamp) { + const Slice val = it->value(); + memcpy(&rowId, val.data(), sizeof(rowId)); + break; + } + } return rowId; } - case ts_type_t::udt: { - std::string ts_buf; Slice tsSlice = EncodeU64Ts(timestamp, &ts_buf); ropt.timestamp = &tsSlice; - - Status s = db->Get(ropt, k, &val_str, &ts_buf); + Status s = db->Get(ropt, k, &val_str); if (!s.ok()) { - if (s.IsNotFound()) return -1; + if (s.IsNotFound()) + { + return -1; + } std::cerr << "DB Get failed: " << s.ToString() << std::endl; return -1; } @@ -104,8 +119,9 @@ class RocksDBPerfTest : public ::testing::Test { for (int i = 0; i < cfg->ops_per_thread_; ++i) { int key = i % cfg->id_range_; - long long ts = std::chrono::system_clock::now().time_since_epoch().count(); + uint64_t ts = std::chrono::system_clock::now().time_since_epoch().count(); long long oldRow = getUniqueRowId(indexId, key, ts); + ASSERT_GE(oldRow, 0); long long newRow = (oldRow == -1) ? rng() : oldRow + 1; std::string k = encodeKey(*cfg, indexId, key, ts); std::string v = encodeValue(newRow); diff --git a/index/rocksdb/util.h b/index/rocksdb/util.h index dd9d3f0..bca21c8 100644 --- a/index/rocksdb/util.h +++ b/index/rocksdb/util.h @@ -7,12 +7,6 @@ #include #include -inline rocksdb::Slice longToSlice(long long value, std::string& buf) { - buf.resize(sizeof(long long)); // make sure buffer has 8 bytes - memcpy(&buf[0], &value, sizeof(long long)); // serialize value into buffer - return rocksdb::Slice(buf); -} - std::string encodeKey(Config cfg, int indexId, int key, long long timestamp) { ts_type_t ts_type = cfg.ts_type_; if (ts_type == ts_type_t::embed_asc) { From d44c001c125e17885f3be7e51875bc9e5886e540 Mon Sep 17 00:00:00 2001 From: AntiO2 Date: Thu, 23 Oct 2025 16:36:29 +0800 Subject: [PATCH 6/9] Revert --- index/rocksdb/rocksdb_test.cc | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/index/rocksdb/rocksdb_test.cc b/index/rocksdb/rocksdb_test.cc index 337df60..8a6890f 100644 --- a/index/rocksdb/rocksdb_test.cc +++ b/index/rocksdb/rocksdb_test.cc @@ -70,27 +70,14 @@ class RocksDBPerfTest : public ::testing::Test { } case ts_type_t::embed_desc: { + ropt.prefix_same_as_start = true; std::unique_ptr it(db->NewIterator(ropt)); - Slice prefix(k.data(), 8); - it->Seek(prefix); - long long rowId = -1; - uint64_t bestTs = 0; - for (; it->Valid() && it->key().starts_with(prefix); it->Next()) { - const Slice curKey = it->key(); - if(curKey.size() != 16) - { - return -1; - } - uint64_t ts; - Slice tsSlice(curKey.data() + 8, 8); - DecodeU64Ts(tsSlice, &ts); - auto real_ts = LONG_LONG_MAX - ts; - if (real_ts < timestamp) { - const Slice val = it->value(); - memcpy(&rowId, val.data(), sizeof(rowId)); - break; - } - } + it->Seek(k); + if (!it->Valid()) return -1; + + const Slice val = it->value(); + long long rowId; + memcpy(&rowId, val.data(), sizeof(rowId)); return rowId; } case ts_type_t::udt: From a0dd45043b14dab74e4caad37ca21d9fabc1dacc Mon Sep 17 00:00:00 2001 From: AntiO2 Date: Thu, 23 Oct 2025 11:42:33 +0000 Subject: [PATCH 7/9] fix cmake --- index/rocksdb/config.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/index/rocksdb/config.h b/index/rocksdb/config.h index b38ff32..6bbbd25 100644 --- a/index/rocksdb/config.h +++ b/index/rocksdb/config.h @@ -7,6 +7,7 @@ #pragma once #include +#include enum class ts_type_t { embed_asc, @@ -36,7 +37,7 @@ struct Config { public: using scale_t = int; - const std::string db_path_prefix = "/tmp/rocksdb"; // hard code + const std::string db_path_prefix = "/home/ubuntu/disk1"; // hard code std::string db_path_; int thread_num_; int id_range_; @@ -57,7 +58,7 @@ struct Config { destroy_before_start_ = true; ts_type_ = ts_type; std::ostringstream oss; - oss << "/tmp/rocksdb/" << "test_" << id_range_ << "_" << ts_type_; + oss << db_path_prefix << "/test_" << id_range_ << "_" << ts_type_; db_path_ = oss.str(); } private: From 8c2e10facae0e0a52dace1ee28fd7b2479eed320 Mon Sep 17 00:00:00 2001 From: AntiO2 Date: Fri, 24 Oct 2025 12:00:43 +0000 Subject: [PATCH 8/9] fix buffer --- index/rocksdb/rocksdb_factory.cpp | 6 ++++-- index/rocksdb/rocksdb_load.cpp | 5 ++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/index/rocksdb/rocksdb_factory.cpp b/index/rocksdb/rocksdb_factory.cpp index 70023e3..2172bba 100644 --- a/index/rocksdb/rocksdb_factory.cpp +++ b/index/rocksdb/rocksdb_factory.cpp @@ -54,7 +54,6 @@ std::unique_ptr createRocksDB( cfOptions.comparator = rocksdb::BytewiseComparatorWithU64Ts(); } else { } - descriptors.emplace_back(name, cfOptions); if(cfg.ts_type_ == ts_type_t::embed_desc) { @@ -65,6 +64,9 @@ std::unique_ptr createRocksDB( cfOptions.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(8)); cfOptions.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options)); } + + cfOptions.write_buffer_size = (size_t)6 << 30; + descriptors.emplace_back(name, cfOptions); } DBOptions dbOptions; @@ -87,4 +89,4 @@ std::unique_ptr createRocksDB( << ", ts_type=" << (cfg.ts_type_ == ts_type_t::udt ? "UDT" : "Retina") << std::endl; return std::unique_ptr(db); -} \ No newline at end of file +} diff --git a/index/rocksdb/rocksdb_load.cpp b/index/rocksdb/rocksdb_load.cpp index 8ff31b0..9b4f3fb 100644 --- a/index/rocksdb/rocksdb_load.cpp +++ b/index/rocksdb/rocksdb_load.cpp @@ -52,7 +52,7 @@ class RocksDBSequentialLoadTest : public ::testing::Test { std::string tsBuffer(8, '\0'); const Slice& tsSlice = EncodeU64Ts(timestamp, &tsBuffer); for (int t = 0; t < cfg->thread_num_; ++t) { -// threads.emplace_back([this, t, &counter, timestamp, &tsSlice]() + threads.emplace_back([this, t, &counter, timestamp, &tsSlice]() { WriteOptions wopt; for (int key = 0; key < cfg->id_range_; ++key) { @@ -66,8 +66,7 @@ class RocksDBSequentialLoadTest : public ::testing::Test { counter.fetch_add(1, std::memory_order_relaxed); } } - -// ); + ); } for (auto& th : threads) th.join(); std::cout << "Total loaded: " << counter.load() << " entries" << std::endl; From d3891853385584a0879deb43e8683678d76434bb Mon Sep 17 00:00:00 2001 From: AntiO2 Date: Fri, 24 Oct 2025 13:09:55 +0000 Subject: [PATCH 9/9] Use RocksDB 10.2.1 --- index/rocksdb/CMakeLists.txt | 12 +++++------- index/rocksdb/rocksdb_load.cpp | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/index/rocksdb/CMakeLists.txt b/index/rocksdb/CMakeLists.txt index c1bdf5a..1044d69 100644 --- a/index/rocksdb/CMakeLists.txt +++ b/index/rocksdb/CMakeLists.txt @@ -9,15 +9,13 @@ include_directories(/usr/local/include) link_directories(/usr/local/lib) include(FetchContent) - FetchContent_Declare( googletest - URL https://github.com/google/googletest/archive/refs/tags/v1.15.2.zip - DOWNLOAD_EXTRACT_TIMESTAMP TRUE -) + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG release-1.12.1 +) FetchContent_MakeAvailable(googletest) - enable_testing() add_executable(rocksdb_ts_test @@ -39,7 +37,7 @@ target_link_libraries(rocksdb_ts_test snappy lz4 zstd - + bz2 ) target_link_libraries(rocksdb_load @@ -51,7 +49,7 @@ target_link_libraries(rocksdb_load snappy lz4 zstd - + bz2 ) include(GoogleTest) diff --git a/index/rocksdb/rocksdb_load.cpp b/index/rocksdb/rocksdb_load.cpp index 9b4f3fb..1134c42 100644 --- a/index/rocksdb/rocksdb_load.cpp +++ b/index/rocksdb/rocksdb_load.cpp @@ -127,7 +127,7 @@ TEST(RocksDBSmallTest, UDT) { ropt.timestamp = &ts2; std::string rvalue = std::string(8, '\0'); auto s = db->Get(ropt, key, &rvalue); - s.AssertOK(); + ASSERT_TRUE(s.ok()); std::cout << rvalue << std::endl; handles.clear(); db = nullptr;