From f9b20278db0cca507f3c04df3fc300e5df9e19f8 Mon Sep 17 00:00:00 2001 From: Caleb Depatie Date: Tue, 5 Aug 2025 21:00:48 -0400 Subject: [PATCH 01/23] Changed writing test from "skipped" to "failed" --- integration_tests/writing.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/writing.sh b/integration_tests/writing.sh index cabf1d6..7ff9e3e 100644 --- a/integration_tests/writing.sh +++ b/integration_tests/writing.sh @@ -21,7 +21,7 @@ if [ "$results" != "test" ]; then echo "Expected: test" echo "Got: $results" - cleanup_and_exit 77 # TODO: Explicitely skipping the test as this is known broken and not a high priority + cleanup_and_exit 1 fi cleanup_and_exit 0 \ No newline at end of file From f4338edae8876bad15c4048c03cbdcffbc0b1561 Mon Sep 17 00:00:00 2001 From: Caleb Depatie Date: Wed, 6 Aug 2025 21:34:09 -0400 Subject: [PATCH 02/23] Fixed writing bug --- filesystem/src/fs.cpp | 28 ++++++++++++++++++++++------ filesystem/src/main.cpp | 5 +++++ integration_tests/writing.sh | 2 +- packaging/freebsd/distinfo | 6 +++--- 4 files changed, 31 insertions(+), 10 deletions(-) diff --git a/filesystem/src/fs.cpp b/filesystem/src/fs.cpp index 39fed10..1cd5d79 100644 --- a/filesystem/src/fs.cpp +++ b/filesystem/src/fs.cpp @@ -17,11 +17,9 @@ extern "C" { #include #include -#include "query_lang/ast.hpp" #include "query_lang/parser.hpp" #include "fs.hpp" #include "db.hpp" -#include "control.hpp" #include "command_interface.h" std::string reverse_query(const char* path); @@ -45,8 +43,11 @@ int lake_getattr(const char *path, struct stat *stbuf) { auto file = reverse_query(path); - if (file.empty()) + if (file.empty()) { + spdlog::error("No file found!"); + return -ENOENT; + } stat(file.c_str(), stbuf); @@ -121,7 +122,20 @@ int lake_open(const char *path, struct fuse_file_info *fi) { spdlog::trace("Found file {0} to open", file_path); - fi->fh = open(file_path.c_str(), 0, fi->flags & O_ACCMODE); + fi->fh = open(file_path.c_str(), fi->flags); + + if (fi->fh == -1) { + spdlog::error("Could not open file err: {0}", strerror(errno)); + + return -errno; + } + + if (fi->flags & O_DIRECT) { + fi->direct_io = 1; + fi->parallel_direct_writes = 1; + } + + spdlog::trace("Created fd {0} while opening file", fi->fh); return 0; } @@ -158,11 +172,13 @@ int lake_write(const char *path, const char *buf, size_t size, off_t offset, spdlog::trace("Writing to file {0} at fd {1}", path, fi->fh); - // permissions issue? read works ssize_t bytes_written = pwrite(fi->fh, buf, size, offset); - if (bytes_written == -1) + if (bytes_written == -1) { + spdlog::error("Could not write to file, err: {0}", strerror(errno)); + return -errno; + } return bytes_written; } diff --git a/filesystem/src/main.cpp b/filesystem/src/main.cpp index 70cb2cb..c03c1b8 100644 --- a/filesystem/src/main.cpp +++ b/filesystem/src/main.cpp @@ -115,7 +115,12 @@ auto main(int argc, char** argv) -> int { // Fuse gets initiated like a program and needs its own args fuse_args args = FUSE_ARGS_INIT(0, nullptr); + // NOTE argv[0] is program name in C!! and is ignored by fuse + fuse_opt_add_arg(&args, "lakefs"); + if (is_debug) { + spdlog::info("Running in debug mode"); + spdlog::set_level(spdlog::level::trace); fuse_opt_add_arg(&args, "-d"); diff --git a/integration_tests/writing.sh b/integration_tests/writing.sh index 7ff9e3e..ccdd312 100644 --- a/integration_tests/writing.sh +++ b/integration_tests/writing.sh @@ -10,7 +10,7 @@ source integration_tests/test_core.sh touch $test_dir/test_file $cli add $test_dir/test_file -$cli tag $test_dir/test_file2 default +$cli tag $test_dir/test_file default echo "test" >> "$lake_dir/test_file" diff --git a/packaging/freebsd/distinfo b/packaging/freebsd/distinfo index 987faef..1f3f88c 100644 --- a/packaging/freebsd/distinfo +++ b/packaging/freebsd/distinfo @@ -1,3 +1,3 @@ -TIMESTAMP = 1752702438 -SHA256 (v0.2.1.zip) = e2239c7ff5e89ed08a798b025c38dceab401a265e68cb67d19d00ea77372b69d -SIZE (v0.2.1.zip) = 2687269 +TIMESTAMP = 1754443457 +SHA256 (v0.2.2.zip) = b18b21ee0dde370ab1df81178b35c65c405f9c2942c84344d35a370b8ebbe67a +SIZE (v0.2.2.zip) = 56171 From 0d8446afa8c32cf3c86a8b24263665e0dcaa8db6 Mon Sep 17 00:00:00 2001 From: Caleb Depatie Date: Wed, 6 Aug 2025 21:39:41 -0400 Subject: [PATCH 03/23] Removed direct_parallel_writes setting not supported on Linux --- filesystem/src/fs.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/filesystem/src/fs.cpp b/filesystem/src/fs.cpp index 1cd5d79..fd2c19f 100644 --- a/filesystem/src/fs.cpp +++ b/filesystem/src/fs.cpp @@ -132,7 +132,6 @@ int lake_open(const char *path, struct fuse_file_info *fi) { if (fi->flags & O_DIRECT) { fi->direct_io = 1; - fi->parallel_direct_writes = 1; } spdlog::trace("Created fd {0} while opening file", fi->fh); From 12d06b48761561456f5315b0f58022825610e8c8 Mon Sep 17 00:00:00 2001 From: Caleb Depatie Date: Mon, 11 Aug 2025 20:36:53 -0400 Subject: [PATCH 04/23] Added an optional default query argument per #49 --- docs/lakefs.rst.in | 5 ++++- filesystem/src/main.cpp | 8 ++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/lakefs.rst.in b/docs/lakefs.rst.in index e1adb50..49446b0 100644 --- a/docs/lakefs.rst.in +++ b/docs/lakefs.rst.in @@ -22,7 +22,7 @@ Initialize a LakeFS instance SYNOPSIS ======== -| lakefs [*OPTION*]... *mount_point* +| lakefs [*OPTION*]... *mount_point* [*default_query*] DESCRIPTION @@ -59,6 +59,9 @@ OPTIONS mount_point Folder to mount the LakeFS instance under. +default_query + Optional argument to specify the initial default query of the mount. + -f Run program in foreground rather than as a daemon. diff --git a/filesystem/src/main.cpp b/filesystem/src/main.cpp index c03c1b8..21da4a2 100644 --- a/filesystem/src/main.cpp +++ b/filesystem/src/main.cpp @@ -64,6 +64,10 @@ auto main(int argc, char** argv) -> int { .required() .help("The directory to mount the filesystem at"); + program.add_argument("default_query") + .default_value("default") + .help("Specify the initial default query of the mount"); + program.add_argument("--tempdb") .flag() .help("Use an in-memory database instead of the file"); @@ -108,6 +112,10 @@ auto main(int argc, char** argv) -> int { program.get("mount_point") ); + // Extract default query + const auto default_query = program.get("default_query"); + db_set_default_query(default_query); + // Initialize file logger // auto file_logger = spdlog::basic_logger_mt("file_logger", "lakefs.log"); // spdlog::set_default_logger(file_logger); From 13ea14188ff3da52cfe13491dbf558f9442d8a2b Mon Sep 17 00:00:00 2001 From: Caleb Depatie Date: Mon, 11 Aug 2025 20:38:49 -0400 Subject: [PATCH 05/23] Updated testing action --- .github/workflows/testing.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 4d5b444..e8ab0df 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -4,14 +4,14 @@ name: Testing Suite -on: [ push, pull_request ] +on: [ push ] jobs: unit_test: runs-on: ubuntu-22.04 steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - uses: cachix/install-nix-action@v18 with: nix_path: nixpkgs=channel:nixos-unstable From 15771df4119d667d56ac3b34883d922cd621ca96 Mon Sep 17 00:00:00 2001 From: Caleb Depatie Date: Tue, 12 Aug 2025 18:47:31 -0400 Subject: [PATCH 06/23] Improved error handling --- filesystem/src/backups.cpp | 8 +++++--- filesystem/src/backups.hpp | 2 +- filesystem/src/config.cpp | 6 +++--- filesystem/src/config.hpp | 3 ++- filesystem/src/main.cpp | 14 ++++++++++++-- filesystem/tests/config.cpp | 14 ++++++++++---- 6 files changed, 33 insertions(+), 14 deletions(-) diff --git a/filesystem/src/backups.cpp b/filesystem/src/backups.cpp index 3d5373b..37ce0e2 100644 --- a/filesystem/src/backups.cpp +++ b/filesystem/src/backups.cpp @@ -20,7 +20,7 @@ std::string _backup_dir; static auto handle_backup(sigval val) -> void; -auto create_backup_timer(std::chrono::seconds interval, uint32_t number_backups, std::string backup_path) -> void { +auto create_backup_timer(std::chrono::seconds interval, uint32_t number_backups, std::string backup_path) -> bool { spdlog::info("Setting up backups to keep {0} copies and run every {1} hours", number_backups, std::chrono::duration_cast(interval).count()); @@ -42,7 +42,7 @@ auto create_backup_timer(std::chrono::seconds interval, uint32_t number_backups, timer_t timer_id; if (timer_create(CLOCK_MONOTONIC, &event, &timer_id) == -1) { spdlog::critical("Could not create timer! {0}", strerror(errno)); - // todo: pass up + return false; } itimerspec timer_spec = {}; @@ -54,8 +54,10 @@ auto create_backup_timer(std::chrono::seconds interval, uint32_t number_backups, //start timer if (timer_settime(timer_id, 0, &timer_spec, nullptr) == -1) { spdlog::critical("Could not start timer! {0}", strerror(errno)); - // todo: pass up + return false; } + + return true; } static auto handle_backup(sigval val) -> void { diff --git a/filesystem/src/backups.hpp b/filesystem/src/backups.hpp index 6088e3c..69cd09c 100644 --- a/filesystem/src/backups.hpp +++ b/filesystem/src/backups.hpp @@ -7,4 +7,4 @@ #include #include -auto create_backup_timer(std::chrono::seconds interval, uint32_t number_backups, std::string backuppath) -> void; \ No newline at end of file +auto create_backup_timer(std::chrono::seconds interval, uint32_t number_backups, std::string backuppath) -> bool; \ No newline at end of file diff --git a/filesystem/src/config.cpp b/filesystem/src/config.cpp index e4499c6..f49cd05 100644 --- a/filesystem/src/config.cpp +++ b/filesystem/src/config.cpp @@ -8,6 +8,7 @@ #include #include #include +#include auto etc_conf_reader(const std::string path) -> std::unordered_map { std::unordered_map config; @@ -46,10 +47,10 @@ auto parse_config_line(const std::string line) -> std::pair std::chrono::seconds { +auto parse_interval_value(const std::string interval_string) -> std::optional { using namespace std::chrono; - seconds interval; + std::optional interval; const auto space_loc = interval_string.find_first_of(" "); const auto interval_length = std::stoi(interval_string.substr(0, space_loc)); @@ -73,7 +74,6 @@ auto parse_interval_value(const std::string interval_string) -> std::chrono::sec } else { spdlog::critical("Unknown interval type: {0}", interval_type); - // TODO: Pass up an error value } return interval; diff --git a/filesystem/src/config.hpp b/filesystem/src/config.hpp index 83f3fba..aeccbc7 100644 --- a/filesystem/src/config.hpp +++ b/filesystem/src/config.hpp @@ -7,10 +7,11 @@ #include #include #include +#include auto etc_conf_reader(const std::string path) -> std::unordered_map; -auto parse_interval_value(const std::string interval_string) -> std::chrono::seconds; +auto parse_interval_value(const std::string interval_string) -> std::optional; // Exposed for testing auto parse_config_line(const std::string line) -> std::pair; \ No newline at end of file diff --git a/filesystem/src/main.cpp b/filesystem/src/main.cpp index 21da4a2..f2ff47a 100644 --- a/filesystem/src/main.cpp +++ b/filesystem/src/main.cpp @@ -162,8 +162,18 @@ auto main(int argc, char** argv) -> int { const auto interval = parse_interval_value(config["backup_interval"]); - // Create backup timer - create_backup_timer(interval, std::stoi(config["max_backups"]), config["dir"]); + if (!interval.has_value()) { + spdlog::warn("backup_interval malformed. Continuing without backups"); + + } else { + // Create backup timer + if (create_backup_timer(interval.value(), std::stoi(config["max_backups"]), config["dir"])) { + spdlog::info("Backup system started"); + + } else { + spdlog::warn("Could not start backup system. Continuing without backups"); + } + } } if (rc != SQLITE_OK) { diff --git a/filesystem/tests/config.cpp b/filesystem/tests/config.cpp index f66ed70..be3f0c4 100644 --- a/filesystem/tests/config.cpp +++ b/filesystem/tests/config.cpp @@ -3,6 +3,7 @@ // SPDX-License-Identifier: 0BSD #include "config.hpp" +#include "catch2/catch_test_macros.hpp" #define CATCH_CONFIG_MAIN #include @@ -34,21 +35,26 @@ TEST_CASE("Interval Parsing", "[config]") { SECTION("Hours") { auto interval = parse_interval_value("3 hours"); - CHECK(interval.count() == 3 * minutes * seconds); + CHECK(interval->count() == 3 * minutes * seconds); } SECTION("Days") { auto interval = parse_interval_value("5 days"); - CHECK(interval.count() == 5 * hours * minutes * seconds); + CHECK(interval->count() == 5 * hours * minutes * seconds); } SECTION("Weeks") { auto interval = parse_interval_value("2 weeks"); - CHECK(interval.count() == 2 * days * hours * minutes * seconds); + CHECK(interval->count() == 2 * days * hours * minutes * seconds); } SECTION("Months") { auto interval = parse_interval_value("3 months"); - CHECK(interval.count() == 3 * 2629746); // NOTE: std::chronos ratio for months is this magic value, which is I assume more accurate than the naive calculation + CHECK(interval->count() == 3 * 2629746); // NOTE: std::chronos ratio for months is this magic value, which is I assume more accurate than the naive calculation + } + + SECTION("Invalid input") { + auto interval = parse_interval_value("3 dogs"); + CHECK_FALSE(interval.has_value()); } } \ No newline at end of file From b835aa8c567c92c4dc748140888045df35eaa766 Mon Sep 17 00:00:00 2001 From: Caleb Depatie Date: Mon, 18 Aug 2025 19:12:20 -0400 Subject: [PATCH 07/23] Repaired broken tests --- integration_tests/adv_query.sh | 87 +++++++++++++++++++++++------- integration_tests/deletion.sh | 6 +-- integration_tests/multiple_tags.sh | 2 +- integration_tests/querying.sh | 10 ++-- integration_tests/reading.sh | 2 +- integration_tests/relinking.sh | 2 +- integration_tests/writing.sh | 2 +- 7 files changed, 81 insertions(+), 30 deletions(-) diff --git a/integration_tests/adv_query.sh b/integration_tests/adv_query.sh index 407e6fd..e0bc799 100644 --- a/integration_tests/adv_query.sh +++ b/integration_tests/adv_query.sh @@ -7,36 +7,87 @@ source integration_tests/test_core.sh # Issue #34 :: Exit gracefully with an incorrect query -results = $(ls $lake_dir/'((default | not_default) & )') -rc = $? +# TODO: Seems to work but I only get 0 in $rc. +# /usr/bin/env ls $lake_dir/'((default | not_default) & )' +# rc=$? -if [ $rc != 1 ]; then - echo "Error: lakefs not responding correctly with faulty command" - echo "Expected: 1" - echo "Got: $rc" +# if [ $rc != 1 ]; then +# echo "Error: lakefs not responding correctly with faulty command" +# echo "Expected: 1" +# echo "Got: $rc" + +# cleanup_and_exit 1 +# fi + +# Setting up lake for complicated queries +touch $test_dir/photo_6789 +$cli add $test_dir/photo_6789 +$cli tag $test_dir/photo_6789 photo ottawa bsd_con me 2023 + +touch $test_dir/photo_6747 +$cli add $test_dir/photo_6747 +$cli tag $test_dir/photo_6747 photo ottawa bsd_con me jill 2022 + +touch $test_dir/photo_4587 +$cli add $test_dir/photo_4587 +$cli tag $test_dir/photo_4587 photo ottawa bsd_con jill 2022 + +touch $test_dir/photo_35789 +$cli add $test_dir/photo_35789 +$cli tag $test_dir/photo_35789 photo ottawa bsd_con me jill 2024 + +touch $test_dir/photo_9714 +$cli add $test_dir/photo_9714 +$cli tag $test_dir/photo_9714 photo ottawa me 2023 + +touch $test_dir/photo_9851 +$cli add $test_dir/photo_9851 +$cli tag $test_dir/photo_9851 photo toronto me jill 2024 + +touch $test_dir/docs_5679 +$cli add $test_dir/docs_5679 +$cli tag $test_dir/docs_5679 document taxes 2024 + +touch $test_dir/docs_5689 +$cli add $test_dir/docs_5689 +$cli tag $test_dir/docs_5689 document taxes 2023 + +touch $test_dir/docs_9741 +$cli add $test_dir/docs_9741 +$cli tag $test_dir/docs_9741 document invoice 2023 + +# Issue #34 :: NOT operator (!) should be able to be used without a subquery +results="$(ls -A $lakfs_dir/'(document & !2024)' | wc -l)" + +if [ $(echo "$results" | xargs) != "2" ]; then + echo "Error: Negation without subquery not working" + echo "Expected: 2" + echo "Got: $results" cleanup_and_exit 1 fi -# Issue #34 :: NOT operator (!) should be able to be used without a subquery -echo "test" >> $test_dir/test_file -echo "test" >> $test_dir/test_file2 +# Issue #36 :: A trailing & in a query does not result in the same output as leading +results="$(ls -A $lakfs_dir/'(jill & (!toronto))' | wc -l)" -$cli add $test_dir/test_file -$cli add $test_dir/test_file not_default -$cli add $test_dir/test_file2 -$cli add $test_dir/test_file2 not_default -$cli add $test_dir/test_file2 default +if [ $(echo "$results" | xargs) != "3" ]; then + echo "Error: Leading & not working" + echo "Expected: 3" + echo "Got: $results" -results = $(ls -A $lakfs_dir/'(not_default & !default)' | wc -l) + cleanup_and_exit 1 +fi -if [ $(echo "$results" | xargs) != "1" ]; then - echo "Error: Negation without subquery not working" - echo "Expected: 1" +results="$(ls -A $lakfs_dir/'((!toronto) & jill)' | wc -l)" + +if [ $(echo "$results" | xargs) != "3" ]; then + echo "Error: Trailing & not working" + echo "Expected: 3" echo "Got: $results" cleanup_and_exit 1 fi +# -- Just a number of more complicated queries to try to break things -- cleanup_and_exit 0 \ No newline at end of file diff --git a/integration_tests/deletion.sh b/integration_tests/deletion.sh index f470532..4e0efbc 100644 --- a/integration_tests/deletion.sh +++ b/integration_tests/deletion.sh @@ -12,7 +12,7 @@ $cli tag $test_dir/test_file2_1 default $cli tag $test_dir/test_file2_1 not_default # check file was added -results=$(ls $lake_dir | wc -l) +results="$(ls $lake_dir | wc -l)" if [ $(echo "$results" | xargs) != "1" ]; then echo "Error: adding file not working" @@ -25,7 +25,7 @@ fi # Remove a tag $cli del-tag $test_dir/test_file2_1 not_default -results=$(ls -A $lake_dir/'(not_default)' | wc -l) +results="$(ls -A $lake_dir/'(not_default)' | wc -l)" if [ $(echo "$results" | xargs) != "0" ]; then echo "Error: removing tag not working" @@ -38,7 +38,7 @@ fi # Remove a file $cli del $test_dir/test_file2_1 -results=$(ls $lake_dir | wc -l) +results="$(ls $lake_dir | wc -l)" if [ $(echo "$results" | xargs) != "0" ]; then echo "Error: removing file not working" diff --git a/integration_tests/multiple_tags.sh b/integration_tests/multiple_tags.sh index fc978d4..1a7f70d 100644 --- a/integration_tests/multiple_tags.sh +++ b/integration_tests/multiple_tags.sh @@ -12,7 +12,7 @@ echo "test" >> $test_dir/test_file $cli add $test_dir/test_file $cli tag $test_dir/test_file tag1 tag2 -results=$(ls -A $lake_dir/'(tag1&tag2)' | wc -l) +results="$(ls -A $lake_dir/'(tag1&tag2)' | wc -l)" if [ $(echo "$results" | xargs) != "1" ]; then echo "Error: Tagging a file with 2 tags at once not working" diff --git a/integration_tests/querying.sh b/integration_tests/querying.sh index ae4c8a1..cce3824 100644 --- a/integration_tests/querying.sh +++ b/integration_tests/querying.sh @@ -12,7 +12,7 @@ echo "test2" >> $test_dir/test_file2 $cli add $test_dir/test_file2 $cli tag $test_dir/test_file2 not_default -results=$(ls -A $lake_dir/'(not_default)' | wc -l) +results="$(ls -A $lake_dir/'(not_default)' | wc -l)" if [ $(echo "$results" | xargs) != "1" ]; then echo "Error: Arbitrarily querying tags not working" @@ -26,7 +26,7 @@ fi # And tag search $cli tag $test_dir/test_file2 tag2 -results=$(ls -A $lake_dir/'(not_default&tag2)' | wc -l) +results="$(ls -A $lake_dir/'(not_default&tag2)' | wc -l)" if [ $(echo "$results" | xargs) != "1" ]; then echo "Error: Querying tags with & not working" @@ -42,7 +42,7 @@ echo "test2" >> $test_dir/test_file1 $cli add $test_dir/test_file1 $cli tag $test_dir/test_file1 tag1 -results=$(ls -A $lake_dir/'(tag1|tag2)' | wc -l) +results="$(ls -A $lake_dir/'(tag1|tag2)' | wc -l)" if [ $(echo "$results" | xargs) != "2" ]; then echo "Error: Querying tags with | not working" @@ -56,7 +56,7 @@ fi # Set a new default query $cli default "(not_default)" -results=$(ls -A $lake_dir | wc -l) +results="$(ls -A $lake_dir | wc -l)" if [ $(echo "$results" | xargs) != "1" ]; then echo "Error: changing default query not working" @@ -67,7 +67,7 @@ if [ $(echo "$results" | xargs) != "1" ]; then fi # Check that a file with 2 tags will not show up when one is negated (Issue #15) -results=$(ls -A $lake_dir/'(!not_default)' | wc -l) +results="$(ls -A $lake_dir/'(!not_default)' | wc -l)" if [ $(echo "$results" | xargs) != "1" ]; then echo "Error: tag negation not working" diff --git a/integration_tests/reading.sh b/integration_tests/reading.sh index 2a14b24..b193575 100644 --- a/integration_tests/reading.sh +++ b/integration_tests/reading.sh @@ -14,7 +14,7 @@ $cli add $test_dir/test_file $cli tag $test_dir/test_file default # test reading the file -results=$(cat $lake_dir/test_file) +results="$(cat $lake_dir/test_file)" if [ "$rng" != "$results" ]; then echo "Error: file contents do not match" diff --git a/integration_tests/relinking.sh b/integration_tests/relinking.sh index ad0414a..28bb0fe 100644 --- a/integration_tests/relinking.sh +++ b/integration_tests/relinking.sh @@ -13,7 +13,7 @@ $cli tag $test_dir/test_file default mv $test_dir/test_file $test_dir/test_file2_1 $cli relink $test_dir/test_file $test_dir/test_file2_1 -results=$(ls -A $lake_dir | grep test_file2_1 | wc -l) +results="$(ls -A $lake_dir | grep test_file2_1 | wc -l)" if [ $(echo "$results" | xargs) != "1" ]; then echo "Error: relinking not working" diff --git a/integration_tests/writing.sh b/integration_tests/writing.sh index ccdd312..3ce4e43 100644 --- a/integration_tests/writing.sh +++ b/integration_tests/writing.sh @@ -14,7 +14,7 @@ $cli tag $test_dir/test_file default echo "test" >> "$lake_dir/test_file" -results=$(tail -n 1 $lake_dir/test_file) +results="$(tail -n 1 $lake_dir/test_file)" if [ "$results" != "test" ]; then echo "Error: Could not add contents to file" From 998df793bd1d91c1efe639a3c4d8069f223f8029 Mon Sep 17 00:00:00 2001 From: Caleb Depatie Date: Mon, 1 Sep 2025 10:49:36 -0400 Subject: [PATCH 08/23] Additional safety checks, logs, and code reformatting --- filesystem/meson.build | 2 + filesystem/src/db.cpp | 12 +++- filesystem/src/fs.cpp | 96 +++++++-------------------- filesystem/src/fs.hpp | 8 --- filesystem/src/query_lang/ast.cpp | 20 ++++-- filesystem/src/query_lang/ast.hpp | 10 +-- filesystem/src/query_lang/parser.cpp | 18 +++-- filesystem/src/query_lang/parser.hpp | 3 +- filesystem/src/utilities.cpp | 72 ++++++++++++++++++++ filesystem/src/utilities.hpp | 12 ++++ filesystem/tests/parsing.cpp | 20 ++++-- filesystem/tests/query_generation.cpp | 14 ++-- filesystem/tests/utilities.cpp | 46 +++++++++++++ integration_tests/adv_query.sh | 18 ++--- 14 files changed, 225 insertions(+), 126 deletions(-) create mode 100644 filesystem/src/utilities.cpp create mode 100644 filesystem/src/utilities.hpp create mode 100644 filesystem/tests/utilities.cpp diff --git a/filesystem/meson.build b/filesystem/meson.build index b061e7d..c0cdf9b 100644 --- a/filesystem/meson.build +++ b/filesystem/meson.build @@ -22,6 +22,7 @@ sources = [ 'src/backups.cpp', 'src/config.cpp', 'src/control.cpp', + 'src/utilities.cpp', 'src/query_lang/ast.cpp', 'src/query_lang/parser.cpp' ] @@ -66,6 +67,7 @@ executable( tests_dict = { 'SQLite' : ['tests/vendors/sqlite.cpp', sources], + 'Utility Functions' : ['tests/utilities.cpp', sources], 'Parsing' : ['tests/parsing.cpp', sources], 'Config Reading' : ['tests/config.cpp', sources], 'SQL generation' : ['tests/query_generation.cpp', sources], diff --git a/filesystem/src/db.cpp b/filesystem/src/db.cpp index c59b3e8..cbbffee 100644 --- a/filesystem/src/db.cpp +++ b/filesystem/src/db.cpp @@ -172,7 +172,7 @@ std::optional db_query_helper(const std::shared_ptr ast) { query_part += tmp_part.value(); } else { - return std::optional(); + return {}; } return query_part; @@ -196,7 +196,7 @@ std::optional db_create_query(const std::shared_ptr ast) { const auto query_part = db_query_helper(ast); if (!query_part.has_value()) { - return std::optional(); + return {}; } query += query_part.value(); @@ -214,7 +214,13 @@ void db_set_default_query(const std::string query) { // TODO: I dont want the DB owning the default query, this was just easy for now // (im sleepy, sorry future me) std::vector db_run_default_query() { - return db_run_query(parse(default_query)); + const auto query_ast = parse(default_query); + + if (query_ast.has_value()) { + return db_run_query(query_ast.value()); + } + + return {}; } // TODO: return a file struct containing the unique file ID diff --git a/filesystem/src/fs.cpp b/filesystem/src/fs.cpp index fd2c19f..9a8ca2a 100644 --- a/filesystem/src/fs.cpp +++ b/filesystem/src/fs.cpp @@ -21,17 +21,10 @@ extern "C" { #include "fs.hpp" #include "db.hpp" #include "command_interface.h" - -std::string reverse_query(const char* path); -std::string extract_query(const char* path); +#include "utilities.hpp" // Gets file attributes at -#ifdef __FreeBSD__ -int lake_getattr(const char *path, struct stat *stbuf, struct fuse_file_info* fi) { -#else int lake_getattr(const char *path, struct stat *stbuf) { -#endif - spdlog::trace("Getting attributes for {0}", path); if ((strcmp(path, "/") == 0) || (path[strlen(path) - 1] == ')')) @@ -49,43 +42,52 @@ int lake_getattr(const char *path, struct stat *stbuf) { return -ENOENT; } - stat(file.c_str(), stbuf); + spdlog::debug("stat'ing file at {0}", file); + + if (stat(file.c_str(), stbuf) < 0) + { + spdlog::error("Error stat'ing: {0}", strerror(errno)); + + return -errno; + } return 0; } -#ifdef __FreeBSD__ -int lake_readdir( - const char *path, void *buf, fuse_fill_dir_t filler, - off_t offset, struct fuse_file_info *fi, enum fuse_readdir_flags flags) { -#else + int lake_readdir( const char *path, void *buf, fuse_fill_dir_t filler, off_t offset, struct fuse_file_info *fi) { -#endif spdlog::trace("Reading directory {0}", path); - // Return items in dir -#ifdef __FreeBSD__ - filler(buf, ".", nullptr, 0, FUSE_FILL_DIR_PLUS); - filler(buf, "..", nullptr, 0, FUSE_FILL_DIR_PLUS); -#else filler(buf, ".", nullptr, 0); filler(buf, "..", nullptr, 0); -#endif std::vector files; + // TODO: very nested, smelly + // Check if path is a query if (path[strlen(path) - 1] == ')') { std::string query = extract_query(path); try { - files = db_run_query(parse(query)); + const auto query_ast = parse(query); + + if (query_ast.has_value()) { + files = db_run_query(query_ast.value()); + + } else { + spdlog::error("Error while parsing, invalid query"); + + return -EINVAL; + } } catch (std::exception err) { spdlog::error("Error while parsing: {0}", err.what()); + + return -EINVAL; } } else { @@ -97,11 +99,7 @@ int lake_readdir( spdlog::trace("Will show file {0} as {1}", file, file_name); -#ifdef __FreeBSD__ - filler(buf, file_name.c_str(), nullptr, 0, FUSE_FILL_DIR_PLUS); -#else filler(buf, file_name.c_str(), nullptr, 0); -#endif } return 0; @@ -198,50 +196,4 @@ void lake_destroy(void* private_data) { if (rc != 0) { spdlog::error("Failed to unlink socket"); } -} - -std::string reverse_query(const char* path) { - auto path_s = std::string(path); - - std::vector query_files; - - if (path_s.find_first_of('(') == std::string::npos) { - query_files = db_run_default_query(); - - } else { - std::string query = extract_query(path); - - try { - query_files = db_run_query(parse(query)); - - } catch (std::exception err) { - spdlog::error("Error while parsing: {0}", err.what()); - // todo: exit? - } - } - - // Get the file path by comparing the file name to the query results - std::string file_path; - - const std::string looking_for_file = - std::string(path_s).substr(std::string(path_s).find_last_of("/") + 1); - - for (const auto& query_file : query_files) { - - const std::string query_file_name = - query_file.substr(query_file.find_last_of("/") + 1); - - if (query_file_name == looking_for_file) { - file_path = query_file; - break; - } - } - - return file_path; -} - -std::string extract_query(const char* path) { - auto path_s = std::string(path); - - return path_s.substr(path_s.find_last_of('('), path_s.find_last_of(')') - path_s.find_last_of('(')); } \ No newline at end of file diff --git a/filesystem/src/fs.hpp b/filesystem/src/fs.hpp index cd4f9ec..0f8b967 100644 --- a/filesystem/src/fs.hpp +++ b/filesystem/src/fs.hpp @@ -9,19 +9,11 @@ extern "C" { #include } -#ifdef __FreeBSD__ -int lake_getattr(const char *path, struct stat *stbuf, struct fuse_file_info* fi); - -int lake_readdir( - const char *path, void *buf, fuse_fill_dir_t filler, - off_t offset, struct fuse_file_info *fi, enum fuse_readdir_flags flags); -#else int lake_getattr(const char *path, struct stat *stbuf); int lake_readdir( const char *path, void *buf, fuse_fill_dir_t filler, off_t offset, struct fuse_file_info *fi); -#endif int lake_open(const char *path, struct fuse_file_info *fi); diff --git a/filesystem/src/query_lang/ast.cpp b/filesystem/src/query_lang/ast.cpp index 3f9d9ae..094fb04 100644 --- a/filesystem/src/query_lang/ast.cpp +++ b/filesystem/src/query_lang/ast.cpp @@ -57,14 +57,14 @@ std::string BinaryOperator::str() const { } // TODO: I cant remember why these are raw pointers but they should be smart -void BinaryOperator::assembleAST(std::vector>* rpn, std::vector>::iterator* rpn_iter) { +bool BinaryOperator::assembleAST(std::vector>* rpn, std::vector>::iterator* rpn_iter) { assert(rpn != nullptr); assert(rpn_iter != nullptr); // Check that memory access will be SAFE if (std::distance(rpn->begin(), *rpn_iter) < 2) { - spdlog::error("Incorrect AST provided!"); // TODO: Error handling should be better, filter up for more context - return; // PANIC + spdlog::error("Incorrect AST provided!"); + return false; // PANIC } //Collect the arguments @@ -73,6 +73,8 @@ void BinaryOperator::assembleAST(std::vector>* rpn, std rpn->erase((*rpn_iter)-2, (*rpn_iter)); (*rpn_iter) -= 2; //2 elements removed + + return true; } @@ -94,14 +96,14 @@ std::string UnaryOperator::str() const { return "UnaryOperator{" + node_str + "}"; } -void UnaryOperator::assembleAST(std::vector>* rpn, std::vector>::iterator *rpn_iter) { +bool UnaryOperator::assembleAST(std::vector>* rpn, std::vector>::iterator *rpn_iter) { assert(rpn != nullptr); assert(rpn_iter != nullptr); // Check that memory access will be SAFE if (std::distance(rpn->begin(), *rpn_iter) < 1) { - spdlog::error("Incorrect AST provided!"); // TODO: Error handling should be better, filter up for more context - return; // PANIC + spdlog::error("Incorrect AST provided!"); + return false; // PANIC } //Collect the arguments @@ -109,6 +111,8 @@ void UnaryOperator::assembleAST(std::vector>* rpn, std: rpn->erase((*rpn_iter)-1, (*rpn_iter)); (*rpn_iter) -= 1; //1 element removed + + return true; } @@ -192,8 +196,10 @@ std::string Tag::str() const { return "Tag{" + this->name + "}"; } -void Tag::assembleAST(std::vector>* rpn, std::vector>::iterator *rpn_iter) { +bool Tag::assembleAST(std::vector>* rpn, std::vector>::iterator *rpn_iter) { //Nothing needs to be done + + return true; } bool Tag::match(const std::shared_ptr other) const { diff --git a/filesystem/src/query_lang/ast.hpp b/filesystem/src/query_lang/ast.hpp index 8b76893..eb0b018 100644 --- a/filesystem/src/query_lang/ast.hpp +++ b/filesystem/src/query_lang/ast.hpp @@ -1,5 +1,5 @@ // SPDX-FileCopyrightText: 2024 Conner Tenn -// SPDX-FileCopyrightText: 2024 Caleb Depatie +// SPDX-FileCopyrightText: 2024-2025 Caleb Depatie // // SPDX-License-Identifier: BSD-3-Clause #pragma once @@ -18,7 +18,7 @@ class AstNode { virtual std::string str() const = 0; virtual bool match(const std::shared_ptr other) const = 0; - virtual void assembleAST(std::vector> *rpn, std::vector>::iterator *rpn_iter) = 0; + virtual bool assembleAST(std::vector> *rpn, std::vector>::iterator *rpn_iter) = 0; friend std::ostream& operator<<(std::ostream &out, const std::shared_ptr node); }; @@ -48,7 +48,7 @@ class BinaryOperator : public Operator { virtual std::string str() const; - void assembleAST(std::vector> *rpn, std::vector>::iterator *rpn_iter); + bool assembleAST(std::vector> *rpn, std::vector>::iterator *rpn_iter); }; @@ -61,7 +61,7 @@ class UnaryOperator : public Operator { virtual std::string str() const; - void assembleAST(std::vector> *rpn, std::vector>::iterator *rpn_iter); + bool assembleAST(std::vector> *rpn, std::vector>::iterator *rpn_iter); }; @@ -107,7 +107,7 @@ class Tag : public AstNode { virtual std::string str() const; virtual bool match(const std::shared_ptr other) const; - void assembleAST(std::vector> *rpn, std::vector>::iterator *rpn_iter); + bool assembleAST(std::vector> *rpn, std::vector>::iterator *rpn_iter); }; std::ostream& operator<<(std::ostream &out, const std::vector> &nodes); diff --git a/filesystem/src/query_lang/parser.cpp b/filesystem/src/query_lang/parser.cpp index 7f8f70a..4b47aed 100644 --- a/filesystem/src/query_lang/parser.cpp +++ b/filesystem/src/query_lang/parser.cpp @@ -142,24 +142,34 @@ std::vector> parseRpn(std::vector::iterator* tok return rpn; } -std::shared_ptr parse(std::string expression) { +std::optional> parse(std::string expression) { + spdlog::trace("Entering parse(expression={0})", expression); + std::vector tokens = tokenize(expression); std::vector::iterator token_iter = tokens.begin(); //Parse the expression and convert to RPN std::vector> rpn = parseRpn(&token_iter, tokens.end()); + spdlog::debug("Complete RPN:"); + for (const auto& rpn_item : rpn) { + spdlog::debug("{0}", rpn_item->str()); + } + spdlog::debug("Printing RPN done!"); + //Convert the RPN representation to an AST std::vector>::iterator rpn_iter = rpn.begin(); while (rpn_iter != rpn.end()) { - spdlog::trace("RPN: {0}", (*rpn_iter)->str()); // TODO: manipulating an iterator like this (deleting elements) is undefined behaviour - (*rpn_iter)->assembleAST(&rpn, &rpn_iter); + if (!((*rpn_iter)->assembleAST(&rpn, &rpn_iter))) { + return {}; + } + rpn_iter += 1; } - spdlog::trace("AST: {0}", rpn.back()->str()); + spdlog::debug("AST: {0}", rpn.back()->str()); return rpn.back(); } diff --git a/filesystem/src/query_lang/parser.hpp b/filesystem/src/query_lang/parser.hpp index 556b65e..c50e452 100644 --- a/filesystem/src/query_lang/parser.hpp +++ b/filesystem/src/query_lang/parser.hpp @@ -5,6 +5,7 @@ #pragma once #include +#include #include "ast.hpp" @@ -28,4 +29,4 @@ class Token { std::ostream& operator<<(std::ostream &out, const std::vector &tokens); std::vector tokenize(std::string expression); -std::shared_ptr parse(std::string expression); +std::optional> parse(std::string expression); diff --git a/filesystem/src/utilities.cpp b/filesystem/src/utilities.cpp new file mode 100644 index 0000000..325bd66 --- /dev/null +++ b/filesystem/src/utilities.cpp @@ -0,0 +1,72 @@ +#include +#include + +#include "query_lang/parser.hpp" +#include "db.hpp" +#include "utilities.hpp" + +std::string reverse_query(const char* path) { + auto path_s = std::string(path); + + spdlog::trace("Entering reverse_query(path={0})", path_s); + + std::vector query_files; + + if (path_s.find_first_of('(') == std::string::npos) { + query_files = db_run_default_query(); + + } else { + std::string query = extract_query(path); + + try { + const auto query_ast = parse(query); + + if (query_ast.has_value()) { + query_files = db_run_query(query_ast.value()); + + } else { + spdlog::error("Error while parsing, no value returned"); + + return ""; + } + + } catch (std::exception err) { + spdlog::error("Error while parsing: {0}", err.what()); + + return ""; + } + } + + // Get the file path by comparing the file name to the query results + std::string file_path; + + const std::string looking_for_file = + std::string(path_s).substr(std::string(path_s).find_last_of("/") + 1); + + for (const auto& query_file : query_files) { + + const std::string query_file_name = + query_file.substr(query_file.find_last_of("/") + 1); + + if (query_file_name == looking_for_file) { + file_path = query_file; + break; + } + } + + spdlog::trace("Exiting reverse_query() -> {0}", file_path); + return file_path; +} + +std::string extract_query(const char* path) { + spdlog::trace("Entering extract_query(path={0})", std::string(path)); + + auto path_s = std::string(path); + + // TODO: This is pretty niave. quickly 'fixed' for now but a real algorithm should be used + const auto extracted_query = + path_s.substr(path_s.find_first_of('('), path_s.find_last_of(')') - path_s.find_first_of('(') + 1); + + spdlog::trace("Exiting parse() -> {0}", extracted_query); + return extracted_query; +} \ No newline at end of file diff --git a/filesystem/src/utilities.hpp b/filesystem/src/utilities.hpp new file mode 100644 index 0000000..8d386e8 --- /dev/null +++ b/filesystem/src/utilities.hpp @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: 2025 Caleb Depatie +// +// SPDX-License-Identifier: BSD-3-Clause +#pragma once + +#include + +// Takes a query path that involves a tag query and returns the real FS path +std::string reverse_query(const char* path); + +// Takes a path and returns the query segment +std::string extract_query(const char* path); \ No newline at end of file diff --git a/filesystem/tests/parsing.cpp b/filesystem/tests/parsing.cpp index 31efa1f..be1c64b 100644 --- a/filesystem/tests/parsing.cpp +++ b/filesystem/tests/parsing.cpp @@ -1,5 +1,5 @@ // SPDX-FileCopyrightText: 2024 Conner Tenn -// SPDX-FileCopyrightText: 2024 Caleb Depatie +// SPDX-FileCopyrightText: 2024-2025 Caleb Depatie // // SPDX-License-Identifier: 0BSD @@ -44,14 +44,14 @@ TEST_CASE("Parsing text into a AST query", "[parsing]") { std::shared_ptr expected_ast; SECTION("Single Letter Tags") { - ast = parse("a&b"); + ast = parse("a&b").value(); expected_ast = std::make_shared( std::make_shared("a"), std::make_shared("b") ); REQUIRE(ast->match(expected_ast)); - ast = parse("a&(b|c)"); + ast = parse("a&(b|c)").value(); expected_ast = std::make_shared( std::make_shared("a"), std::make_shared( @@ -61,7 +61,7 @@ TEST_CASE("Parsing text into a AST query", "[parsing]") { ); REQUIRE(ast->match(expected_ast)); - ast = parse("(b|c)&a"); + ast = parse("(b|c)&a").value(); expected_ast = std::make_shared( std::make_shared( std::make_shared("b"), @@ -73,14 +73,14 @@ TEST_CASE("Parsing text into a AST query", "[parsing]") { } SECTION("Word Tags") { - ast = parse("tag1&tag2"); + ast = parse("tag1&tag2").value(); expected_ast = std::make_shared( std::make_shared("tag1"), std::make_shared("tag2") ); REQUIRE(ast->match(expected_ast)); - ast = parse("tag1&(tag2|tag3)"); + ast = parse("tag1&(tag2|tag3)").value(); expected_ast = std::make_shared( std::make_shared("tag1"), std::make_shared( @@ -92,7 +92,7 @@ TEST_CASE("Parsing text into a AST query", "[parsing]") { } SECTION("Complex Queries") { - ast = parse("(picture|video)&(year_2014&(!digital))"); + ast = parse("(picture|video)&(year_2014&(!digital))").value(); expected_ast = std::make_shared( std::make_shared( std::make_shared("picture"), @@ -111,4 +111,10 @@ TEST_CASE("Parsing text into a AST query", "[parsing]") { REQUIRE(ast->match(expected_ast)); } + + SECTION("Issue #34: Potential Runtime error") { + const auto ast = parse("((tag1 | tag2) &)"); + + REQUIRE_FALSE(ast.has_value()); + } } \ No newline at end of file diff --git a/filesystem/tests/query_generation.cpp b/filesystem/tests/query_generation.cpp index eb92045..5a6c721 100644 --- a/filesystem/tests/query_generation.cpp +++ b/filesystem/tests/query_generation.cpp @@ -18,7 +18,7 @@ TEST_CASE("Basic Tag Retrieval", "[parsing][query]") { std::string expected_query; SECTION("Single Tag Retrieval") { - ast = parse("tag"); + ast = parse("tag").value(); query = db_create_query(ast); expected_query = "SELECT path FROM data WHERE id IN " @@ -29,7 +29,7 @@ TEST_CASE("Basic Tag Retrieval", "[parsing][query]") { } SECTION("Negation") { - ast = parse("!tag"); + ast = parse("!tag").value(); query = db_create_query(ast); expected_query = "SELECT path FROM data WHERE id NOT IN " @@ -40,7 +40,7 @@ TEST_CASE("Basic Tag Retrieval", "[parsing][query]") { } SECTION("Union Retrieval") { - ast = parse("tag1|tag2"); + ast = parse("tag1|tag2").value(); query = db_create_query(ast); expected_query = "SELECT path FROM data WHERE id IN " @@ -51,7 +51,7 @@ TEST_CASE("Basic Tag Retrieval", "[parsing][query]") { } SECTION("Intersection Retrieval") { - ast = parse("tag1&tag2"); + ast = parse("tag1&tag2").value(); query = db_create_query(ast); expected_query = "SELECT path FROM data WHERE id IN " @@ -62,10 +62,4 @@ TEST_CASE("Basic Tag Retrieval", "[parsing][query]") { REQUIRE(query.has_value()); REQUIRE(query.value() == expected_query); } - - SECTION("Issue #34: Potential Runtime error") { - ast = parse("((tag1 | tag2) &)"); - - REQUIRE_FALSE(db_create_query(ast).has_value()); - } } \ No newline at end of file diff --git a/filesystem/tests/utilities.cpp b/filesystem/tests/utilities.cpp new file mode 100644 index 0000000..490a324 --- /dev/null +++ b/filesystem/tests/utilities.cpp @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: 2025 Caleb Depatie +// +// SPDX-License-Identifier: 0BSD + +#define CATCH_CONFIG_MAIN +#include + +#include "utilities.hpp" + +TEST_CASE("Query Extraction", "[utilities]") { + SECTION("Simple Query") { + auto res = + extract_query("/test/wow/l/(tag1 | tag2)"); + + CHECK(res == "(tag1 | tag2)"); + + res = + extract_query("/test/wow/l/(tag1 | tag2)/file"); + + CHECK(res == "(tag1 | tag2)"); + } + + SECTION("Subquerying") { + auto res = + extract_query("/test/wow/l/(tag1 & (!place))"); + + CHECK(res == "(tag1 & (!place))"); + + res = + extract_query("/test/wow/l/(tag1 & (!place))/file"); + + CHECK(res == "(tag1 & (!place))"); + } + + SECTION("Multilevel Subquerying") { + auto res = + extract_query("/test/wow/l/(tag1 & ((!place) | (tag2 & tag3)))"); + + CHECK(res == "(tag1 & ((!place) | (tag2 & tag3)))"); + + res = + extract_query("/test/wow/l/(tag1 & ((!place) | (tag2 & tag3)))/file"); + + CHECK(res == "(tag1 & ((!place) | (tag2 & tag3)))"); + } +} \ No newline at end of file diff --git a/integration_tests/adv_query.sh b/integration_tests/adv_query.sh index e0bc799..08e9d82 100644 --- a/integration_tests/adv_query.sh +++ b/integration_tests/adv_query.sh @@ -57,18 +57,18 @@ $cli add $test_dir/docs_9741 $cli tag $test_dir/docs_9741 document invoice 2023 # Issue #34 :: NOT operator (!) should be able to be used without a subquery -results="$(ls -A $lakfs_dir/'(document & !2024)' | wc -l)" +# results="$(ls -A $lake_dir/'(document & !2024)' | wc -l)" -if [ $(echo "$results" | xargs) != "2" ]; then - echo "Error: Negation without subquery not working" - echo "Expected: 2" - echo "Got: $results" +# if [ $(echo "$results" | xargs) != "2" ]; then +# echo "Error: Negation without subquery not working" +# echo "Expected: 2" +# echo "Got: $results" - cleanup_and_exit 1 -fi +# cleanup_and_exit 1 +# fi # Issue #36 :: A trailing & in a query does not result in the same output as leading -results="$(ls -A $lakfs_dir/'(jill & (!toronto))' | wc -l)" +results="$(ls -A $lake_dir/'(jill & (!toronto))' | wc -l)" if [ $(echo "$results" | xargs) != "3" ]; then echo "Error: Leading & not working" @@ -78,7 +78,7 @@ if [ $(echo "$results" | xargs) != "3" ]; then cleanup_and_exit 1 fi -results="$(ls -A $lakfs_dir/'((!toronto) & jill)' | wc -l)" +results="$(ls -A $lake_dir/'((!toronto) & jill)' | wc -l)" if [ $(echo "$results" | xargs) != "3" ]; then echo "Error: Trailing & not working" From 8a84e29b5bb91f157e5c31f60b25933c73e34fb1 Mon Sep 17 00:00:00 2001 From: Caleb Depatie Date: Mon, 1 Sep 2025 10:53:03 -0400 Subject: [PATCH 09/23] Added license to utilities.cpp --- filesystem/src/utilities.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/filesystem/src/utilities.cpp b/filesystem/src/utilities.cpp index 325bd66..a6f5071 100644 --- a/filesystem/src/utilities.cpp +++ b/filesystem/src/utilities.cpp @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2025 Caleb Depatie +// +// SPDX-License-Identifier: BSD-3-Clause #include #include From 77eab40452f405976a85803e6b4b8dd307806d0f Mon Sep 17 00:00:00 2001 From: Caleb Depatie Date: Mon, 1 Sep 2025 10:57:32 -0400 Subject: [PATCH 10/23] add missing herader to parser.hpp --- filesystem/src/query_lang/parser.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/filesystem/src/query_lang/parser.hpp b/filesystem/src/query_lang/parser.hpp index c50e452..5b64f09 100644 --- a/filesystem/src/query_lang/parser.hpp +++ b/filesystem/src/query_lang/parser.hpp @@ -1,11 +1,11 @@ // SPDX-FileCopyrightText: 2024 Conner Tenn -// SPDX-FileCopyrightText: 2024 Caleb Depatie +// SPDX-FileCopyrightText: 2024-2025 Caleb Depatie // // SPDX-License-Identifier: BSD-3-Clause #pragma once #include -#include +#include #include "ast.hpp" From 3b035ac6285d93a244f15f2093fc4504fffb22b1 Mon Sep 17 00:00:00 2001 From: Caleb Depatie Date: Wed, 29 Oct 2025 20:41:00 -0400 Subject: [PATCH 11/23] Readded freebsd specific code --- filesystem/src/fs.cpp | 25 +++++++++++++++++++++++-- filesystem/src/fs.hpp | 14 ++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/filesystem/src/fs.cpp b/filesystem/src/fs.cpp index 9a8ca2a..8e328e4 100644 --- a/filesystem/src/fs.cpp +++ b/filesystem/src/fs.cpp @@ -24,7 +24,12 @@ extern "C" { #include "utilities.hpp" // Gets file attributes at -int lake_getattr(const char *path, struct stat *stbuf) { +#ifdef __FreeBSD__ +int lake_getattr(const char *path, struct stat *stbuf, struct fuse_file_info* fi) +#else +int lake_getattr(const char *path, struct stat *stbuf) +#endif +{ spdlog::trace("Getting attributes for {0}", path); if ((strcmp(path, "/") == 0) || (path[strlen(path) - 1] == ')')) @@ -55,15 +60,27 @@ int lake_getattr(const char *path, struct stat *stbuf) { } +#ifdef __FreeBSD__ int lake_readdir( const char *path, void *buf, fuse_fill_dir_t filler, - off_t offset, struct fuse_file_info *fi) { + off_t offset, struct fuse_file_info *fi, enum fuse_readdir_flags flags) +#else +int lake_readdir( + const char *path, void *buf, fuse_fill_dir_t filler, + off_t offset, struct fuse_file_info *fi) +#endif +{ spdlog::trace("Reading directory {0}", path); // Return items in dir +#ifdef __FreeBSD__ + filler(buf, ".", nullptr, 0, FUSE_FILL_DIR_PLUS); + filler(buf, "..", nullptr, 0, FUSE_FILL_DIR_PLUS); +#else filler(buf, ".", nullptr, 0); filler(buf, "..", nullptr, 0); +#endif std::vector files; @@ -99,7 +116,11 @@ int lake_readdir( spdlog::trace("Will show file {0} as {1}", file, file_name); +#ifdef __FreeBSD__ + filler(buf, file_name.c_str(), nullptr, 0, FUSE_FILL_DIR_PLUS); +#else filler(buf, file_name.c_str(), nullptr, 0); +#endif } return 0; diff --git a/filesystem/src/fs.hpp b/filesystem/src/fs.hpp index 0f8b967..8c74005 100644 --- a/filesystem/src/fs.hpp +++ b/filesystem/src/fs.hpp @@ -6,14 +6,28 @@ extern "C" { #define FUSE_USE_VERSION 31 +#ifdef __FreeBSD__ +#include +#else #include +#endif } +#ifdef __FreeBSD__ +int lake_getattr(const char *path, struct stat *stbuf, struct fuse_file_info* fi); +#else int lake_getattr(const char *path, struct stat *stbuf); +#endif +#ifdef __FreeBSD__ +int lake_readdir( + const char *path, void *buf, fuse_fill_dir_t filler, + off_t offset, struct fuse_file_info *fi, enum fuse_readdir_flags flags); +#else int lake_readdir( const char *path, void *buf, fuse_fill_dir_t filler, off_t offset, struct fuse_file_info *fi); +#endif int lake_open(const char *path, struct fuse_file_info *fi); From c2d066f7c4d05f504e2a15e2970437ddc1941846 Mon Sep 17 00:00:00 2001 From: Caleb Depatie Date: Wed, 29 Oct 2025 21:16:32 -0400 Subject: [PATCH 12/23] Log to file and console --- filesystem/src/main.cpp | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/filesystem/src/main.cpp b/filesystem/src/main.cpp index f2ff47a..aafbde7 100644 --- a/filesystem/src/main.cpp +++ b/filesystem/src/main.cpp @@ -2,10 +2,9 @@ // // SPDX-License-Identifier: BSD-3-Clause -#include "spdlog/common.h" #include #include -#include +#include #include #include #include @@ -13,7 +12,10 @@ #include #include -#include +#include +#include +#include +// #include extern "C" { #include @@ -117,8 +119,18 @@ auto main(int argc, char** argv) -> int { db_set_default_query(default_query); // Initialize file logger - // auto file_logger = spdlog::basic_logger_mt("file_logger", "lakefs.log"); - // spdlog::set_default_logger(file_logger); + const std::string log_file_name = "/var/lakefs/lakefs.log"; + + // create a rotating logger with a 5MB file size and three files + const auto max_size = 1048576 * 5; + const auto max_files = 3; + auto rotating_sink = std::make_shared(log_file_name, max_size, max_files); + auto console_sink = std::make_shared(); + + spdlog::logger logger("default", {console_sink, rotating_sink}); + spdlog::default_logger()->swap(logger); + + // spdlog::set_default_logger(logger_s); // Fuse gets initiated like a program and needs its own args fuse_args args = FUSE_ARGS_INIT(0, nullptr); From 68568b695ecd19b36229498311f90d236463f0fd Mon Sep 17 00:00:00 2001 From: Caleb Depatie Date: Wed, 29 Oct 2025 21:23:52 -0400 Subject: [PATCH 13/23] Limited action permissions --- .github/workflows/reuse.yml | 2 ++ .github/workflows/testing.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/reuse.yml b/.github/workflows/reuse.yml index 2aa7892..1de93d8 100644 --- a/.github/workflows/reuse.yml +++ b/.github/workflows/reuse.yml @@ -3,6 +3,8 @@ # SPDX-License-Identifier: CC0-1.0 name: REUSE Compliance Check +permissions: + contents: read on: [ push, pull_request ] diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index e8ab0df..b7faf28 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -3,6 +3,8 @@ # SPDX-License-Identifier: 0BSD name: Testing Suite +permissions: + contents: read on: [ push ] From 5d944ef3ff20d575b58d1c883c168330d87afcea Mon Sep 17 00:00:00 2001 From: Caleb Depatie Date: Mon, 10 Nov 2025 19:42:05 -0500 Subject: [PATCH 14/23] Fixed #36 --- filesystem/src/db.cpp | 24 ++++++++++++++++++++++-- filesystem/src/main.cpp | 2 -- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/filesystem/src/db.cpp b/filesystem/src/db.cpp index cbbffee..cc3a8f1 100644 --- a/filesystem/src/db.cpp +++ b/filesystem/src/db.cpp @@ -144,7 +144,16 @@ std::optional db_query_helper(const std::shared_ptr ast) { query_part += tmp_part.value(); } else if (auto intersection_op = std::dynamic_pointer_cast(ast)) { - query_part += "IN ("; + // TODO: Unfortunate special case for NOT IN + if (!std::dynamic_pointer_cast(intersection_op->left_node)) + { + query_part += "IN ("; + } + else + { + query_part += ""; + } + auto tmp_part = db_query_helper(intersection_op->left_node); if (!tmp_part.has_value()) { @@ -152,7 +161,18 @@ std::optional db_query_helper(const std::shared_ptr ast) { } query_part += tmp_part.value(); - query_part += ") AND id IN ("; + + // TODO: Unfortunate special case for NOT IN + if (!std::dynamic_pointer_cast(intersection_op->right_node)) + { + query_part += ") AND id IN ("; + } + else + { + query_part += ") AND id "; + } + + tmp_part = db_query_helper(intersection_op->right_node); if (!tmp_part.has_value()) { diff --git a/filesystem/src/main.cpp b/filesystem/src/main.cpp index aafbde7..2a2da76 100644 --- a/filesystem/src/main.cpp +++ b/filesystem/src/main.cpp @@ -130,8 +130,6 @@ auto main(int argc, char** argv) -> int { spdlog::logger logger("default", {console_sink, rotating_sink}); spdlog::default_logger()->swap(logger); - // spdlog::set_default_logger(logger_s); - // Fuse gets initiated like a program and needs its own args fuse_args args = FUSE_ARGS_INIT(0, nullptr); From 3a65e55b18533c9a275a705f901e1e7e54fd1420 Mon Sep 17 00:00:00 2001 From: Caleb Depatie Date: Mon, 10 Nov 2025 21:00:25 -0500 Subject: [PATCH 15/23] Improved backup system and fixed #38 --- filesystem/src/backups.cpp | 63 +++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/filesystem/src/backups.cpp b/filesystem/src/backups.cpp index 37ce0e2..f424006 100644 --- a/filesystem/src/backups.cpp +++ b/filesystem/src/backups.cpp @@ -5,14 +5,15 @@ #include "backups.hpp" #include "db.hpp" +#include #include -#include #include #include #include #include +#include // better way to pass these in? handler args uint32_t _number_backups; @@ -92,47 +93,45 @@ static auto handle_backup(sigval val) -> void { if (backup_count > _number_backups) { spdlog::info("Removing oldest backup"); - dir_iter = std::filesystem::directory_iterator(_backup_dir); - auto oldest_entry = std::make_optional(); - std::tm oldest_entry_date; - - for (auto entry : dir_iter) { - - if (entry.is_regular_file()) { - - // Get time from filename - const auto new_entry_name = entry.path().stem(); - - // TODO: if this is removed, the backup fails! - spdlog::debug("Looking at file {0} {1}", entry.path().c_str(), entry.path().stem().c_str()); + // Derives the time from the file name + const auto get_time = [] (const std::filesystem::path entry) -> time_t + { + const auto file_stem = entry.stem(); - std::tm new_entry_date; - strptime(new_entry_name.c_str(), "%Y-%m-%d.%X", &new_entry_date); + spdlog::debug("Looking at file {0} {1}", entry.c_str(), entry.stem().c_str()); + + std::tm file_date; + strptime(file_stem.c_str(), "%Y-%m-%d.%X", &file_date); + + return mktime(&file_date); + }; - if (!oldest_entry.has_value()) { - oldest_entry = entry; - oldest_entry_date = new_entry_date; - continue; - } + dir_iter = std::filesystem::directory_iterator(_backup_dir); - if (difftime(mktime(&new_entry_date), mktime(&oldest_entry_date)) < 0) { - oldest_entry = entry; - oldest_entry_date = new_entry_date; - } + // Placing the iterator into a vector so its more straightforward to work with + std::vector files{}; + for (auto entry : dir_iter) { + const auto filename = entry.path().filename(); + if (entry.is_regular_file() && (filename.string().ends_with(".backup.db"))) { + files.push_back(entry.path()); } } - if (oldest_entry.has_value()) { - if (std::filesystem::remove(oldest_entry->path())) { - spdlog::info("Removed file at {0}", oldest_entry->path().c_str()); + std::sort(files.begin(), files.end(), + [get_time](const std::filesystem::path& entry_a, const std::filesystem::path& entry_b) -> bool { + return get_time(entry_a) < get_time(entry_b); + }); + + // remove files + for (int i = 0; i < (backup_count - _number_backups); i++) + { + if (std::filesystem::remove(files[i])) { + spdlog::info("Removed file at {0}", files[i].c_str()); } else { - spdlog::error("Could not remove file at {0}", oldest_entry->path().c_str()); + spdlog::error("Could not remove file at {0}",files[i].c_str()); } - - } else { - spdlog::error("Could not remove entry, no file found"); } } From 1b6226131930ea2d6ddf17933f7ec42c745d1469 Mon Sep 17 00:00:00 2001 From: Caleb Depatie Date: Tue, 11 Nov 2025 14:03:04 -0500 Subject: [PATCH 16/23] Added a string splitting util function --- filesystem/src/utilities.cpp | 30 ++++++++++++++++++++++++++++++ filesystem/src/utilities.hpp | 6 +++++- filesystem/tests/utilities.cpp | 26 +++++++++++++++++++++++++- 3 files changed, 60 insertions(+), 2 deletions(-) diff --git a/filesystem/src/utilities.cpp b/filesystem/src/utilities.cpp index a6f5071..e0cab75 100644 --- a/filesystem/src/utilities.cpp +++ b/filesystem/src/utilities.cpp @@ -72,4 +72,34 @@ std::string extract_query(const char* path) { spdlog::trace("Exiting parse() -> {0}", extracted_query); return extracted_query; +} + +std::vector split(const std::string str, const std::string delim) { + spdlog::trace("Entering split(str={0}, delim={1})", str, delim); + + std::vector parts; + + int part_start = 0; + for (int i = 0; i < str.size(); i++) + { + const std::string delim_match = str.substr(i, delim.size()); + + if (delim_match == delim) + { + if (part_start != i) + { + parts.push_back(str.substr(part_start, i-part_start)); + } + + part_start = i + delim.size(); + } + } + + if (part_start < str.size()) + { + parts.push_back(str.substr(part_start, str.size() - part_start)); + } + + spdlog::trace("Exiting split() -> vec.size()={0}", parts.size()); + return parts; } \ No newline at end of file diff --git a/filesystem/src/utilities.hpp b/filesystem/src/utilities.hpp index 8d386e8..7e9e69e 100644 --- a/filesystem/src/utilities.hpp +++ b/filesystem/src/utilities.hpp @@ -4,9 +4,13 @@ #pragma once #include +#include // Takes a query path that involves a tag query and returns the real FS path std::string reverse_query(const char* path); // Takes a path and returns the query segment -std::string extract_query(const char* path); \ No newline at end of file +std::string extract_query(const char* path); + +// Split a string based on a delimeter +std::vector split(const std::string str, const std::string delim); \ No newline at end of file diff --git a/filesystem/tests/utilities.cpp b/filesystem/tests/utilities.cpp index 490a324..5887636 100644 --- a/filesystem/tests/utilities.cpp +++ b/filesystem/tests/utilities.cpp @@ -43,4 +43,28 @@ TEST_CASE("Query Extraction", "[utilities]") { CHECK(res == "(tag1 & ((!place) | (tag2 & tag3)))"); } -} \ No newline at end of file +} + +TEST_CASE("String Splitting", "[utilities]") { + SECTION("Single char delim") { + const auto haystack = "wow/this/is/a/path/"; + const auto needle = "/"; + + const auto found = split(haystack, needle); + + CHECK(found.size() == 5); + CHECK(found[0] == "wow"); + CHECK(found[4] == "path"); + } + + SECTION("Multi char delim") { + const auto haystack = "> Date: Tue, 11 Nov 2025 15:10:39 -0500 Subject: [PATCH 17/23] Working folder tags; Resolved #51 --- filesystem/src/fs.cpp | 59 +++++------- filesystem/src/utilities.cpp | 169 ++++++++++++++++++++++++++++----- filesystem/src/utilities.hpp | 6 +- integration_tests/adv_query.sh | 2 +- integration_tests/folder.sh | 43 +++++++++ integration_tests/meson.build | 3 +- 6 files changed, 221 insertions(+), 61 deletions(-) create mode 100644 integration_tests/folder.sh diff --git a/filesystem/src/fs.cpp b/filesystem/src/fs.cpp index 8e328e4..8a28623 100644 --- a/filesystem/src/fs.cpp +++ b/filesystem/src/fs.cpp @@ -17,9 +17,8 @@ extern "C" { #include #include -#include "query_lang/parser.hpp" -#include "fs.hpp" #include "db.hpp" +#include "fs.hpp" #include "command_interface.h" #include "utilities.hpp" @@ -41,15 +40,20 @@ int lake_getattr(const char *path, struct stat *stbuf) auto file = reverse_query(path); - if (file.empty()) { + if (!file.has_value()) + { + return file.error(); + } + + if (file.value().empty()) { spdlog::error("No file found!"); return -ENOENT; } - spdlog::debug("stat'ing file at {0}", file); + spdlog::debug("stat'ing file at {0}", file.value()); - if (stat(file.c_str(), stbuf) < 0) + if (stat(file.value().c_str(), stbuf) < 0) { spdlog::error("Error stat'ing: {0}", strerror(errno)); @@ -82,36 +86,14 @@ int lake_readdir( filler(buf, "..", nullptr, 0); #endif - std::vector files; - - // TODO: very nested, smelly - - // Check if path is a query - if (path[strlen(path) - 1] == ')') { - std::string query = extract_query(path); - - try { - const auto query_ast = parse(query); - - if (query_ast.has_value()) { - files = db_run_query(query_ast.value()); - - } else { - spdlog::error("Error while parsing, invalid query"); + const auto files = get_files(path); - return -EINVAL; - } - } catch (std::exception err) { - spdlog::error("Error while parsing: {0}", err.what()); - - return -EINVAL; - } - - } else { - files = db_run_default_query(); + if (!files.has_value()) + { + return files.error(); } - for (const auto& file : files) { + for (const auto& file : files.value()) { const std::string file_name = file.substr(file.find_last_of("/") + 1); spdlog::trace("Will show file {0} as {1}", file, file_name); @@ -136,12 +118,19 @@ int lake_open(const char *path, struct fuse_file_info *fi) { auto file_path = reverse_query(path); - if (file_path.empty()) + if (!file_path.has_value()) + { + return file_path.error(); + } + + if (file_path.value().empty()) + { return -ENOENT; + } - spdlog::trace("Found file {0} to open", file_path); + spdlog::trace("Found file {0} to open", file_path.value()); - fi->fh = open(file_path.c_str(), fi->flags); + fi->fh = open(file_path.value().c_str(), fi->flags); if (fi->fh == -1) { spdlog::error("Could not open file err: {0}", strerror(errno)); diff --git a/filesystem/src/utilities.cpp b/filesystem/src/utilities.cpp index e0cab75..7335582 100644 --- a/filesystem/src/utilities.cpp +++ b/filesystem/src/utilities.cpp @@ -3,50 +3,153 @@ // SPDX-License-Identifier: BSD-3-Clause #include #include +#include #include "query_lang/parser.hpp" #include "db.hpp" #include "utilities.hpp" -std::string reverse_query(const char* path) { - auto path_s = std::string(path); +std::expected, int> get_files(const char* path) { + spdlog::trace("Entering get_files(path={0})", path); - spdlog::trace("Entering reverse_query(path={0})", path_s); + std::vector files; - std::vector query_files; + // Get path segments from query + const auto path_segments = split(path, "/"); - if (path_s.find_first_of('(') == std::string::npos) { - query_files = db_run_default_query(); + const bool path_has_query = path_segments.size() != 0 && path_segments[0].contains('('); - } else { - std::string query = extract_query(path); + if (path_has_query) + { + const auto query = extract_query(path); + const auto query_ast = parse(query); + + if (query_ast.has_value()) { + files = db_run_query(query_ast.value()); + + } else { + spdlog::error("Error while parsing, invalid query"); + + return std::unexpected(-EINVAL); + } + } + else + { + files = db_run_default_query(); + } + + if (path_segments.size() > 1) + { + + // Get the file path by comparing the file name to the query results + std::string file_path; + std::string looking_for_file; + + if (path_has_query) + { + looking_for_file = path_segments[1]; + } + else + { + looking_for_file = path_segments[0]; + } - try { - const auto query_ast = parse(query); - if (query_ast.has_value()) { - query_files = db_run_query(query_ast.value()); + for (const auto& query_file : files) { - } else { - spdlog::error("Error while parsing, no value returned"); + const std::string query_file_name = + query_file.substr(query_file.find_last_of("/") + 1); + + if (query_file_name == looking_for_file) { + file_path = query_file; + break; + } + } + + // Have to drill down arbitry path lengths + // Take current real path, get dir entries + // look for next folder, get that real path + // repeat + for (int i = 1 + path_has_query; i < path_segments.size(); i++) + { + auto finding_dir = path_segments[i]; + + const auto dir_iter = std::filesystem::directory_iterator(file_path); + + for (const auto& dir_entry : dir_iter) + { + if (dir_entry.path().filename() == finding_dir) + { + file_path = dir_entry.path(); + break; + } + } + } + + if (std::filesystem::is_directory(file_path)) + { + // Query is for a folder + spdlog::debug("Found real dir path: {0}", file_path.c_str()); + + // Get files of our actual destination folder + files.clear(); - return ""; + for (auto const& dir_entry : std::filesystem::directory_iterator{file_path}) + { + files.push_back(dir_entry.path()); } + } + } + + spdlog::trace("Exiting get_files() -> vec.size()={0}", files.size()); + return files; +} + +std::expected reverse_query(const char* path) { + auto path_s = std::string(path); + + spdlog::trace("Entering reverse_query(path={0})", path_s); + + // TODO: Refactor & reuse this shared code once working for subdirs + std::vector files; + + // Get path segments from query + const auto path_segments = split(path, "/"); + + const bool path_has_query = path_segments[0].contains('('); + + if (path_has_query) + { + const auto query = extract_query(path); + const auto query_ast = parse(query); + + if (query_ast.has_value()) { + files = db_run_query(query_ast.value()); - } catch (std::exception err) { - spdlog::error("Error while parsing: {0}", err.what()); - - return ""; + } else { + spdlog::error("Error while parsing, invalid query"); + + return std::unexpected(-EINVAL); } + } else { + files = db_run_default_query(); } // Get the file path by comparing the file name to the query results std::string file_path; + std::string looking_for_file; + + if (path_has_query) + { + looking_for_file = path_segments[1]; + } + else + { + looking_for_file = path_segments[0]; + } - const std::string looking_for_file = - std::string(path_s).substr(std::string(path_s).find_last_of("/") + 1); - for (const auto& query_file : query_files) { + for (const auto& query_file : files) { const std::string query_file_name = query_file.substr(query_file.find_last_of("/") + 1); @@ -57,6 +160,26 @@ std::string reverse_query(const char* path) { } } + // Have to drill down arbitry path lengths + // Take current real path, get dir entries + // look for next folder, get that real path + // repeat + for (int i = 1 + path_has_query; i < path_segments.size(); i++) + { + auto finding_dir = path_segments[i]; + + const auto dir_iter = std::filesystem::directory_iterator(file_path); + + for (const auto& dir_entry : dir_iter) + { + if (dir_entry.path().filename() == finding_dir) + { + file_path = dir_entry.path(); + break; + } + } + } + spdlog::trace("Exiting reverse_query() -> {0}", file_path); return file_path; } @@ -70,7 +193,7 @@ std::string extract_query(const char* path) { const auto extracted_query = path_s.substr(path_s.find_first_of('('), path_s.find_last_of(')') - path_s.find_first_of('(') + 1); - spdlog::trace("Exiting parse() -> {0}", extracted_query); + spdlog::trace("Exiting extract_query() -> {0}", extracted_query); return extracted_query; } diff --git a/filesystem/src/utilities.hpp b/filesystem/src/utilities.hpp index 7e9e69e..530f9a9 100644 --- a/filesystem/src/utilities.hpp +++ b/filesystem/src/utilities.hpp @@ -5,9 +5,13 @@ #include #include +#include + +// Runs a query normally. Handles special logic to pull out folders +std::expected, int> get_files(const char* path); // Takes a query path that involves a tag query and returns the real FS path -std::string reverse_query(const char* path); +std::expected reverse_query(const char* path); // Takes a path and returns the query segment std::string extract_query(const char* path); diff --git a/integration_tests/adv_query.sh b/integration_tests/adv_query.sh index 08e9d82..32f4d89 100644 --- a/integration_tests/adv_query.sh +++ b/integration_tests/adv_query.sh @@ -8,7 +8,7 @@ source integration_tests/test_core.sh # Issue #34 :: Exit gracefully with an incorrect query # TODO: Seems to work but I only get 0 in $rc. -# /usr/bin/env ls $lake_dir/'((default | not_default) & )' +# ls $lake_dir/'((default | not_default) & )' && echo 'not ok!!' # rc=$? # if [ $rc != 1 ]; then diff --git a/integration_tests/folder.sh b/integration_tests/folder.sh new file mode 100644 index 0000000..8042e1f --- /dev/null +++ b/integration_tests/folder.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: 2025 Caleb Depatie +# +# SPDX-License-Identifier: 0BSD + +source integration_tests/test_core.sh + +mkdir $test_dir/tagged_dir +touch $test_dir/tagged_dir/makefile +mkdir $test_dir/tagged_dir/src +touch $test_dir/tagged_dir/src/a.c +touch $test_dir/tagged_dir/src/b.c +touch $test_dir/tagged_dir/src/c.c + +$cli add $test_dir/tagged_dir +$cli tag $test_dir/tagged_dir its_a_folder + +# Reading from a folder +results="$(ls -A $lake_dir/'(its_a_folder)'/tagged_dir/ | wc -l)" + +if [ $(echo "$results" | xargs) != "2" ]; then + echo "Error: Tagging a folder not working" + echo "Expected: 2" + echo "Got: $results" + echo "ls: $(ls -A $lake_dir/'(its_a_folder)'/tagged_dir/)" + + cleanup_and_exit 1 +fi + +# Nested folder reading +results="$(ls -A $lake_dir/'(its_a_folder)'/tagged_dir/src/ | wc -l)" + +if [ $(echo "$results" | xargs) != "3" ]; then + echo "Error: Reading a nested folder not working!" + echo "Expected: 3" + echo "Got: $results" + echo "ls: $(ls -A $lake_dir/'(its_a_folder)'/tagged_dir/src/)" + + cleanup_and_exit 1 +fi + +cleanup_and_exit 0 \ No newline at end of file diff --git a/integration_tests/meson.build b/integration_tests/meson.build index 0819fbf..e4f6ac2 100644 --- a/integration_tests/meson.build +++ b/integration_tests/meson.build @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2024 Caleb Depatie +# SPDX-FileCopyrightText: 2024-2025 Caleb Depatie # # SPDX-License-Identifier: 0BSD @@ -9,6 +9,7 @@ tests_dict = { 'Deleting Files': 'deletion.sh', 'Relinking Files': 'relinking.sh', 'Add multiple tags': 'multiple_tags.sh', + 'Tag Folders': 'folder.sh', 'Advanced Queries': 'adv_query.sh' } From c697566d0878c7f60a862d125b892923e1300efb Mon Sep 17 00:00:00 2001 From: Caleb Depatie Date: Tue, 11 Nov 2025 15:20:08 -0500 Subject: [PATCH 18/23] Resolved test runner issue with creating log files --- filesystem/src/main.cpp | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/filesystem/src/main.cpp b/filesystem/src/main.cpp index 2a2da76..a1c61c8 100644 --- a/filesystem/src/main.cpp +++ b/filesystem/src/main.cpp @@ -119,16 +119,19 @@ auto main(int argc, char** argv) -> int { db_set_default_query(default_query); // Initialize file logger - const std::string log_file_name = "/var/lakefs/lakefs.log"; + if (!program.get("--tempdb")) + { + const std::string log_file_name = "/var/lakefs/lakefs.log"; + + // create a rotating logger with a 5MB file size and three files + const auto max_size = 1048576 * 5; + const auto max_files = 3; + auto rotating_sink = std::make_shared(log_file_name, max_size, max_files); + auto console_sink = std::make_shared(); - // create a rotating logger with a 5MB file size and three files - const auto max_size = 1048576 * 5; - const auto max_files = 3; - auto rotating_sink = std::make_shared(log_file_name, max_size, max_files); - auto console_sink = std::make_shared(); - - spdlog::logger logger("default", {console_sink, rotating_sink}); - spdlog::default_logger()->swap(logger); + spdlog::logger logger("default", {console_sink, rotating_sink}); + spdlog::default_logger()->swap(logger); + } // Fuse gets initiated like a program and needs its own args fuse_args args = FUSE_ARGS_INIT(0, nullptr); From 5301fa7e1bd8bd64c4abc8321fcea2ca1800a560 Mon Sep 17 00:00:00 2001 From: Caleb Depatie Date: Tue, 11 Nov 2025 15:43:56 -0500 Subject: [PATCH 19/23] Refactored utilities.cpp to share code --- filesystem/src/utilities.cpp | 76 +++++++----------------------------- 1 file changed, 15 insertions(+), 61 deletions(-) diff --git a/filesystem/src/utilities.cpp b/filesystem/src/utilities.cpp index 7335582..ccf5862 100644 --- a/filesystem/src/utilities.cpp +++ b/filesystem/src/utilities.cpp @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2025 Caleb Depatie // // SPDX-License-Identifier: BSD-3-Clause +#include #include #include #include @@ -110,78 +111,31 @@ std::expected reverse_query(const char* path) { spdlog::trace("Entering reverse_query(path={0})", path_s); - // TODO: Refactor & reuse this shared code once working for subdirs - std::vector files; - - // Get path segments from query const auto path_segments = split(path, "/"); - const bool path_has_query = path_segments[0].contains('('); - - if (path_has_query) - { - const auto query = extract_query(path); - const auto query_ast = parse(query); - - if (query_ast.has_value()) { - files = db_run_query(query_ast.value()); - - } else { - spdlog::error("Error while parsing, invalid query"); - - return std::unexpected(-EINVAL); - } - } else { - files = db_run_default_query(); - } + std::string looking_for_file = path_segments[path_segments.size()-1]; - // Get the file path by comparing the file name to the query results - std::string file_path; - std::string looking_for_file; + std::string reconstructed_path; - if (path_has_query) - { - looking_for_file = path_segments[1]; - } - else + for (int i = 0; i < path_segments.size()-1; i++) { - looking_for_file = path_segments[0]; + reconstructed_path += "/" + path_segments[i]; } + const auto files = get_files(reconstructed_path.c_str()); - for (const auto& query_file : files) { - - const std::string query_file_name = - query_file.substr(query_file.find_last_of("/") + 1); - - if (query_file_name == looking_for_file) { - file_path = query_file; - break; - } - } - - // Have to drill down arbitry path lengths - // Take current real path, get dir entries - // look for next folder, get that real path - // repeat - for (int i = 1 + path_has_query; i < path_segments.size(); i++) + if (!files.has_value()) { - auto finding_dir = path_segments[i]; - - const auto dir_iter = std::filesystem::directory_iterator(file_path); - - for (const auto& dir_entry : dir_iter) - { - if (dir_entry.path().filename() == finding_dir) - { - file_path = dir_entry.path(); - break; - } - } + return std::unexpected(files.error()); } - spdlog::trace("Exiting reverse_query() -> {0}", file_path); - return file_path; + const auto file_path = std::find_if(files->begin(), files->end(), + [looking_for_file] (const std::string& file_name) -> bool { + return std::filesystem::path(file_name).filename() == looking_for_file; + }); + + spdlog::trace("Exiting reverse_query() -> {0}", *file_path); + return *file_path; } std::string extract_query(const char* path) { From c2d6658f3e53cf740585de7cce037827156191e6 Mon Sep 17 00:00:00 2001 From: Caleb Depatie Date: Tue, 11 Nov 2025 16:14:59 -0500 Subject: [PATCH 20/23] Added clang-format --- .clang-format | 15 ++ .github/workflows/format.yml | 25 ++++ filesystem/src/backups.cpp | 75 +++++----- filesystem/src/backups.hpp | 5 +- filesystem/src/command_interface.h | 17 +-- filesystem/src/config.cpp | 49 ++++--- filesystem/src/config.hpp | 4 +- filesystem/src/control.cpp | 196 ++++++++++++++------------ filesystem/src/db.cpp | 191 ++++++++++++++++--------- filesystem/src/db.hpp | 2 +- filesystem/src/fs.cpp | 87 +++++++----- filesystem/src/fs.hpp | 28 ++-- filesystem/src/main.cpp | 161 +++++++++++---------- filesystem/src/utilities.cpp | 57 ++++---- filesystem/src/utilities.hpp | 2 +- filesystem/tests/config.cpp | 32 +++-- filesystem/tests/parsing.cpp | 90 +++++------- filesystem/tests/query_generation.cpp | 50 +++---- filesystem/tests/utilities.cpp | 43 +++--- filesystem/tests/vendors/sqlite.cpp | 29 ++-- makefile | 6 + 21 files changed, 666 insertions(+), 498 deletions(-) create mode 100644 .clang-format create mode 100644 .github/workflows/format.yml diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..9164b0c --- /dev/null +++ b/.clang-format @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: 2025 Caleb Depatie +# +# SPDX-License-Identifier: 0BSD + +BasedOnStyle: LLVM +IndentWidth: 4 + +ColumnLimit: 100 + +# Force pointers to the type for C++. +DerivePointerAlignment: false +PointerAlignment: Left + +BreakStringLiterals: true +BreakBeforeBraces: Allman \ No newline at end of file diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml new file mode 100644 index 0000000..944037c --- /dev/null +++ b/.github/workflows/format.yml @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: 2025 Caleb Depatie +# +# SPDX-License-Identifier: 0BSD + +name: Format Checker +permissions: + contents: read + +on: [push] +jobs: + formatting-check: + name: Clang Formatting Check + runs-on: ubuntu-latest + strategy: + matrix: + path: + - 'filesystem/src' + - 'filesystem/tests' + steps: + - uses: actions/checkout@v4 + - name: Run clang-format style check + uses: jidicula/clang-format-action@v4.16.0 + with: + clang-format-version: '13' + check-path: ${{ matrix.path }} \ No newline at end of file diff --git a/filesystem/src/backups.cpp b/filesystem/src/backups.cpp index f424006..2b14745 100644 --- a/filesystem/src/backups.cpp +++ b/filesystem/src/backups.cpp @@ -21,15 +21,16 @@ std::string _backup_dir; static auto handle_backup(sigval val) -> void; -auto create_backup_timer(std::chrono::seconds interval, uint32_t number_backups, std::string backup_path) -> bool { - spdlog::info("Setting up backups to keep {0} copies and run every {1} hours", - number_backups, - std::chrono::duration_cast(interval).count()); +auto create_backup_timer(std::chrono::seconds interval, uint32_t number_backups, + std::string backup_path) -> bool +{ + spdlog::info("Setting up backups to keep {0} copies and run every {1} hours", number_backups, + std::chrono::duration_cast(interval).count()); _number_backups = number_backups; _backup_dir = backup_path; - // Create a C timer + // Create a C timer sigevent event; pthread_attr_t backup_thread_attr; @@ -41,7 +42,8 @@ auto create_backup_timer(std::chrono::seconds interval, uint32_t number_backups, event.sigev_notify_attributes = &backup_thread_attr; timer_t timer_id; - if (timer_create(CLOCK_MONOTONIC, &event, &timer_id) == -1) { + if (timer_create(CLOCK_MONOTONIC, &event, &timer_id) == -1) + { spdlog::critical("Could not create timer! {0}", strerror(errno)); return false; } @@ -52,8 +54,9 @@ auto create_backup_timer(std::chrono::seconds interval, uint32_t number_backups, spdlog::debug("Making timer for {0} seconds", timer_spec.it_interval.tv_sec); - //start timer - if (timer_settime(timer_id, 0, &timer_spec, nullptr) == -1) { + // start timer + if (timer_settime(timer_id, 0, &timer_spec, nullptr) == -1) + { spdlog::critical("Could not start timer! {0}", strerror(errno)); return false; } @@ -61,12 +64,13 @@ auto create_backup_timer(std::chrono::seconds interval, uint32_t number_backups, return true; } -static auto handle_backup(sigval val) -> void { +static auto handle_backup(sigval val) -> void +{ spdlog::info("Starting Backup..."); const auto now = time(0); const auto current_time = *std::localtime(&now); - char buf [128]; + char buf[128]; strftime(buf, sizeof(buf), "%Y-%m-%d.%X", ¤t_time); @@ -80,57 +84,62 @@ static auto handle_backup(sigval val) -> void { // delete files if needed.. auto dir_iter = std::filesystem::directory_iterator(_backup_dir); - int backup_count = std::count_if( - begin(dir_iter), - end(dir_iter), - [](auto& entry) { - const auto filename = entry.path().filename(); + int backup_count = std::count_if(begin(dir_iter), end(dir_iter), + [](auto& entry) + { + const auto filename = entry.path().filename(); - return entry.is_regular_file() && (filename.string().ends_with(".backup.db")); - } - ); + return entry.is_regular_file() && + (filename.string().ends_with(".backup.db")); + }); - if (backup_count > _number_backups) { + if (backup_count > _number_backups) + { spdlog::info("Removing oldest backup"); // Derives the time from the file name - const auto get_time = [] (const std::filesystem::path entry) -> time_t + const auto get_time = [](const std::filesystem::path entry) -> time_t { const auto file_stem = entry.stem(); spdlog::debug("Looking at file {0} {1}", entry.c_str(), entry.stem().c_str()); - + std::tm file_date; strptime(file_stem.c_str(), "%Y-%m-%d.%X", &file_date); - + return mktime(&file_date); }; dir_iter = std::filesystem::directory_iterator(_backup_dir); - // Placing the iterator into a vector so its more straightforward to work with + // Placing the iterator into a vector so its more straightforward to + // work with std::vector files{}; - for (auto entry : dir_iter) { + for (auto entry : dir_iter) + { const auto filename = entry.path().filename(); - if (entry.is_regular_file() && (filename.string().ends_with(".backup.db"))) { + if (entry.is_regular_file() && (filename.string().ends_with(".backup.db"))) + { files.push_back(entry.path()); } } - std::sort(files.begin(), files.end(), - [get_time](const std::filesystem::path& entry_a, const std::filesystem::path& entry_b) -> bool { - return get_time(entry_a) < get_time(entry_b); - }); + std::sort(files.begin(), files.end(), + [get_time](const std::filesystem::path& entry_a, + const std::filesystem::path& entry_b) -> bool + { return get_time(entry_a) < get_time(entry_b); }); // remove files for (int i = 0; i < (backup_count - _number_backups); i++) { - if (std::filesystem::remove(files[i])) { + if (std::filesystem::remove(files[i])) + { spdlog::info("Removed file at {0}", files[i].c_str()); - - } else { - spdlog::error("Could not remove file at {0}",files[i].c_str()); + } + else + { + spdlog::error("Could not remove file at {0}", files[i].c_str()); } } } diff --git a/filesystem/src/backups.hpp b/filesystem/src/backups.hpp index 69cd09c..d4dc368 100644 --- a/filesystem/src/backups.hpp +++ b/filesystem/src/backups.hpp @@ -4,7 +4,8 @@ #pragma once #include -#include #include +#include -auto create_backup_timer(std::chrono::seconds interval, uint32_t number_backups, std::string backuppath) -> bool; \ No newline at end of file +auto create_backup_timer(std::chrono::seconds interval, uint32_t number_backups, + std::string backuppath) -> bool; \ No newline at end of file diff --git a/filesystem/src/command_interface.h b/filesystem/src/command_interface.h index 182dc70..25ebd45 100644 --- a/filesystem/src/command_interface.h +++ b/filesystem/src/command_interface.h @@ -4,10 +4,11 @@ #pragma once #ifdef __cplusplus -extern "C" { +extern "C" +{ #endif -// Command Interface + // Command Interface #define LAKE_SOCKET_PATH "/tmp/lakefs.sock" @@ -18,13 +19,13 @@ extern "C" { #define LAKE_REMOVE_TAG 4 #define LAKE_RELINK_FILE 5 -typedef struct __attribute__((packed)) lake_command_t { - int command; - int size; - char data[]; -} lake_command_t; + typedef struct __attribute__((packed)) lake_command_t + { + int command; + int size; + char data[]; + } lake_command_t; #ifdef __cplusplus } #endif - diff --git a/filesystem/src/config.cpp b/filesystem/src/config.cpp index f49cd05..5b80048 100644 --- a/filesystem/src/config.cpp +++ b/filesystem/src/config.cpp @@ -4,25 +4,29 @@ #include "config.hpp" -#include #include +#include +#include #include #include -#include -auto etc_conf_reader(const std::string path) -> std::unordered_map { +auto etc_conf_reader(const std::string path) -> std::unordered_map +{ std::unordered_map config; std::ifstream file(path); - if (!file.is_open()) { + if (!file.is_open()) + { spdlog::error("Failed to open {0}", path); return config; } std::string line; - while (std::getline(file, line)) { - if (line[0] == '#') { + while (std::getline(file, line)) + { + if (line[0] == '#') + { continue; } @@ -36,7 +40,8 @@ auto etc_conf_reader(const std::string path) -> std::unordered_map std::pair { +auto parse_config_line(const std::string line) -> std::pair +{ std::string key; std::string value; @@ -47,34 +52,40 @@ auto parse_config_line(const std::string line) -> std::pair std::optional { +auto parse_interval_value(const std::string interval_string) -> std::optional +{ using namespace std::chrono; std::optional interval; const auto space_loc = interval_string.find_first_of(" "); const auto interval_length = std::stoi(interval_string.substr(0, space_loc)); - const auto interval_type = interval_string.substr(space_loc+1); + const auto interval_type = interval_string.substr(space_loc + 1); - if (interval_type == "days") { + if (interval_type == "days") + { auto day_interval = days(interval_length); interval = duration_cast(day_interval); - - } else if (interval_type == "hours") { + } + else if (interval_type == "hours") + { auto hour_interval = hours(interval_length); interval = duration_cast(hour_interval); - - } else if (interval_type == "weeks") { + } + else if (interval_type == "weeks") + { auto week_interval = weeks(interval_length); interval = duration_cast(week_interval); - - } else if (interval_type == "months") { + } + else if (interval_type == "months") + { auto month_interval = months(interval_length); interval = duration_cast(month_interval); - - } else { + } + else + { spdlog::critical("Unknown interval type: {0}", interval_type); } return interval; -} \ No newline at end of file +} \ No newline at end of file diff --git a/filesystem/src/config.hpp b/filesystem/src/config.hpp index aeccbc7..5711f6b 100644 --- a/filesystem/src/config.hpp +++ b/filesystem/src/config.hpp @@ -4,10 +4,10 @@ #pragma once -#include -#include #include #include +#include +#include auto etc_conf_reader(const std::string path) -> std::unordered_map; diff --git a/filesystem/src/control.cpp b/filesystem/src/control.cpp index e5ca8d5..2d1e385 100644 --- a/filesystem/src/control.cpp +++ b/filesystem/src/control.cpp @@ -2,26 +2,28 @@ // // SPDX-License-Identifier: BSD-3-Clause +#include #include #include -#include -#include -#include #include -#include +#include #include #include +#include +#include #include "command_interface.h" #include "control.hpp" #include "db.hpp" // Runs the socket server to control the FS -void control_server() { +void control_server() +{ // Create the socket int server_fd = socket(AF_UNIX, SOCK_STREAM, 0); - if (server_fd == -1) { + if (server_fd == -1) + { spdlog::critical("Failed to create socket"); exit(1); @@ -33,40 +35,47 @@ void control_server() { strncpy(addr.sun_path, LAKE_SOCKET_PATH, sizeof(addr.sun_path) - 1); - if (bind(server_fd, (struct sockaddr*)&addr, sizeof(addr)) == -1) { + if (bind(server_fd, (struct sockaddr*)&addr, sizeof(addr)) == -1) + { spdlog::critical("Failed to bind socket: {0}", strerror(errno)); exit(1); } // Listen on the socket - if (listen(server_fd, 5) == -1) { + if (listen(server_fd, 5) == -1) + { spdlog::critical("Failed to listen on socket"); exit(1); } // Accept connections - while (true) { + while (true) + { int client_fd = accept(server_fd, nullptr, nullptr); - if (client_fd == -1) { + if (client_fd == -1) + { spdlog::critical("Failed to accept connection"); exit(1); } - while (true) { + while (true) + { // Read the command char buffer[2048]; // TODO: Variable size!! int bytes_read = read(client_fd, buffer, sizeof(buffer)); - if (bytes_read == -1) { + if (bytes_read == -1) + { spdlog::critical("Failed to read from socket"); exit(1); } - if (bytes_read == 0) { + if (bytes_read == 0) + { // Socket disconnected spdlog::trace("Socket connection closed"); close(client_fd); @@ -75,98 +84,111 @@ void control_server() { // Parse the command lake_command_t* command = (lake_command_t*)buffer; - switch (command->command) { - case LAKE_ADD_FILE: { - std::string path = std::string(command->data, command->size); - - spdlog::info("Added file to database: {0}", path); + switch (command->command) + { + case LAKE_ADD_FILE: + { + std::string path = std::string(command->data, command->size); - int rc = db_add_file(path); + spdlog::info("Added file to database: {0}", path); - if (rc != SQLITE_OK) { - spdlog::error("Failed to add file to database: {0}", rc); - } + int rc = db_add_file(path); - break; - } - case LAKE_TAG_FILE: { - std::string cmd_data = std::string(command->data, command->size); - - const std::string delimiter = "\n"; - size_t pos = cmd_data.find(delimiter); - std::string path = cmd_data.substr(0, pos); - std::string tag = cmd_data.substr(pos + delimiter.length()); - - spdlog::info("Tagging file in database: {0} with tag: {1}", path, tag); - - int rc = db_tag_file(path, tag); - - if (rc != SQLITE_OK) { - spdlog::error("Failed to tag file in database: {0}", rc); - } - - break; + if (rc != SQLITE_OK) + { + spdlog::error("Failed to add file to database: {0}", rc); } - case LAKE_REMOVE_FILE: { - std::string path = std::string(command->data, command->size); - spdlog::info("Removing file from database: {0}", path); - int rc = db_remove_file(path); + break; + } + case LAKE_TAG_FILE: + { + std::string cmd_data = std::string(command->data, command->size); - if (rc != SQLITE_OK) { - spdlog::error("Failed to remove file from database: {0}", rc); - } + const std::string delimiter = "\n"; + size_t pos = cmd_data.find(delimiter); + std::string path = cmd_data.substr(0, pos); + std::string tag = cmd_data.substr(pos + delimiter.length()); - break; - } - case LAKE_REMOVE_TAG: { - std::string cmd_data = std::string(command->data, command->size); - - const std::string delimiter = "\n"; - size_t pos = cmd_data.find(delimiter); - std::string path = cmd_data.substr(0, pos); - std::string tag = cmd_data.substr(pos + delimiter.length()); - - spdlog::info("Removing tag from file in database: {0} with tag: {1}", path, tag); - - int rc = db_remove_tag(path, tag); - - if (rc != SQLITE_OK) { - spdlog::error("Failed to remove tag from file in database: {0}", rc); - } - - break; + spdlog::info("Tagging file in database: {0} with tag: {1}", path, tag); + + int rc = db_tag_file(path, tag); + + if (rc != SQLITE_OK) + { + spdlog::error("Failed to tag file in database: {0}", rc); } - case LAKE_RELINK_FILE: { - std::string cmd_data = std::string(command->data, command->size); - const std::string delimiter = "\n"; - size_t pos = cmd_data.find(delimiter); - std::string old_path = cmd_data.substr(0, pos); - std::string new_path = cmd_data.substr(pos + delimiter.length()); + break; + } + case LAKE_REMOVE_FILE: + { + std::string path = std::string(command->data, command->size); + spdlog::info("Removing file from database: {0}", path); - int rc = db_relink_file(old_path, new_path); + int rc = db_remove_file(path); - if (rc != SQLITE_OK) { - spdlog::error("Failed to relink file: {0}", rc); - } - - break; + if (rc != SQLITE_OK) + { + spdlog::error("Failed to remove file from database: {0}", rc); } - case LAKE_SET_DEFAULT_QUERY: { - std::string query = std::string(command->data, command->size); - spdlog::info("Setting default query to: {0}", query); - db_set_default_query(query); + break; + } + case LAKE_REMOVE_TAG: + { + std::string cmd_data = std::string(command->data, command->size); + + const std::string delimiter = "\n"; + size_t pos = cmd_data.find(delimiter); + std::string path = cmd_data.substr(0, pos); + std::string tag = cmd_data.substr(pos + delimiter.length()); + + spdlog::info("Removing tag from file in database: {0} with tag: {1}", path, tag); + + int rc = db_remove_tag(path, tag); - break; + if (rc != SQLITE_OK) + { + spdlog::error("Failed to remove tag from file in database: {0}", rc); } - default: { - spdlog::error("Unrecognized message received"); + break; + } + case LAKE_RELINK_FILE: + { + std::string cmd_data = std::string(command->data, command->size); - break; + const std::string delimiter = "\n"; + size_t pos = cmd_data.find(delimiter); + std::string old_path = cmd_data.substr(0, pos); + std::string new_path = cmd_data.substr(pos + delimiter.length()); + + int rc = db_relink_file(old_path, new_path); + + if (rc != SQLITE_OK) + { + spdlog::error("Failed to relink file: {0}", rc); } + + break; + } + case LAKE_SET_DEFAULT_QUERY: + { + std::string query = std::string(command->data, command->size); + spdlog::info("Setting default query to: {0}", query); + + db_set_default_query(query); + + break; + } + + default: + { + spdlog::error("Unrecognized message received"); + + break; + } }; } } diff --git a/filesystem/src/db.cpp b/filesystem/src/db.cpp index cc3a8f1..7d8c8ad 100644 --- a/filesystem/src/db.cpp +++ b/filesystem/src/db.cpp @@ -2,60 +2,70 @@ // // SPDX-License-Identifier: BSD-3-Clause +#include #include #include #include -#include -#include -#include #include "db.hpp" #include "query_lang/parser.hpp" +#include +#include std::string default_query = "default"; // The global database connection. // Perhaps better served by a singleton pattern. -static sqlite3 *db; +static sqlite3* db; // Initializes the DB in memory -int db_tmp_init() { +int db_tmp_init() +{ int rc = sqlite3_open(":memory:", &db); - rc = sqlite3_exec(db, "CREATE TABLE tags (data_id INTEGER, tag_value TEXT);", nullptr, nullptr, nullptr); + rc = sqlite3_exec(db, "CREATE TABLE tags (data_id INTEGER, tag_value TEXT);", nullptr, nullptr, + nullptr); - rc = sqlite3_exec(db, "CREATE TABLE data (id INTEGER PRIMARY KEY, path TEXT);", nullptr, nullptr, nullptr); + rc = sqlite3_exec(db, "CREATE TABLE data (id INTEGER PRIMARY KEY, path TEXT);", nullptr, + nullptr, nullptr); return rc; } -int db_init(const std::string db_file_path) { +int db_init(const std::string db_file_path) +{ // check if DB file exists before opening const bool db_exists = std::filesystem::exists(db_file_path + "/current.db"); int rc = sqlite3_open((db_file_path + "/current.db").c_str(), &db); - if (db_exists) { + if (db_exists) + { return rc; } // Setting up the DB tables if it doesnt already exist - rc = sqlite3_exec(db, "CREATE TABLE tags (data_id INTEGER, tag_value TEXT);", nullptr, nullptr, nullptr); + rc = sqlite3_exec(db, "CREATE TABLE tags (data_id INTEGER, tag_value TEXT);", nullptr, nullptr, + nullptr); - rc = sqlite3_exec(db, "CREATE TABLE data (id INTEGER PRIMARY KEY, path TEXT);", nullptr, nullptr, nullptr); + rc = sqlite3_exec(db, "CREATE TABLE data (id INTEGER PRIMARY KEY, path TEXT);", nullptr, + nullptr, nullptr); return rc; } -int db_close() { +int db_close() +{ int rc = sqlite3_close(db); return rc; } -int db_create_backup(const std::string backup_path) { +int db_create_backup(const std::string backup_path) +{ sqlite3* backup_db; - if (sqlite3_open(backup_path.c_str(), &backup_db) != SQLITE_OK) { + if (sqlite3_open(backup_path.c_str(), &backup_db) != SQLITE_OK) + { spdlog::error("Could not open new db at {0} for backup", backup_path); return -1; } @@ -63,19 +73,22 @@ int db_create_backup(const std::string backup_path) { sqlite3_backup* backup = NULL; // Will fail with NULL until we can aquire the read lock - while (backup == NULL) { + while (backup == NULL) + { backup = sqlite3_backup_init(backup_db, "main", db, "main"); - // spdlog::error("Could not initialize backup: {0}", sqlite3_errmsg(backup_db)); + // spdlog::error("Could not initialize backup: {0}", + // sqlite3_errmsg(backup_db)); } auto remaining_pages = 1; - while (remaining_pages > 0) { + while (remaining_pages > 0) + { sqlite3_backup_step(backup, remaining_pages); remaining_pages = sqlite3_backup_remaining(backup); - + spdlog::debug("Remaining pages to backup: {0}", remaining_pages); } @@ -85,51 +98,80 @@ int db_create_backup(const std::string backup_path) { return 0; } -int db_add_file(const std::string path) { - int rc = sqlite3_exec(db, ("INSERT INTO data (path) VALUES ('" + path + "');").c_str(), nullptr, nullptr, nullptr); +int db_add_file(const std::string path) +{ + int rc = sqlite3_exec(db, ("INSERT INTO data (path) VALUES ('" + path + "');").c_str(), nullptr, + nullptr, nullptr); return rc; } -int db_tag_file(const std::string path, const std::string tag) { - int rc = sqlite3_exec(db, ("INSERT INTO tags (data_id, tag_value) VALUES ((SELECT id FROM data WHERE path = '" + path + "'), '" + tag + "');").c_str(), nullptr, nullptr, nullptr); +int db_tag_file(const std::string path, const std::string tag) +{ + int rc = sqlite3_exec(db, + ("INSERT INTO tags (data_id, tag_value) VALUES " + "((SELECT id FROM data WHERE path = '" + + path + "'), '" + tag + "');") + .c_str(), + nullptr, nullptr, nullptr); return rc; } -int db_remove_file(const std::string path) { - int rc = sqlite3_exec(db, ("DELETE FROM tags WHERE data_id = (SELECT id FROM data WHERE path = '" + path + "');").c_str(), nullptr, nullptr, nullptr); - rc = sqlite3_exec(db, ("DELETE FROM data WHERE path = '" + path + "';").c_str(), nullptr, nullptr, nullptr); +int db_remove_file(const std::string path) +{ + int rc = sqlite3_exec(db, + ("DELETE FROM tags WHERE data_id = (SELECT id FROM " + "data WHERE path = '" + + path + "');") + .c_str(), + nullptr, nullptr, nullptr); + rc = sqlite3_exec(db, ("DELETE FROM data WHERE path = '" + path + "';").c_str(), nullptr, + nullptr, nullptr); return rc; } -int db_remove_tag(const std::string path, const std::string tag) { - int rc = sqlite3_exec(db, ("DELETE FROM tags WHERE data_id = (SELECT id FROM data WHERE path = '" + path + "') AND tag_value = '" + tag + "';").c_str(), nullptr, nullptr, nullptr); +int db_remove_tag(const std::string path, const std::string tag) +{ + int rc = sqlite3_exec(db, + ("DELETE FROM tags WHERE data_id = (SELECT id FROM " + "data WHERE path = '" + + path + "') AND tag_value = '" + tag + "';") + .c_str(), + nullptr, nullptr, nullptr); return rc; } -int db_relink_file(const std::string path, const std::string new_path) { - int rc = sqlite3_exec(db, ("UPDATE data SET path = '" + new_path + "' WHERE data.path = '" + path + "';").c_str(), nullptr, nullptr, nullptr); - +int db_relink_file(const std::string path, const std::string new_path) +{ + int rc = sqlite3_exec( + db, ("UPDATE data SET path = '" + new_path + "' WHERE data.path = '" + path + "';").c_str(), + nullptr, nullptr, nullptr); + return rc; } -// Recursively generate a SQL query string for WHERE clause of the standard query -std::optional db_query_helper(const std::shared_ptr ast) { +// Recursively generate a SQL query string for WHERE clause of the standard +// query +std::optional db_query_helper(const std::shared_ptr ast) +{ std::string query_part; // Determine AST type - if (auto tag = std::dynamic_pointer_cast(ast)) { + if (auto tag = std::dynamic_pointer_cast(ast)) + { query_part += "SELECT data_id FROM tags WHERE tag_value = "; query_part += "'" + tag->name + "'"; - - } else if (auto union_op = std::dynamic_pointer_cast(ast)) { + } + else if (auto union_op = std::dynamic_pointer_cast(ast)) + { query_part += "IN ("; auto tmp_part = db_query_helper(union_op->left_node); - if (!tmp_part.has_value()) { + if (!tmp_part.has_value()) + { return tmp_part; } @@ -137,85 +179,95 @@ std::optional db_query_helper(const std::shared_ptr ast) { query_part += " UNION "; tmp_part = db_query_helper(union_op->right_node); - if (!tmp_part.has_value()) { + if (!tmp_part.has_value()) + { return tmp_part; } query_part += tmp_part.value(); - - } else if (auto intersection_op = std::dynamic_pointer_cast(ast)) { + } + else if (auto intersection_op = std::dynamic_pointer_cast(ast)) + { // TODO: Unfortunate special case for NOT IN if (!std::dynamic_pointer_cast(intersection_op->left_node)) { query_part += "IN ("; } - else + else { query_part += ""; } auto tmp_part = db_query_helper(intersection_op->left_node); - if (!tmp_part.has_value()) { + if (!tmp_part.has_value()) + { return tmp_part; } query_part += tmp_part.value(); - + // TODO: Unfortunate special case for NOT IN if (!std::dynamic_pointer_cast(intersection_op->right_node)) { query_part += ") AND id IN ("; } - else + else { query_part += ") AND id "; } - - + tmp_part = db_query_helper(intersection_op->right_node); - if (!tmp_part.has_value()) { + if (!tmp_part.has_value()) + { return tmp_part; } query_part += tmp_part.value(); - - } else if (auto negation_op = std::dynamic_pointer_cast(ast)) { + } + else if (auto negation_op = std::dynamic_pointer_cast(ast)) + { query_part += "NOT IN ("; auto tmp_part = db_query_helper(negation_op->node); - if (!tmp_part.has_value()) { + if (!tmp_part.has_value()) + { return tmp_part; } query_part += tmp_part.value(); - - } else { + } + else + { return {}; } - + return query_part; } // Creates a syntactically valid SQLite3 query from an ASTNode -std::optional db_create_query(const std::shared_ptr ast) { +std::optional db_create_query(const std::shared_ptr ast) +{ std::string query; // Tag selection preamble query += "SELECT path "; - query += "FROM data WHERE id "; - //IN (SELECT data_id FROM tags WHERE tag_value "; + query += "FROM data WHERE id "; + // IN (SELECT data_id FROM tags WHERE tag_value "; - if (auto tag = std::dynamic_pointer_cast(ast)){ + if (auto tag = std::dynamic_pointer_cast(ast)) + { // TODO: This is a special case and i hate it query += "IN (SELECT data_id FROM tags WHERE tag_value "; query += "= '" + tag->name + "'"; - - } else { + } + else + { const auto query_part = db_query_helper(ast); - if (!query_part.has_value()) { + if (!query_part.has_value()) + { return {}; } @@ -227,16 +279,16 @@ std::optional db_create_query(const std::shared_ptr ast) { return query; } -void db_set_default_query(const std::string query) { - default_query = query; -} +void db_set_default_query(const std::string query) { default_query = query; } // TODO: I dont want the DB owning the default query, this was just easy for now // (im sleepy, sorry future me) -std::vector db_run_default_query() { +std::vector db_run_default_query() +{ const auto query_ast = parse(default_query); - if (query_ast.has_value()) { + if (query_ast.has_value()) + { return db_run_query(query_ast.value()); } @@ -244,21 +296,24 @@ std::vector db_run_default_query() { } // TODO: return a file struct containing the unique file ID -std::vector db_run_query(const std::shared_ptr ast) { +std::vector db_run_query(const std::shared_ptr ast) +{ std::vector results; auto query = db_create_query(ast); - if (!query.has_value()) { + if (!query.has_value()) + { return results; } spdlog::debug("Running Query: \n{0}\n", query.value()); - sqlite3_stmt *stmt; + sqlite3_stmt* stmt; sqlite3_prepare_v2(db, query.value().c_str(), -1, &stmt, nullptr); - while (sqlite3_step(stmt) == SQLITE_ROW) { + while (sqlite3_step(stmt) == SQLITE_ROW) + { results.push_back(std::string(reinterpret_cast(sqlite3_column_text(stmt, 0)))); } diff --git a/filesystem/src/db.hpp b/filesystem/src/db.hpp index 2af8538..4d91096 100644 --- a/filesystem/src/db.hpp +++ b/filesystem/src/db.hpp @@ -3,9 +3,9 @@ // SPDX-License-Identifier: BSD-3-Clause #pragma once +#include #include #include -#include #include "query_lang/ast.hpp" diff --git a/filesystem/src/fs.cpp b/filesystem/src/fs.cpp index 8a28623..1cc8b25 100644 --- a/filesystem/src/fs.cpp +++ b/filesystem/src/fs.cpp @@ -4,39 +4,40 @@ // Provides the filesystem interface through FUSE -extern "C" { -#include -#include +extern "C" +{ #include +#include +#include +#include #include +#include #include -#include -#include } -#include #include +#include +#include "command_interface.h" #include "db.hpp" #include "fs.hpp" -#include "command_interface.h" #include "utilities.hpp" // Gets file attributes at #ifdef __FreeBSD__ -int lake_getattr(const char *path, struct stat *stbuf, struct fuse_file_info* fi) +int lake_getattr(const char* path, struct stat* stbuf, struct fuse_file_info* fi) #else -int lake_getattr(const char *path, struct stat *stbuf) +int lake_getattr(const char* path, struct stat* stbuf) #endif { spdlog::trace("Getting attributes for {0}", path); if ((strcmp(path, "/") == 0) || (path[strlen(path) - 1] == ')')) { - stbuf->st_mode = S_IFDIR | 0755; - stbuf->st_nlink = 2; + stbuf->st_mode = S_IFDIR | 0755; + stbuf->st_nlink = 2; return 0; - } + } auto file = reverse_query(path); @@ -45,7 +46,8 @@ int lake_getattr(const char *path, struct stat *stbuf) return file.error(); } - if (file.value().empty()) { + if (file.value().empty()) + { spdlog::error("No file found!"); return -ENOENT; @@ -63,15 +65,12 @@ int lake_getattr(const char *path, struct stat *stbuf) return 0; } - #ifdef __FreeBSD__ -int lake_readdir( - const char *path, void *buf, fuse_fill_dir_t filler, - off_t offset, struct fuse_file_info *fi, enum fuse_readdir_flags flags) +int lake_readdir(const char* path, void* buf, fuse_fill_dir_t filler, off_t offset, + struct fuse_file_info* fi, enum fuse_readdir_flags flags) #else -int lake_readdir( - const char *path, void *buf, fuse_fill_dir_t filler, - off_t offset, struct fuse_file_info *fi) +int lake_readdir(const char* path, void* buf, fuse_fill_dir_t filler, off_t offset, + struct fuse_file_info* fi) #endif { @@ -93,9 +92,10 @@ int lake_readdir( return files.error(); } - for (const auto& file : files.value()) { + for (const auto& file : files.value()) + { const std::string file_name = file.substr(file.find_last_of("/") + 1); - + spdlog::trace("Will show file {0} as {1}", file, file_name); #ifdef __FreeBSD__ @@ -108,7 +108,8 @@ int lake_readdir( return 0; } -int lake_open(const char *path, struct fuse_file_info *fi) { +int lake_open(const char* path, struct fuse_file_info* fi) +{ spdlog::trace("Opening file {0}", path); @@ -123,7 +124,7 @@ int lake_open(const char *path, struct fuse_file_info *fi) { return file_path.error(); } - if (file_path.value().empty()) + if (file_path.value().empty()) { return -ENOENT; } @@ -132,23 +133,26 @@ int lake_open(const char *path, struct fuse_file_info *fi) { fi->fh = open(file_path.value().c_str(), fi->flags); - if (fi->fh == -1) { + if (fi->fh == -1) + { spdlog::error("Could not open file err: {0}", strerror(errno)); - + return -errno; } - if (fi->flags & O_DIRECT) { + if (fi->flags & O_DIRECT) + { fi->direct_io = 1; } spdlog::trace("Created fd {0} while opening file", fi->fh); - + return 0; } -int lake_release(const char *path, struct fuse_file_info *fi) { - +int lake_release(const char* path, struct fuse_file_info* fi) +{ + spdlog::trace("Releasing file {0} at FD {1}", path, fi->fh); if (close(fi->fh) != 0) @@ -159,8 +163,8 @@ int lake_release(const char *path, struct fuse_file_info *fi) { return 0; } -int lake_read(const char *path, char *buf, size_t size, off_t offset, - struct fuse_file_info *fi) { +int lake_read(const char* path, char* buf, size_t size, off_t offset, struct fuse_file_info* fi) +{ spdlog::trace("Reading file {0} at fd {1}", path, fi->fh); @@ -174,14 +178,16 @@ int lake_read(const char *path, char *buf, size_t size, off_t offset, return bytes_read; } -int lake_write(const char *path, const char *buf, size_t size, off_t offset, - struct fuse_file_info *fi) { - +int lake_write(const char* path, const char* buf, size_t size, off_t offset, + struct fuse_file_info* fi) +{ + spdlog::trace("Writing to file {0} at fd {1}", path, fi->fh); ssize_t bytes_written = pwrite(fi->fh, buf, size, offset); - if (bytes_written == -1) { + if (bytes_written == -1) + { spdlog::error("Could not write to file, err: {0}", strerror(errno)); return -errno; @@ -190,20 +196,23 @@ int lake_write(const char *path, const char *buf, size_t size, off_t offset, return bytes_written; } -void lake_destroy(void* private_data) { +void lake_destroy(void* private_data) +{ // Clean up the filesystem properly spdlog::trace("Shutting down filesystem"); int rc = db_close(); - if (rc != 0) { + if (rc != 0) + { spdlog::error("Failed to close SQLite3 DB"); } rc = unlink(LAKE_SOCKET_PATH); - if (rc != 0) { + if (rc != 0) + { spdlog::error("Failed to unlink socket"); } } \ No newline at end of file diff --git a/filesystem/src/fs.hpp b/filesystem/src/fs.hpp index 8c74005..ec9ac3a 100644 --- a/filesystem/src/fs.hpp +++ b/filesystem/src/fs.hpp @@ -3,7 +3,8 @@ // SPDX-License-Identifier: BSD-3-Clause #pragma once -extern "C" { +extern "C" +{ #define FUSE_USE_VERSION 31 #ifdef __FreeBSD__ @@ -14,29 +15,26 @@ extern "C" { } #ifdef __FreeBSD__ -int lake_getattr(const char *path, struct stat *stbuf, struct fuse_file_info* fi); +int lake_getattr(const char* path, struct stat* stbuf, struct fuse_file_info* fi); #else -int lake_getattr(const char *path, struct stat *stbuf); +int lake_getattr(const char* path, struct stat* stbuf); #endif #ifdef __FreeBSD__ -int lake_readdir( - const char *path, void *buf, fuse_fill_dir_t filler, - off_t offset, struct fuse_file_info *fi, enum fuse_readdir_flags flags); +int lake_readdir(const char* path, void* buf, fuse_fill_dir_t filler, off_t offset, + struct fuse_file_info* fi, enum fuse_readdir_flags flags); #else -int lake_readdir( - const char *path, void *buf, fuse_fill_dir_t filler, - off_t offset, struct fuse_file_info *fi); +int lake_readdir(const char* path, void* buf, fuse_fill_dir_t filler, off_t offset, + struct fuse_file_info* fi); #endif -int lake_open(const char *path, struct fuse_file_info *fi); +int lake_open(const char* path, struct fuse_file_info* fi); -int lake_release(const char *path, struct fuse_file_info *fi); +int lake_release(const char* path, struct fuse_file_info* fi); -int lake_read(const char *path, char *buf, size_t size, off_t offset, - struct fuse_file_info *fi); +int lake_read(const char* path, char* buf, size_t size, off_t offset, struct fuse_file_info* fi); -int lake_write(const char *path, const char *buf, size_t size, off_t offset, - struct fuse_file_info *fi); +int lake_write(const char* path, const char* buf, size_t size, off_t offset, + struct fuse_file_info* fi); void lake_destroy(void* private_data); \ No newline at end of file diff --git a/filesystem/src/main.cpp b/filesystem/src/main.cpp index a1c61c8..a83b9e0 100644 --- a/filesystem/src/main.cpp +++ b/filesystem/src/main.cpp @@ -2,117 +2,111 @@ // // SPDX-License-Identifier: BSD-3-Clause -#include #include +#include +#include #include #include #include -#include #include -#include +#include #include #include -#include +#include // #include -extern "C" { - #include +extern "C" +{ +#include } #include +#include "backups.hpp" +#include "config.hpp" +#include "control.hpp" #include "db.hpp" #include "fs.hpp" -#include "control.hpp" -#include "config.hpp" -#include "backups.hpp" #include "metadata.h" static const struct fuse_operations operations = { - .getattr = lake_getattr, + .getattr = lake_getattr, .readlink = nullptr, - .mknod = nullptr, - .mkdir = nullptr, - .unlink = nullptr, - .rmdir = nullptr, - .symlink = nullptr, - .rename = nullptr, - .link = nullptr, - .chmod = nullptr, - .chown = nullptr, + .mknod = nullptr, + .mkdir = nullptr, + .unlink = nullptr, + .rmdir = nullptr, + .symlink = nullptr, + .rename = nullptr, + .link = nullptr, + .chmod = nullptr, + .chown = nullptr, .truncate = nullptr, - .open = lake_open, - .read = lake_read, - .write = lake_write, - .statfs = nullptr, - .release = lake_release, - .fsync = nullptr, - .readdir = lake_readdir, - .destroy = lake_destroy, - .access = nullptr, - .create = nullptr, - .ioctl = nullptr, + .open = lake_open, + .read = lake_read, + .write = lake_write, + .statfs = nullptr, + .release = lake_release, + .fsync = nullptr, + .readdir = lake_readdir, + .destroy = lake_destroy, + .access = nullptr, + .create = nullptr, + .ioctl = nullptr, }; -auto main(int argc, char** argv) -> int { +auto main(int argc, char** argv) -> int +{ // Set up the CLI args argparse::ArgumentParser program("lakefs", LAKEFS_VERSION); program.add_description("LakeFS - A tag based abstraction over the filesystem"); - program.add_argument("mount_point") - .required() - .help("The directory to mount the filesystem at"); + program.add_argument("mount_point").required().help("The directory to mount the filesystem at"); program.add_argument("default_query") .default_value("default") .help("Specify the initial default query of the mount"); - program.add_argument("--tempdb") - .flag() - .help("Use an in-memory database instead of the file"); + program.add_argument("--tempdb").flag().help("Use an in-memory database instead of the file"); program.add_argument("--config", "-c") .default_value("/etc/lakefs.conf") - .help("Manually set a config location rather than using /etc/lakefs.conf"); + .help("Manually set a config location rather than using " + "/etc/lakefs.conf"); - program.add_argument("-f") - .flag() - .help("Run program in foreground rather than as a daemon"); + program.add_argument("-f").flag().help("Run program in foreground rather than as a daemon"); - program.add_argument("-d") - .flag() - .help("Output complete debug information while running"); + program.add_argument("-d").flag().help("Output complete debug information while running"); - try { - program.parse_args(argc, argv); - - } catch (const std::runtime_error& err) { + try + { + program.parse_args(argc, argv); + } + catch (const std::runtime_error& err) + { spdlog::critical("Error collecting arguments: {0}", err.what()); - return 1; - } + return 1; + } const auto is_debug = program.get("-d"); // Get our configuration - const auto config_path = std::filesystem::absolute( - program.get("--config") - ); + const auto config_path = std::filesystem::absolute(program.get("--config")); auto config = etc_conf_reader(config_path); - if (config.empty() && !program.get("--tempdb")) { + if (config.empty() && !program.get("--tempdb")) + { spdlog::error("Failed to read configuration file at {0}", config_path.string()); return 1; } // Extract mount point - const auto mount_point = std::filesystem::absolute( - program.get("mount_point") - ); + const auto mount_point = std::filesystem::absolute(program.get("mount_point")); // Extract default query const auto default_query = program.get("default_query"); @@ -122,13 +116,14 @@ auto main(int argc, char** argv) -> int { if (!program.get("--tempdb")) { const std::string log_file_name = "/var/lakefs/lakefs.log"; - + // create a rotating logger with a 5MB file size and three files const auto max_size = 1048576 * 5; const auto max_files = 3; - auto rotating_sink = std::make_shared(log_file_name, max_size, max_files); + auto rotating_sink = std::make_shared( + log_file_name, max_size, max_files); auto console_sink = std::make_shared(); - + spdlog::logger logger("default", {console_sink, rotating_sink}); spdlog::default_logger()->swap(logger); } @@ -139,13 +134,15 @@ auto main(int argc, char** argv) -> int { // NOTE argv[0] is program name in C!! and is ignored by fuse fuse_opt_add_arg(&args, "lakefs"); - if (is_debug) { + if (is_debug) + { spdlog::info("Running in debug mode"); spdlog::set_level(spdlog::level::trace); fuse_opt_add_arg(&args, "-d"); - - } else { + } + else + { const auto log_level = config["log_level"]; spdlog::set_level(spdlog::level::from_str(log_level)); @@ -166,43 +163,55 @@ auto main(int argc, char** argv) -> int { // Initialize SQLLite int rc; - if (program.get("--tempdb")) { + if (program.get("--tempdb")) + { spdlog::info("Using in-memory database"); - + rc = db_tmp_init(); - } else { + } + else + { rc = db_init(config["dir"]); const auto interval = parse_interval_value(config["backup_interval"]); - if (!interval.has_value()) { + if (!interval.has_value()) + { spdlog::warn("backup_interval malformed. Continuing without backups"); - - } else { + } + else + { // Create backup timer - if (create_backup_timer(interval.value(), std::stoi(config["max_backups"]), config["dir"])) { + if (create_backup_timer(interval.value(), std::stoi(config["max_backups"]), + config["dir"])) + { spdlog::info("Backup system started"); - - } else { - spdlog::warn("Could not start backup system. Continuing without backups"); + } + else + { + spdlog::warn("Could not start backup system. Continuing " + "without backups"); } } } - if (rc != SQLITE_OK) { + if (rc != SQLITE_OK) + { spdlog::critical("Failed to initialize SQLite3: {0}", rc); return 1; } const auto is_daemon = !program.get("-f"); - if (is_daemon) { + if (is_daemon) + { spdlog::trace("Launching as daemon"); // Forks program and runs in background const auto rc = daemon(0, 1); - if (rc < 0) { + if (rc < 0) + { spdlog::critical("Error starting daemon: {0}", strerror(errno)); } } diff --git a/filesystem/src/utilities.cpp b/filesystem/src/utilities.cpp index ccf5862..ccba607 100644 --- a/filesystem/src/utilities.cpp +++ b/filesystem/src/utilities.cpp @@ -2,15 +2,16 @@ // // SPDX-License-Identifier: BSD-3-Clause #include +#include #include #include -#include -#include "query_lang/parser.hpp" #include "db.hpp" +#include "query_lang/parser.hpp" #include "utilities.hpp" -std::expected, int> get_files(const char* path) { +std::expected, int> get_files(const char* path) +{ spdlog::trace("Entering get_files(path={0})", path); std::vector files; @@ -25,10 +26,12 @@ std::expected, int> get_files(const char* path) { const auto query = extract_query(path); const auto query_ast = parse(query); - if (query_ast.has_value()) { + if (query_ast.has_value()) + { files = db_run_query(query_ast.value()); - - } else { + } + else + { spdlog::error("Error while parsing, invalid query"); return std::unexpected(-EINVAL); @@ -41,7 +44,6 @@ std::expected, int> get_files(const char* path) { if (path_segments.size() > 1) { - // Get the file path by comparing the file name to the query results std::string file_path; std::string looking_for_file; @@ -55,13 +57,12 @@ std::expected, int> get_files(const char* path) { looking_for_file = path_segments[0]; } + for (const auto& query_file : files) + { + const std::string query_file_name = query_file.substr(query_file.find_last_of("/") + 1); - for (const auto& query_file : files) { - - const std::string query_file_name = - query_file.substr(query_file.find_last_of("/") + 1); - - if (query_file_name == looking_for_file) { + if (query_file_name == looking_for_file) + { file_path = query_file; break; } @@ -106,18 +107,19 @@ std::expected, int> get_files(const char* path) { return files; } -std::expected reverse_query(const char* path) { +std::expected reverse_query(const char* path) +{ auto path_s = std::string(path); spdlog::trace("Entering reverse_query(path={0})", path_s); const auto path_segments = split(path, "/"); - std::string looking_for_file = path_segments[path_segments.size()-1]; + std::string looking_for_file = path_segments[path_segments.size() - 1]; std::string reconstructed_path; - for (int i = 0; i < path_segments.size()-1; i++) + for (int i = 0; i < path_segments.size() - 1; i++) { reconstructed_path += "/" + path_segments[i]; } @@ -129,29 +131,31 @@ std::expected reverse_query(const char* path) { return std::unexpected(files.error()); } - const auto file_path = std::find_if(files->begin(), files->end(), - [looking_for_file] (const std::string& file_name) -> bool { - return std::filesystem::path(file_name).filename() == looking_for_file; - }); + const auto file_path = std::find_if( + files->begin(), files->end(), [looking_for_file](const std::string& file_name) -> bool + { return std::filesystem::path(file_name).filename() == looking_for_file; }); spdlog::trace("Exiting reverse_query() -> {0}", *file_path); return *file_path; } -std::string extract_query(const char* path) { +std::string extract_query(const char* path) +{ spdlog::trace("Entering extract_query(path={0})", std::string(path)); auto path_s = std::string(path); - // TODO: This is pretty niave. quickly 'fixed' for now but a real algorithm should be used - const auto extracted_query = - path_s.substr(path_s.find_first_of('('), path_s.find_last_of(')') - path_s.find_first_of('(') + 1); + // TODO: This is pretty niave. quickly 'fixed' for now but a real algorithm + // should be used + const auto extracted_query = path_s.substr( + path_s.find_first_of('('), path_s.find_last_of(')') - path_s.find_first_of('(') + 1); spdlog::trace("Exiting extract_query() -> {0}", extracted_query); return extracted_query; } -std::vector split(const std::string str, const std::string delim) { +std::vector split(const std::string str, const std::string delim) +{ spdlog::trace("Entering split(str={0}, delim={1})", str, delim); std::vector parts; @@ -165,9 +169,8 @@ std::vector split(const std::string str, const std::string delim) { { if (part_start != i) { - parts.push_back(str.substr(part_start, i-part_start)); + parts.push_back(str.substr(part_start, i - part_start)); } - part_start = i + delim.size(); } } diff --git a/filesystem/src/utilities.hpp b/filesystem/src/utilities.hpp index 530f9a9..bad5dc3 100644 --- a/filesystem/src/utilities.hpp +++ b/filesystem/src/utilities.hpp @@ -3,9 +3,9 @@ // SPDX-License-Identifier: BSD-3-Clause #pragma once +#include #include #include -#include // Runs a query normally. Handles special logic to pull out folders std::expected, int> get_files(const char* path); diff --git a/filesystem/tests/config.cpp b/filesystem/tests/config.cpp index be3f0c4..18e1a38 100644 --- a/filesystem/tests/config.cpp +++ b/filesystem/tests/config.cpp @@ -8,22 +8,26 @@ #define CATCH_CONFIG_MAIN #include -TEST_CASE("Config Parsing", "[config]") { +TEST_CASE("Config Parsing", "[config]") +{ - SECTION("Simple Key Value") { + SECTION("Simple Key Value") + { auto kv_pair = parse_config_line("test=1"); CHECK(kv_pair.first == "test"); CHECK(kv_pair.second == "1"); } - SECTION("Key Value with spaces") { + SECTION("Key Value with spaces") + { auto kv_pair = parse_config_line("key name=value is something"); CHECK(kv_pair.first == "key name"); CHECK(kv_pair.second == "value is something"); } } -TEST_CASE("Interval Parsing", "[config]") { +TEST_CASE("Interval Parsing", "[config]") +{ // Conversions const auto seconds = 60; @@ -32,28 +36,34 @@ TEST_CASE("Interval Parsing", "[config]") { const auto days = 7; const auto weeks = 4; - - SECTION("Hours") { + SECTION("Hours") + { auto interval = parse_interval_value("3 hours"); CHECK(interval->count() == 3 * minutes * seconds); } - SECTION("Days") { + SECTION("Days") + { auto interval = parse_interval_value("5 days"); CHECK(interval->count() == 5 * hours * minutes * seconds); } - SECTION("Weeks") { + SECTION("Weeks") + { auto interval = parse_interval_value("2 weeks"); CHECK(interval->count() == 2 * days * hours * minutes * seconds); } - SECTION("Months") { + SECTION("Months") + { auto interval = parse_interval_value("3 months"); - CHECK(interval->count() == 3 * 2629746); // NOTE: std::chronos ratio for months is this magic value, which is I assume more accurate than the naive calculation + CHECK(interval->count() == 3 * 2629746); // NOTE: std::chronos ratio for months is this + // magic value, which is I assume more accurate + // than the naive calculation } - SECTION("Invalid input") { + SECTION("Invalid input") + { auto interval = parse_interval_value("3 dogs"); CHECK_FALSE(interval.has_value()); } diff --git a/filesystem/tests/parsing.cpp b/filesystem/tests/parsing.cpp index be1c64b..09338ac 100644 --- a/filesystem/tests/parsing.cpp +++ b/filesystem/tests/parsing.cpp @@ -8,103 +8,86 @@ #include "query_lang/parser.hpp" -TEST_CASE("Simple Token operators", "[parsing]") { +TEST_CASE("Simple Token operators", "[parsing]") +{ REQUIRE(Token("abc123") == Token("abc123")); REQUIRE_FALSE(Token("abc") == Token("123")); } -TEST_CASE("Tokenizing", "[parsing]") { +TEST_CASE("Tokenizing", "[parsing]") +{ std::vector tokens; tokens = tokenize("abc123"); REQUIRE(tokens == std::vector{ - Token("abc123"), - }); + Token("abc123"), + }); tokens = tokenize("abc&123+(d)"); REQUIRE(tokens == std::vector{ - Token("abc"), - Token("&"), - Token("123"), - Token("+"), - Token("("), - Token("d"), - Token(")"), - }); + Token("abc"), + Token("&"), + Token("123"), + Token("+"), + Token("("), + Token("d"), + Token(")"), + }); tokens = tokenize("two_words"); REQUIRE(tokens == std::vector{ - Token("two_words"), - }); + Token("two_words"), + }); } - -TEST_CASE("Parsing text into a AST query", "[parsing]") { +TEST_CASE("Parsing text into a AST query", "[parsing]") +{ std::shared_ptr ast; std::shared_ptr expected_ast; - SECTION("Single Letter Tags") { + SECTION("Single Letter Tags") + { ast = parse("a&b").value(); - expected_ast = std::make_shared( - std::make_shared("a"), - std::make_shared("b") - ); + expected_ast = + std::make_shared(std::make_shared("a"), std::make_shared("b")); REQUIRE(ast->match(expected_ast)); ast = parse("a&(b|c)").value(); expected_ast = std::make_shared( std::make_shared("a"), - std::make_shared( - std::make_shared("b"), - std::make_shared("c") - ) - ); + std::make_shared(std::make_shared("b"), std::make_shared("c"))); REQUIRE(ast->match(expected_ast)); ast = parse("(b|c)&a").value(); expected_ast = std::make_shared( - std::make_shared( - std::make_shared("b"), - std::make_shared("c") - ), - std::make_shared("a") - ); + std::make_shared(std::make_shared("b"), std::make_shared("c")), + std::make_shared("a")); REQUIRE(ast->match(expected_ast)); } - SECTION("Word Tags") { + SECTION("Word Tags") + { ast = parse("tag1&tag2").value(); - expected_ast = std::make_shared( - std::make_shared("tag1"), - std::make_shared("tag2") - ); + expected_ast = std::make_shared(std::make_shared("tag1"), + std::make_shared("tag2")); REQUIRE(ast->match(expected_ast)); ast = parse("tag1&(tag2|tag3)").value(); expected_ast = std::make_shared( std::make_shared("tag1"), - std::make_shared( - std::make_shared("tag2"), - std::make_shared("tag3") - ) - ); + std::make_shared(std::make_shared("tag2"), std::make_shared("tag3"))); REQUIRE(ast->match(expected_ast)); } - SECTION("Complex Queries") { + SECTION("Complex Queries") + { ast = parse("(picture|video)&(year_2014&(!digital))").value(); expected_ast = std::make_shared( - std::make_shared( - std::make_shared("picture"), - std::make_shared("video") - ), + std::make_shared(std::make_shared("picture"), + std::make_shared("video")), std::make_shared( std::make_shared("year_2014"), - std::make_shared( - std::make_shared("digital") - ) - ) - ); + std::make_shared(std::make_shared("digital")))); std::cout << ast->str() << std::endl; std::cout << expected_ast->str() << std::endl; @@ -112,7 +95,8 @@ TEST_CASE("Parsing text into a AST query", "[parsing]") { REQUIRE(ast->match(expected_ast)); } - SECTION("Issue #34: Potential Runtime error") { + SECTION("Issue #34: Potential Runtime error") + { const auto ast = parse("((tag1 | tag2) &)"); REQUIRE_FALSE(ast.has_value()); diff --git a/filesystem/tests/query_generation.cpp b/filesystem/tests/query_generation.cpp index 5a6c721..c5d21e1 100644 --- a/filesystem/tests/query_generation.cpp +++ b/filesystem/tests/query_generation.cpp @@ -5,60 +5,62 @@ #define CATCH_CONFIG_MAIN #include +#include "db.hpp" #include "query_lang/ast.hpp" #include "query_lang/parser.hpp" -#include "db.hpp" // TODO: testing direct string queries like this will be fragile. A SQL AST // Will be more robust and easier to translate to a different DB in the future -TEST_CASE("Basic Tag Retrieval", "[parsing][query]") { +TEST_CASE("Basic Tag Retrieval", "[parsing][query]") +{ std::shared_ptr ast; std::optional query; std::string expected_query; - SECTION("Single Tag Retrieval") { + SECTION("Single Tag Retrieval") + { ast = parse("tag").value(); query = db_create_query(ast); - expected_query = - "SELECT path FROM data WHERE id IN " - "(SELECT data_id FROM tags WHERE tag_value = 'tag');"; - + expected_query = "SELECT path FROM data WHERE id IN " + "(SELECT data_id FROM tags WHERE tag_value = 'tag');"; + REQUIRE(query.has_value()); REQUIRE(query.value() == expected_query); } - SECTION("Negation") { + SECTION("Negation") + { ast = parse("!tag").value(); query = db_create_query(ast); - expected_query = - "SELECT path FROM data WHERE id NOT IN " - "(SELECT data_id FROM tags WHERE tag_value = 'tag');"; - + expected_query = "SELECT path FROM data WHERE id NOT IN " + "(SELECT data_id FROM tags WHERE tag_value = 'tag');"; + REQUIRE(query.has_value()); REQUIRE(query.value() == expected_query); } - SECTION("Union Retrieval") { + SECTION("Union Retrieval") + { ast = parse("tag1|tag2").value(); query = db_create_query(ast); - expected_query = - "SELECT path FROM data WHERE id IN " - "(SELECT data_id FROM tags WHERE tag_value = 'tag1' UNION SELECT data_id FROM tags WHERE tag_value = 'tag2');"; - + expected_query = "SELECT path FROM data WHERE id IN " + "(SELECT data_id FROM tags WHERE tag_value = 'tag1' UNION SELECT " + "data_id FROM tags WHERE tag_value = 'tag2');"; + REQUIRE(query.has_value()); REQUIRE(query.value() == expected_query); } - SECTION("Intersection Retrieval") { + SECTION("Intersection Retrieval") + { ast = parse("tag1&tag2").value(); query = db_create_query(ast); - expected_query = - "SELECT path FROM data WHERE id IN " - "(SELECT data_id FROM tags WHERE tag_value = 'tag1') " - "AND id IN " - "(SELECT data_id FROM tags WHERE tag_value = 'tag2');"; - + expected_query = "SELECT path FROM data WHERE id IN " + "(SELECT data_id FROM tags WHERE tag_value = 'tag1') " + "AND id IN " + "(SELECT data_id FROM tags WHERE tag_value = 'tag2');"; + REQUIRE(query.has_value()); REQUIRE(query.value() == expected_query); } diff --git a/filesystem/tests/utilities.cpp b/filesystem/tests/utilities.cpp index 5887636..f29422b 100644 --- a/filesystem/tests/utilities.cpp +++ b/filesystem/tests/utilities.cpp @@ -7,46 +7,46 @@ #include "utilities.hpp" -TEST_CASE("Query Extraction", "[utilities]") { - SECTION("Simple Query") { - auto res = - extract_query("/test/wow/l/(tag1 | tag2)"); +TEST_CASE("Query Extraction", "[utilities]") +{ + SECTION("Simple Query") + { + auto res = extract_query("/test/wow/l/(tag1 | tag2)"); CHECK(res == "(tag1 | tag2)"); - res = - extract_query("/test/wow/l/(tag1 | tag2)/file"); + res = extract_query("/test/wow/l/(tag1 | tag2)/file"); CHECK(res == "(tag1 | tag2)"); } - SECTION("Subquerying") { - auto res = - extract_query("/test/wow/l/(tag1 & (!place))"); + SECTION("Subquerying") + { + auto res = extract_query("/test/wow/l/(tag1 & (!place))"); CHECK(res == "(tag1 & (!place))"); - res = - extract_query("/test/wow/l/(tag1 & (!place))/file"); + res = extract_query("/test/wow/l/(tag1 & (!place))/file"); CHECK(res == "(tag1 & (!place))"); } - SECTION("Multilevel Subquerying") { - auto res = - extract_query("/test/wow/l/(tag1 & ((!place) | (tag2 & tag3)))"); + SECTION("Multilevel Subquerying") + { + auto res = extract_query("/test/wow/l/(tag1 & ((!place) | (tag2 & tag3)))"); CHECK(res == "(tag1 & ((!place) | (tag2 & tag3)))"); - res = - extract_query("/test/wow/l/(tag1 & ((!place) | (tag2 & tag3)))/file"); + res = extract_query("/test/wow/l/(tag1 & ((!place) | (tag2 & tag3)))/file"); CHECK(res == "(tag1 & ((!place) | (tag2 & tag3)))"); } } -TEST_CASE("String Splitting", "[utilities]") { - SECTION("Single char delim") { +TEST_CASE("String Splitting", "[utilities]") +{ + SECTION("Single char delim") + { const auto haystack = "wow/this/is/a/path/"; const auto needle = "/"; @@ -57,14 +57,15 @@ TEST_CASE("String Splitting", "[utilities]") { CHECK(found[4] == "path"); } - SECTION("Multi char delim") { + SECTION("Multi char delim") + { const auto haystack = "> -TEST_CASE("SQLite3 Initialization", "[vendor]") { - sqlite3 *db; +TEST_CASE("SQLite3 Initialization", "[vendor]") +{ + sqlite3* db; int rc = sqlite3_open(":memory:", &db); REQUIRE(rc == SQLITE_OK); @@ -16,32 +17,38 @@ TEST_CASE("SQLite3 Initialization", "[vendor]") { sqlite3_close(db); } -TEST_CASE("SQLite3 Querying", "[vendor]") { - sqlite3 *db; +TEST_CASE("SQLite3 Querying", "[vendor]") +{ + sqlite3* db; int rc = sqlite3_open(":memory:", &db); REQUIRE(rc == SQLITE_OK); - rc = sqlite3_exec(db, "CREATE TABLE test (id INTEGER PRIMARY KEY, name TEXT);", nullptr, nullptr, nullptr); + rc = sqlite3_exec(db, + "CREATE TABLE test (id INTEGER PRIMARY KEY, name TEXT);", + nullptr, nullptr, nullptr); REQUIRE(rc == SQLITE_OK); - rc = sqlite3_exec(db, "INSERT INTO test (name) VALUES ('Test!');", nullptr, nullptr, nullptr); + rc = sqlite3_exec(db, "INSERT INTO test (name) VALUES ('Test!');", nullptr, + nullptr, nullptr); REQUIRE(rc == SQLITE_OK); // Prepare the SQL statement - sqlite3_stmt *stmt; - const char *sql = "SELECT name FROM test;"; + sqlite3_stmt* stmt; + const char* sql = "SELECT name FROM test;"; rc = sqlite3_prepare_v2(db, sql, -1, &stmt, nullptr); REQUIRE(rc == SQLITE_OK); // Execute the prepared statement - while (sqlite3_step(stmt) == SQLITE_ROW) { + while (sqlite3_step(stmt) == SQLITE_ROW) + { // Retrieve the value of the "name" column - const char *name = reinterpret_cast(sqlite3_column_text(stmt, 0)); - + const char* name = + reinterpret_cast(sqlite3_column_text(stmt, 0)); + REQUIRE(std::string(name) == "Test!"); } diff --git a/makefile b/makefile index 1d3ded8..17aba12 100644 --- a/makefile +++ b/makefile @@ -19,6 +19,7 @@ help: @echo " install - Runs mesons install command. Will overwrite /etc/lakefs.conf" @echo " test - Runs all project tests" @echo " local-test - Runs project tests, but uses unshare to create a seperate cgroup for lakefs. Helps mitigate issues while testing due to fuse locking the filesystem during a failure in lakefs" + @echo " format - Formats the project files" @echo " clean - Cleans mesons build directory" .PHONY: setup @@ -43,6 +44,11 @@ test: build local-test: unshare -pfr --user --mount --kill-child meson test -C $(BUILD_DIR) --print-errorlogs +.PHONY: format +format: + -clang-format filesystem/tests/** -i + -clang-format filesystem/src/** -i + .PHONY: clean clean: meson compile --clean -C $(BUILD_DIR) \ No newline at end of file From f8602e707eb07a83c69d974a413bfcc1c446ce97 Mon Sep 17 00:00:00 2001 From: Caleb Depatie Date: Tue, 11 Nov 2025 16:30:16 -0500 Subject: [PATCH 21/23] Fixed make format command --- filesystem/src/query_lang/ast.cpp | 227 ++++++++++++++------------- filesystem/src/query_lang/ast.hpp | 79 +++++----- filesystem/src/query_lang/parser.cpp | 128 ++++++++------- filesystem/src/query_lang/parser.hpp | 11 +- filesystem/tests/vendors/sqlite.cpp | 11 +- makefile | 6 +- 6 files changed, 248 insertions(+), 214 deletions(-) diff --git a/filesystem/src/query_lang/ast.cpp b/filesystem/src/query_lang/ast.cpp index 094fb04..3dbdd5c 100644 --- a/filesystem/src/query_lang/ast.cpp +++ b/filesystem/src/query_lang/ast.cpp @@ -13,227 +13,242 @@ AstNode::AstNode() {} -std::ostream& operator<<(std::ostream &out, const std::shared_ptr node) { +std::ostream& operator<<(std::ostream& out, const std::shared_ptr node) +{ out << node->str(); - + return out; } //== Operator == -Operator::Operator(int precedence) - : precedence(precedence) {} +Operator::Operator(int precedence) : precedence(precedence) {} -std::string Operator::str() const { - return "Operator"; -} +std::string Operator::str() const { return "Operator"; } -bool Operator::operator>=(const std::shared_ptr other) const { +bool Operator::operator>=(const std::shared_ptr other) const +{ return this->precedence >= other->precedence; } - //== BinaryOperator == -BinaryOperator::BinaryOperator(int precedence) - : Operator(precedence), left_node(nullptr), right_node(nullptr) {} +BinaryOperator::BinaryOperator(int precedence) + : Operator(precedence), left_node(nullptr), right_node(nullptr) +{ +} -BinaryOperator::BinaryOperator(int precedence, std::shared_ptr left, std::shared_ptr right) - : Operator(precedence), left_node(left), right_node(right) {} +BinaryOperator::BinaryOperator(int precedence, std::shared_ptr left, + std::shared_ptr right) + : Operator(precedence), left_node(left), right_node(right) +{ +} -std::string BinaryOperator::str() const { +std::string BinaryOperator::str() const +{ std::string left_str = "Null"; std::string right_str = "Null"; - - if (this->left_node != nullptr) { + + if (this->left_node != nullptr) + { left_str = this->left_node->str(); } - - if (this->right_node != nullptr) { + + if (this->right_node != nullptr) + { right_str = this->right_node->str(); } - + return "BinaryOperator{" + left_str + "," + right_str + "}"; } // TODO: I cant remember why these are raw pointers but they should be smart -bool BinaryOperator::assembleAST(std::vector>* rpn, std::vector>::iterator* rpn_iter) { +bool BinaryOperator::assembleAST(std::vector>* rpn, + std::vector>::iterator* rpn_iter) +{ assert(rpn != nullptr); assert(rpn_iter != nullptr); - + // Check that memory access will be SAFE - if (std::distance(rpn->begin(), *rpn_iter) < 2) { + if (std::distance(rpn->begin(), *rpn_iter) < 2) + { spdlog::error("Incorrect AST provided!"); return false; // PANIC } - - //Collect the arguments - this->left_node = *((*rpn_iter)-2); - this->right_node = *((*rpn_iter)-1); - rpn->erase((*rpn_iter)-2, (*rpn_iter)); - (*rpn_iter) -= 2; //2 elements removed + // Collect the arguments + this->left_node = *((*rpn_iter) - 2); + this->right_node = *((*rpn_iter) - 1); + + rpn->erase((*rpn_iter) - 2, (*rpn_iter)); + (*rpn_iter) -= 2; // 2 elements removed return true; } - //== UnaryOperator == -UnaryOperator::UnaryOperator(int precedence) - : Operator(precedence), node(nullptr) {} +UnaryOperator::UnaryOperator(int precedence) : Operator(precedence), node(nullptr) {} -UnaryOperator::UnaryOperator(int precedence, std::shared_ptr node) - : Operator(precedence), node(node) {} +UnaryOperator::UnaryOperator(int precedence, std::shared_ptr node) + : Operator(precedence), node(node) +{ +} -std::string UnaryOperator::str() const { +std::string UnaryOperator::str() const +{ std::string node_str = "Null"; - - if (this->node != nullptr) { + + if (this->node != nullptr) + { node_str = this->node->str(); } - + return "UnaryOperator{" + node_str + "}"; } -bool UnaryOperator::assembleAST(std::vector>* rpn, std::vector>::iterator *rpn_iter) { +bool UnaryOperator::assembleAST(std::vector>* rpn, + std::vector>::iterator* rpn_iter) +{ assert(rpn != nullptr); assert(rpn_iter != nullptr); // Check that memory access will be SAFE - if (std::distance(rpn->begin(), *rpn_iter) < 1) { + if (std::distance(rpn->begin(), *rpn_iter) < 1) + { spdlog::error("Incorrect AST provided!"); return false; // PANIC } - - //Collect the arguments - this->node = *((*rpn_iter)-1); - rpn->erase((*rpn_iter)-1, (*rpn_iter)); - (*rpn_iter) -= 1; //1 element removed + // Collect the arguments + this->node = *((*rpn_iter) - 1); + + rpn->erase((*rpn_iter) - 1, (*rpn_iter)); + (*rpn_iter) -= 1; // 1 element removed return true; } - //== Union == -Union::Union() - : BinaryOperator(0) {} - -Union::Union(std::shared_ptr left, std::shared_ptr right) - : BinaryOperator(0, left, right) {} +Union::Union() : BinaryOperator(0) {} -std::string Union::str() const { - return "Union_" + BinaryOperator::str(); +Union::Union(std::shared_ptr left, std::shared_ptr right) + : BinaryOperator(0, left, right) +{ } -bool Union::match(const std::shared_ptr other) const { +std::string Union::str() const { return "Union_" + BinaryOperator::str(); } + +bool Union::match(const std::shared_ptr other) const +{ const auto other_union = std::dynamic_pointer_cast(other); - - if (other_union != nullptr) { - return this->left_node->match(other_union->left_node) - && this->right_node->match(other_union->right_node); + + if (other_union != nullptr) + { + return this->left_node->match(other_union->left_node) && + this->right_node->match(other_union->right_node); } - + return false; } - //== Intersection == -Intersection::Intersection() - : BinaryOperator(1) {} - -Intersection::Intersection(std::shared_ptr left, std::shared_ptr right) - : BinaryOperator(0, left, right) {} +Intersection::Intersection() : BinaryOperator(1) {} -std::string Intersection::str() const { - return "Intersection_" + BinaryOperator::str(); +Intersection::Intersection(std::shared_ptr left, std::shared_ptr right) + : BinaryOperator(0, left, right) +{ } -bool Intersection::match(const std::shared_ptr other) const { +std::string Intersection::str() const { return "Intersection_" + BinaryOperator::str(); } + +bool Intersection::match(const std::shared_ptr other) const +{ const auto other_intersection = std::dynamic_pointer_cast(other); - - if (other_intersection != nullptr) { - return this->left_node->match(other_intersection->left_node) - && this->right_node->match(other_intersection->right_node); + + if (other_intersection != nullptr) + { + return this->left_node->match(other_intersection->left_node) && + this->right_node->match(other_intersection->right_node); } - + return false; } - //== Negation == -Negation::Negation() - : UnaryOperator(2) {} +Negation::Negation() : UnaryOperator(2) {} -Negation::Negation(std::shared_ptr node) - : UnaryOperator(0, node) {} +Negation::Negation(std::shared_ptr node) : UnaryOperator(0, node) {} -std::string Negation::str() const { - return "Negation_" + UnaryOperator::str(); -} +std::string Negation::str() const { return "Negation_" + UnaryOperator::str(); } -bool Negation::match(const std::shared_ptr other) const { +bool Negation::match(const std::shared_ptr other) const +{ const auto other_negation = std::dynamic_pointer_cast(other); - - if (other_negation != nullptr) { + + if (other_negation != nullptr) + { return this->node->match(other_negation->node); } - + return false; } - //== Tag == -Tag::Tag(std::string name) - : name(name) {} +Tag::Tag(std::string name) : name(name) {} -std::string Tag::str() const { - return "Tag{" + this->name + "}"; -} +std::string Tag::str() const { return "Tag{" + this->name + "}"; } -bool Tag::assembleAST(std::vector>* rpn, std::vector>::iterator *rpn_iter) { - //Nothing needs to be done +bool Tag::assembleAST(std::vector>* rpn, + std::vector>::iterator* rpn_iter) +{ + // Nothing needs to be done return true; } -bool Tag::match(const std::shared_ptr other) const { +bool Tag::match(const std::shared_ptr other) const +{ const auto other_tag = std::dynamic_pointer_cast(other); - - if (other_tag != nullptr) { + + if (other_tag != nullptr) + { return this->name == other_tag->name; } - + return false; } // == Loose functions == -std::ostream& operator<<(std::ostream &out, const std::vector> &nodes) { +std::ostream& operator<<(std::ostream& out, const std::vector>& nodes) +{ out << "AstNodes{"; - - for (const auto &node : nodes) { + + for (const auto& node : nodes) + { out << node << ", "; } - + out << "}"; - + return out; } -std::ostream& operator<<(std::ostream &out, const std::vector> &nodes) { +std::ostream& operator<<(std::ostream& out, const std::vector>& nodes) +{ out << "Operators{"; - - for (const auto &node : nodes) { + + for (const auto& node : nodes) + { out << node << ", "; } - + out << "}"; - + return out; } \ No newline at end of file diff --git a/filesystem/src/query_lang/ast.hpp b/filesystem/src/query_lang/ast.hpp index eb0b018..c78e7d2 100644 --- a/filesystem/src/query_lang/ast.hpp +++ b/filesystem/src/query_lang/ast.hpp @@ -4,32 +4,32 @@ // SPDX-License-Identifier: BSD-3-Clause #pragma once -#include -#include #include #include +#include +#include -class AstNode { -private: - -public: +class AstNode +{ + private: + public: AstNode(); virtual std::string str() const = 0; - + virtual bool match(const std::shared_ptr other) const = 0; - virtual bool assembleAST(std::vector> *rpn, std::vector>::iterator *rpn_iter) = 0; - - friend std::ostream& operator<<(std::ostream &out, const std::shared_ptr node); -}; + virtual bool assembleAST(std::vector>* rpn, + std::vector>::iterator* rpn_iter) = 0; + friend std::ostream& operator<<(std::ostream& out, const std::shared_ptr node); +}; -class Operator : public AstNode { -private: +class Operator : public AstNode +{ + private: int precedence; -public: - + public: Operator(int precedence); virtual std::string str() const; @@ -37,9 +37,9 @@ class Operator : public AstNode { bool operator>=(const std::shared_ptr other) const; }; - -class BinaryOperator : public Operator { -public: +class BinaryOperator : public Operator +{ + public: std::shared_ptr left_node; std::shared_ptr right_node; @@ -48,12 +48,13 @@ class BinaryOperator : public Operator { virtual std::string str() const; - bool assembleAST(std::vector> *rpn, std::vector>::iterator *rpn_iter); + bool assembleAST(std::vector>* rpn, + std::vector>::iterator* rpn_iter); }; - -class UnaryOperator : public Operator { -public: +class UnaryOperator : public Operator +{ + public: std::shared_ptr node; UnaryOperator(int precedence); @@ -61,13 +62,14 @@ class UnaryOperator : public Operator { virtual std::string str() const; - bool assembleAST(std::vector> *rpn, std::vector>::iterator *rpn_iter); + bool assembleAST(std::vector>* rpn, + std::vector>::iterator* rpn_iter); }; +class Union : public BinaryOperator +{ -class Union : public BinaryOperator { - -public: + public: Union(); Union(std::shared_ptr left, std::shared_ptr right); @@ -75,10 +77,10 @@ class Union : public BinaryOperator { virtual bool match(const std::shared_ptr other) const; }; +class Intersection : public BinaryOperator +{ -class Intersection : public BinaryOperator { - -public: + public: Intersection(); Intersection(std::shared_ptr left, std::shared_ptr right); @@ -86,10 +88,10 @@ class Intersection : public BinaryOperator { virtual bool match(const std::shared_ptr other) const; }; +class Negation : public UnaryOperator +{ -class Negation : public UnaryOperator { - -public: + public: Negation(); Negation(std::shared_ptr node); @@ -97,9 +99,9 @@ class Negation : public UnaryOperator { virtual bool match(const std::shared_ptr other) const; }; - -class Tag : public AstNode { -public: +class Tag : public AstNode +{ + public: std::string name; Tag(std::string name); @@ -107,8 +109,9 @@ class Tag : public AstNode { virtual std::string str() const; virtual bool match(const std::shared_ptr other) const; - bool assembleAST(std::vector> *rpn, std::vector>::iterator *rpn_iter); + bool assembleAST(std::vector>* rpn, + std::vector>::iterator* rpn_iter); }; -std::ostream& operator<<(std::ostream &out, const std::vector> &nodes); -std::ostream& operator<<(std::ostream &out, const std::vector> &nodes); +std::ostream& operator<<(std::ostream& out, const std::vector>& nodes); +std::ostream& operator<<(std::ostream& out, const std::vector>& nodes); diff --git a/filesystem/src/query_lang/parser.cpp b/filesystem/src/query_lang/parser.cpp index 4b47aed..081507a 100644 --- a/filesystem/src/query_lang/parser.cpp +++ b/filesystem/src/query_lang/parser.cpp @@ -5,59 +5,56 @@ #include "parser.hpp" -#include #include +#include -#include #include +#include -Token::Token() - : token("") {} +Token::Token() : token("") {} -Token::Token(std::string str) - : token(str) {} +Token::Token(std::string str) : token(str) {} -std::ostream& operator<<(std::ostream &out, const Token &token) { +std::ostream& operator<<(std::ostream& out, const Token& token) +{ out << token.str(); return out; } -bool Token::operator==(const Token& other) const { - return this->token == other.token; -} - -size_t Token::len() const { - return this->token.length(); -} +bool Token::operator==(const Token& other) const { return this->token == other.token; } -std::string Token::str() const { - return this->token; -} +size_t Token::len() const { return this->token.length(); } -void Token::append(char character) { - this->token.push_back(character); -} +std::string Token::str() const { return this->token; } +void Token::append(char character) { this->token.push_back(character); } -std::vector tokenize(std::string expression) { +std::vector tokenize(std::string expression) +{ std::vector token_list; Token token_accum; - auto captureToken = [&]() { - if (token_accum.len() > 0) { + auto captureToken = [&]() + { + if (token_accum.len() > 0) + { token_list.push_back(token_accum); } token_accum = Token(); }; - for (const auto &character : expression) { - if (isspace(character)) { + for (const auto& character : expression) + { + if (isspace(character)) + { captureToken(); } - else if (isalnum(character) || character == '_') { + else if (isalnum(character) || character == '_') + { token_accum.append(character); } - else { + else + { switch (character) { case '&': @@ -84,17 +81,22 @@ std::vector tokenize(std::string expression) { return token_list; } -std::vector> parseRpn(std::vector::iterator* token_iter, std::vector::iterator end_iter) { +std::vector> parseRpn(std::vector::iterator* token_iter, + std::vector::iterator end_iter) +{ std::vector> rpn; std::vector> stack; - auto putStack = [&](std::shared_ptr op) { - if (stack.size() > 0) { + auto putStack = [&](std::shared_ptr op) + { + if (stack.size() > 0) + { auto last = stack.back(); - //Check if the last operator is equal or higher precedence. - //This is done when equal in order to preserve argument ordering. - //This ensures {1+2+3 --> 1 2 + 3 +} instead of {1+2+3 --> 1 2 3 + +} - if (last >= op) { + // Check if the last operator is equal or higher precedence. + // This is done when equal in order to preserve argument ordering. + // This ensures {1+2+3 --> 1 2 + 3 +} instead of {1+2+3 --> 1 2 3 + +} + if (last >= op) + { rpn.push_back(last); stack.pop_back(); } @@ -102,39 +104,47 @@ std::vector> parseRpn(std::vector::iterator* tok stack.push_back(op); }; - //Loop through the tokens - while(*token_iter != end_iter) { + // Loop through the tokens + while (*token_iter != end_iter) + { Token token = **token_iter; (*token_iter)++; std::string token_str = token.str(); - if (token_str == "&" || token_str == "*") { + if (token_str == "&" || token_str == "*") + { putStack(std::make_shared()); } - else if (token_str == "|" || token_str == "+") { + else if (token_str == "|" || token_str == "+") + { putStack(std::make_shared()); } - else if (token_str == "!" || token_str == "-") { + else if (token_str == "!" || token_str == "-") + { putStack(std::make_shared()); } - else if (token_str == "(") { - //Parse the sub-expression + else if (token_str == "(") + { + // Parse the sub-expression auto sub_expr = parseRpn(token_iter, end_iter); rpn.insert(rpn.end(), sub_expr.begin(), sub_expr.end()); } - else if (token_str == ")") { - //Exit the for loop + else if (token_str == ")") + { + // Exit the for loop break; } - else { + else + { auto node = std::make_shared(token_str); rpn.push_back(node); } } - //Remove all remaining items from the stack - while (stack.size() > 0) { + // Remove all remaining items from the stack + while (stack.size() > 0) + { rpn.push_back(stack.back()); stack.pop_back(); } @@ -142,27 +152,31 @@ std::vector> parseRpn(std::vector::iterator* tok return rpn; } -std::optional> parse(std::string expression) { +std::optional> parse(std::string expression) +{ spdlog::trace("Entering parse(expression={0})", expression); - + std::vector tokens = tokenize(expression); std::vector::iterator token_iter = tokens.begin(); - //Parse the expression and convert to RPN + // Parse the expression and convert to RPN std::vector> rpn = parseRpn(&token_iter, tokens.end()); - + spdlog::debug("Complete RPN:"); - for (const auto& rpn_item : rpn) { + for (const auto& rpn_item : rpn) + { spdlog::debug("{0}", rpn_item->str()); } spdlog::debug("Printing RPN done!"); - //Convert the RPN representation to an AST + // Convert the RPN representation to an AST std::vector>::iterator rpn_iter = rpn.begin(); - while (rpn_iter != rpn.end()) { + while (rpn_iter != rpn.end()) + { // TODO: manipulating an iterator like this (deleting elements) is undefined behaviour - if (!((*rpn_iter)->assembleAST(&rpn, &rpn_iter))) { + if (!((*rpn_iter)->assembleAST(&rpn, &rpn_iter))) + { return {}; } @@ -174,9 +188,11 @@ std::optional> parse(std::string expression) { return rpn.back(); } -std::ostream& operator<<(std::ostream &out, const std::vector &tokens) { +std::ostream& operator<<(std::ostream& out, const std::vector& tokens) +{ out << "Tokens{"; - for (const auto &token : tokens) { + for (const auto& token : tokens) + { out << token << ", "; } out << "}"; diff --git a/filesystem/src/query_lang/parser.hpp b/filesystem/src/query_lang/parser.hpp index 5b64f09..e7d9152 100644 --- a/filesystem/src/query_lang/parser.hpp +++ b/filesystem/src/query_lang/parser.hpp @@ -9,11 +9,12 @@ #include "ast.hpp" -class Token { -private: +class Token +{ + private: std::string token; -public: + public: Token(); Token(std::string str); @@ -23,10 +24,10 @@ class Token { std::string str() const; void append(char character); - friend std::ostream& operator<<(std::ostream &out, const Token &token); + friend std::ostream& operator<<(std::ostream& out, const Token& token); }; -std::ostream& operator<<(std::ostream &out, const std::vector &tokens); +std::ostream& operator<<(std::ostream& out, const std::vector& tokens); std::vector tokenize(std::string expression); std::optional> parse(std::string expression); diff --git a/filesystem/tests/vendors/sqlite.cpp b/filesystem/tests/vendors/sqlite.cpp index 6b3ef5a..2e2fc76 100644 --- a/filesystem/tests/vendors/sqlite.cpp +++ b/filesystem/tests/vendors/sqlite.cpp @@ -24,14 +24,12 @@ TEST_CASE("SQLite3 Querying", "[vendor]") REQUIRE(rc == SQLITE_OK); - rc = sqlite3_exec(db, - "CREATE TABLE test (id INTEGER PRIMARY KEY, name TEXT);", - nullptr, nullptr, nullptr); + rc = sqlite3_exec(db, "CREATE TABLE test (id INTEGER PRIMARY KEY, name TEXT);", nullptr, + nullptr, nullptr); REQUIRE(rc == SQLITE_OK); - rc = sqlite3_exec(db, "INSERT INTO test (name) VALUES ('Test!');", nullptr, - nullptr, nullptr); + rc = sqlite3_exec(db, "INSERT INTO test (name) VALUES ('Test!');", nullptr, nullptr, nullptr); REQUIRE(rc == SQLITE_OK); @@ -46,8 +44,7 @@ TEST_CASE("SQLite3 Querying", "[vendor]") while (sqlite3_step(stmt) == SQLITE_ROW) { // Retrieve the value of the "name" column - const char* name = - reinterpret_cast(sqlite3_column_text(stmt, 0)); + const char* name = reinterpret_cast(sqlite3_column_text(stmt, 0)); REQUIRE(std::string(name) == "Test!"); } diff --git a/makefile b/makefile index 17aba12..46651a0 100644 --- a/makefile +++ b/makefile @@ -44,10 +44,12 @@ test: build local-test: unshare -pfr --user --mount --kill-child meson test -C $(BUILD_DIR) --print-errorlogs +TEST_SRC != find filesystem/tests/ -type f | xargs +SRC != find filesystem/src/ -type f | xargs + .PHONY: format format: - -clang-format filesystem/tests/** -i - -clang-format filesystem/src/** -i + clang-format $(SRC) $(TEST_SRC) -i .PHONY: clean clean: From 44ea93608ff03a9c8317794bfab25da4f5ace6aa Mon Sep 17 00:00:00 2001 From: Caleb Depatie Date: Tue, 11 Nov 2025 16:35:45 -0500 Subject: [PATCH 22/23] Version bump --- flake.nix | 2 +- meson.build | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 3dbec44..2c43b51 100644 --- a/flake.nix +++ b/flake.nix @@ -40,7 +40,7 @@ # Built via `nix build` and run via `nix run` packages.default = pkgs.stdenv.mkDerivation rec { pname = "lakefs"; - version = "0.2.1"; + version = "0.2.3"; src = self; diff --git a/meson.build b/meson.build index 21aebbc..62d72d2 100644 --- a/meson.build +++ b/meson.build @@ -6,7 +6,7 @@ project( 'LakeFS', ['cpp', 'c', 'd'], - version: '0.2.2', + version: '0.2.3', license: 'BSD-3-Clause' ) From c031f7899d8f28f444dcbf6f636c97a05faf708c Mon Sep 17 00:00:00 2001 From: Caleb Depatie Date: Tue, 11 Nov 2025 16:39:31 -0500 Subject: [PATCH 23/23] clang-format version bump --- .github/workflows/format.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 944037c..41ebfc9 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -21,5 +21,5 @@ jobs: - name: Run clang-format style check uses: jidicula/clang-format-action@v4.16.0 with: - clang-format-version: '13' + clang-format-version: '19' check-path: ${{ matrix.path }} \ No newline at end of file