dump

.clang-format

@@ -0,0 +1,76 @@
+# Generated from CLion C/C++ Code Style settings
+---
+Language: Cpp
+BasedOnStyle: LLVM
+AccessModifierOffset: -4
+AlignAfterOpenBracket: Align
+AlignConsecutiveAssignments:
+  Enabled: true
+  AcrossEmptyLines: false
+  AcrossComments: false
+AlignConsecutiveBitFields:
+  Enabled: true
+  AcrossEmptyLines: false
+  AcrossComments: false
+AlignConsecutiveDeclarations:
+  Enabled: true
+  AcrossEmptyLines: false
+  AcrossComments: false
+AlignConsecutiveMacros:
+  Enabled: true
+  AcrossEmptyLines: false
+  AcrossComments: false
+AlignTrailingComments:
+  Kind: Always
+  OverEmptyLines: 2
+SpacesBeforeTrailingComments: 1
+AlignOperands: Align
+AlignEscapedNewlines: Right
+AlwaysBreakTemplateDeclarations: Yes
+BraceWrapping:
+  AfterCaseLabel: false
+  AfterClass: false
+  AfterControlStatement: false
+  AfterEnum: false
+  AfterFunction: false
+  AfterNamespace: false
+  AfterStruct: false
+  AfterUnion: false
+  AfterExternBlock: false
+  BeforeCatch: false
+  BeforeElse: false
+  BeforeLambdaBody: false
+  BeforeWhile: false
+  SplitEmptyFunction: true
+  SplitEmptyRecord: true
+  SplitEmptyNamespace: true
+BreakBeforeBraces: Custom
+BreakConstructorInitializers: AfterColon
+BreakConstructorInitializersBeforeComma: false
+ColumnLimit: 120
+ConstructorInitializerAllOnOneLineOrOnePerLine: false
+ContinuationIndentWidth: 8
+IncludeCategories:
+  - Regex: '^<.*'
+    Priority: 1
+  - Regex: '^".*'
+    Priority: 2
+  - Regex: '.*'
+    Priority: 3
+IncludeIsMainRegex: '([-_](test|unittest))?$'
+IndentCaseLabels: true
+IndentWidth: 4
+InsertNewlineAtEOF: true
+MacroBlockBegin: ''
+MacroBlockEnd: ''
+MaxEmptyLinesToKeep: 2
+PointerAlignment: Left
+SpaceAfterCStyleCast: true
+SpaceAfterTemplateKeyword: false
+SpaceBeforeRangeBasedForLoopColon: false
+SpaceInEmptyParentheses: false
+SpacesInAngles: false
+SpacesInConditionalStatement: false
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+...

.gitignore

@@ -0,0 +1,80 @@
+.DS_Store
+
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+.idea
+*.out
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# AWS User-specific
+.idea/**/aws.xml
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/artifacts
+# .idea/compiler.xml
+# .idea/jarRepositories.xml
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# SonarLint plugin
+.idea/sonarlint/
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser

CMakeLists.txt

@@ -0,0 +1,41 @@
+cmake_minimum_required(VERSION 3.15)
+project(remotefs)
+
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED YES)
+
+if (CMAKE_BUILD_TYPE STREQUAL "Debug")
+    if (NOT DEFINED SANITIZE)
+        set(SANITIZE YES)
+    endif ()
+endif ()
+
+if (SANITIZE STREQUAL "YES")
+    message(STATUS "Enabling sanitizers!")
+    add_compile_options(-Werror -O0 -Wall -Wextra -pedantic -Wno-unused-parameter -Wno-unused-variable
+            -Wno-error=unused-function
+            -Wshadow -Wformat=2 -Wfloat-equal -D_GLIBCXX_DEBUG -Wconversion)
+    add_compile_options(-fsanitize=address -fno-sanitize-recover)
+    add_link_options(-fsanitize=address -fno-sanitize-recover)
+endif ()
+
+if (CMAKE_BUILD_TYPE STREQUAL "Release")
+    set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE)
+endif ()
+
+if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
+    add_compile_options(-O3)
+    add_link_options(-O3)
+endif ()
+
+add_compile_options(-Wno-c99-extensions)
+
+add_link_options(-rdynamic)
+
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
+
+enable_testing()
+
+add_subdirectory(utils)
+add_subdirectory(networking)
+add_subdirectory(remotefs)

README.md

@@ -0,0 +1,72 @@
+# RemoteFS
+
+Simple server-client remote filesystem
+
+## How to use
+
+1. Create an SSL certificate for the server `openssl req  -nodes -new -x509  -keyout key.pem -out cert.pem`
+1. Create a password hash for the user `echo -n "password" | openssl dgst -sha256`
+1. Copy the password hash into a `users` file
+1. Start the server `remotefs --mode:server --path:<path to filesystem root> --users_path:users`
+1. Copy `cert.pem` to the client working directory
+1. Connect with the client `remotefs --mode:client --username:<username> --password:<password> --path:<mount point>`
+
+## Advanced options
+
+Various options can be overridden using `--<option>:<value>` syntax
+
+- `ip` - ip server will listen on, or client will connect to, default is `127.0.0.1`
+- `port` - port server will listen on, or client will connect to, default is `42069`
+- `default_log_level` - default logging level, 1 is least verbose, 4 is most verbose, default is `2`
+- `timeout` - timeout, default is 30 (seconds)
+- `ca_path` - path to SSL certificate, default is `cert.pem`
+- `pk_path` - path to SSL private key (for server), default is `key.pem`
+- `mode` - server or client, default is `server`
+- `path` - filesystem root to server or mountpoint for server and client (default is empty)
+- `acl_path` - path for an ACL config file, default is empty (and everything is allowed)
+- `users_path` - path for file with user passwords (default is `users`)
+- `username` - username to use for client
+- `password` - password to use for client
+
+Example with some of these options:
+
+```
+remotefs --mode:client  --default_log_level:3 --timeout:10 --username:user2 --password:password2 --path:"${HOME}/remotefs-test/mount2" --ip:192.168.88.20
+```
+
+## ACL and users configuration
+
+Username passwords can be configured in a `users` file, with a simple format
+
+```
+<username> <sha256 password hash>
+```
+
+For example, two users with `password` password
+
+```
+username 5e884898da28047151d0e56f8dc6292773603d0d6aabbdd62a11ef721d1542d8
+username2 5e884898da28047151d0e56f8dc6292773603d0d6aabbdd62a11ef721d1542d8
+```
+
+ACL can be configured in an optionally specified acl file, with format:
+
+```
+<path prefix> <allowed users, separated by string>
+```
+
+If user list is empty, then everyone is allowed. If there are multiple entries with same prefix,
+the more specific one has higher priority.
+
+Example:
+
+```
+/ user1
+/A user1 user2
+/B user2
+/C
+```
+
+In this example, all files will be accessible "by default" only by user1,
+with files in directory `/A` by user1 and user2, in directory `/B` by user2, and in
+directory `/C` by everyone.

cmake/gTest.cmake

@@ -0,0 +1,10 @@
+include(FetchContent)
+FetchContent_Declare(
+        googletest
+        URL https://github.com/google/googletest/archive/03597a01ee50ed33e9dfd640b249b4be3799d395.zip
+)
+# For Windows: Prevent overriding the parent project's compiler/linker settings
+set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
+FetchContent_MakeAvailable(googletest)
+
+include(GoogleTest)

networking/CMakeLists.txt

@@ -0,0 +1,24 @@
+cmake_minimum_required(VERSION 3.15)
+
+add_library(networking
+        include/Server.hpp
+        src/Server.cpp
+        include/Client.hpp
+        src/Client.cpp
+        include/Helpers.hpp
+        src/Helpers.cpp
+        src/AsyncSslTransport.cpp
+        include/AsyncSslTransport.hpp
+        src/AsyncSslClientTransport.cpp
+        include/AsyncSslClientTransport.hpp
+        include/AsyncSslServerTransport.hpp
+        src/AsyncSslServerTransport.cpp
+)
+
+target_include_directories(networking PUBLIC include)
+
+find_package(OpenSSL REQUIRED)
+find_package(PkgConfig REQUIRED)
+pkg_check_modules(FUSE REQUIRED IMPORTED_TARGET fuse)
+
+target_link_libraries(networking PRIVATE utils OpenSSL::SSL OpenSSL::Crypto PkgConfig::FUSE)

networking/include/AsyncSslClientTransport.hpp

@@ -0,0 +1,34 @@
+//
+// Created by Stepan Usatiuk on 14.12.2024.
+//
+
+#ifndef ASYNCMESSAGECLIENT_HPP
+#define ASYNCMESSAGECLIENT_HPP
+
+#include "AsyncSslTransport.hpp"
+
+class AsyncSslClientTransport : public AsyncSslTransport {
+public:
+    AsyncSslClientTransport(SSL_CTX* ssl_ctx, int fd) : AsyncSslTransport(ssl_ctx, fd) {}
+
+    using SharedMsgPromiseT = std::shared_ptr<std::promise<std::shared_ptr<MsgWrapper>>>;
+
+    std::future<std::shared_ptr<MsgWrapper>> send_msg(std::vector<uint8_t> message);
+    std::vector<uint8_t>                     send_msg_and_wait(std::vector<uint8_t> message);
+
+protected:
+    void handle_message(std::shared_ptr<MsgWrapper> msg) override;
+    void handle_fail() override {
+        stop();
+        std::exit(EXIT_FAILURE);
+    }
+
+    void before_entry() override;
+
+private:
+    std::unordered_map<decltype(MsgWrapper::id), SharedMsgPromiseT> _promises;
+    uint64_t                                                        _msg_id = 0;
+    std::mutex                                                      _promises_mutex;
+};
+
+#endif // ASYNCMESSAGECLIENT_HPP

networking/include/AsyncSslServerTransport.hpp

@@ -0,0 +1,32 @@
+//
+// Created by Stepan Usatiuk on 15.12.2024.
+//
+
+#ifndef SERVERMESSAGEPUMP_HPP
+#define SERVERMESSAGEPUMP_HPP
+
+#include "AsyncSslTransport.hpp"
+
+class AsyncSslServerTransport : public AsyncSslTransport {
+public:
+    AsyncSslServerTransport(SSL_CTX* ssl_ctx, int fd, int client_id) : AsyncSslTransport(ssl_ctx, fd), _client_id(client_id) {}
+
+    // Null if finished
+    std::shared_ptr<MsgWrapper> get_msg();
+
+protected:
+    void handle_message(std::shared_ptr<MsgWrapper> msg) override;
+    void handle_fail() override;
+
+    void before_entry() override;
+
+private:
+    int _client_id;
+
+    std::deque<std::shared_ptr<MsgWrapper>> _msgs;
+    std::mutex                              _msgs_mutex;
+    std::condition_variable                 _msgs_condition;
+};
+
+
+#endif // SERVERMESSAGEPUMP_HPP

networking/include/AsyncSslTransport.hpp

@@ -0,0 +1,62 @@
+//
+// Created by Stepan Usatiuk on 14.12.2024.
+//
+
+#ifndef MESSAGEPUMP_HPP
+#define MESSAGEPUMP_HPP
+
+#include <condition_variable>
+#include <deque>
+#include <future>
+#include <mutex>
+#include <thread>
+#include <unordered_map>
+
+#include <openssl/ssl.h>
+
+#include "Helpers.hpp"
+
+class AsyncSslTransport {
+public:
+    AsyncSslTransport(SSL_CTX* ssl_ctx, int fd);
+    virtual ~AsyncSslTransport() = 0;
+    void run();
+
+    void send_message(std::shared_ptr<MsgWrapper> msg);
+
+    bool is_failed() const { return _failed; }
+    bool is_stopped() const { return _stopped; }
+
+    void stop();
+
+protected:
+    virtual void handle_message(std::shared_ptr<MsgWrapper> msg) = 0;
+    virtual void handle_fail()                                   = 0;
+
+    virtual void before_entry() {}
+
+    std::unique_ptr<SSL, decltype(&SSL_free)> _ssl{nullptr, &SSL_free};
+    int                                       _fd;
+
+private:
+    void thread_entry();
+
+    std::atomic<bool>       _stopped = 0;
+    std::mutex              _stopped_mutex;
+    std::condition_variable _stopped_condition;
+
+    std::thread _thread;
+
+    std::mutex                              _to_send_mutex;
+    int                                     _to_send_notif_pipe[2];
+    std::deque<std::shared_ptr<MsgWrapper>> _to_send;
+
+    std::atomic<bool> _failed;
+
+    AsyncSslTransport(const AsyncSslTransport& other)                = delete;
+    AsyncSslTransport(AsyncSslTransport&& other) noexcept            = delete;
+    AsyncSslTransport& operator=(const AsyncSslTransport& other)     = delete;
+    AsyncSslTransport& operator=(AsyncSslTransport&& other) noexcept = delete;
+};
+
+#endif // MESSAGEPUMP_HPP

networking/include/Client.hpp

@@ -0,0 +1,43 @@
+//
+// Created by stepus53 on 11.12.24.
+//
+
+#ifndef TCPCLIENT_HPP
+#define TCPCLIENT_HPP
+
+#include <cstdint>
+#include <mutex>
+#include <string>
+#include <memory>
+#include <optional>
+
+#include <openssl/ssl.h>
+
+#include "AsyncSslClientTransport.hpp"
+
+class Client {
+public:
+    Client(uint16_t port, std::string ip, std::string cert_path, std::string key_path);
+
+    void run();
+
+    AsyncSslClientTransport& transport() { return *_transport; }
+
+protected:
+    uint16_t    _port;
+    std::string _ip;
+
+    std::string _cert_path;
+    std::string _key_path;
+
+    std::unique_ptr<SSL_CTX, decltype(&SSL_CTX_free)> _ssl_ctx;
+    std::unique_ptr<SSL, decltype(&SSL_free)>         _ssl{nullptr, &SSL_free};
+
+    int    _sock;
+    size_t _msg_id = 0;
+
+private:
+    std::optional<AsyncSslClientTransport> _transport;
+};
+
+#endif // TCPCLIENT_HPP

networking/include/Helpers.hpp

@@ -0,0 +1,37 @@
+
+
+#ifndef NETWORKING_HELPERS_H
+#define NETWORKING_HELPERS_H
+
+#include <cstdint>
+#include <deque>
+#include <functional>
+#include <vector>
+
+#include "Options.h"
+#include "stuff.hpp"
+
+#include <openssl/ssl.h>
+
+using MsgIdType = uint64_t;
+
+struct MsgWrapper {
+    MsgIdType            id;
+    std::vector<uint8_t> data;
+};
+
+struct MsgHeader {
+    uint64_t id;
+    uint64_t len;
+} __attribute__((packed));
+
+namespace Helpers {
+void poll_wait(int fd, bool write, int timeout = checked_cast<int>(Options::get<size_t>("timeout")) * 1000);
+bool SSL_write(SSL* ctx, int fd, const std::vector<uint8_t>& buf);
+void init_nonblock(int fd);
+std::vector<uint8_t> SSL_read_n(SSL* ctx, int fd, size_t n);
+MsgWrapper           SSL_read_msg(SSL* ctx, int fd);
+void                 SSL_send_msg(SSL* ctx, int fd, const MsgWrapper& buf);
+} // namespace Helpers
+
+#endif

networking/include/Server.hpp

@@ -0,0 +1,53 @@
+//
+// Created by Stepan Usatiuk on 09.12.2024.
+//
+
+#ifndef TCPSERVER_HPP
+#define TCPSERVER_HPP
+
+#include <atomic>
+#include <condition_variable>
+#include <cstdint>
+#include <functional>
+#include <optional>
+#include <string>
+
+#include <openssl/ssl.h>
+
+#include "AsyncSslServerTransport.hpp"
+#include "Helpers.hpp"
+
+struct ClientCtx {
+    std::optional<std::string> client_name;
+    AsyncSslServerTransport    transport;
+    std::mutex                 ctx_mutex;
+};
+
+class Server {
+public:
+    Server(uint16_t port, uint32_t ip, std::string cert_path, std::string key_path);
+
+    void run();
+
+protected:
+    uint16_t _port;
+    uint32_t _ip;
+
+    std::string _cert_path;
+    std::string _key_path;
+
+    std::unique_ptr<SSL_CTX, decltype(&SSL_CTX_free)> _ssl_ctx;
+
+    void process_req(int conn_fd);
+
+    virtual std::vector<uint8_t> handle_message(ClientCtx& client, std::vector<uint8_t> data) = 0;
+
+private:
+    std::atomic<int>        _total_req{0};
+    std::atomic<int>        _req_in_progress{0};
+    std::mutex              _req_in_progress_mutex;
+    std::condition_variable _req_in_progress_cond;
+};
+
+
+#endif // TCPSERVER_HPP

networking/src/AsyncSslClientTransport.cpp

@@ -0,0 +1,60 @@
+//
+// Created by Stepan Usatiuk on 14.12.2024.
+//
+
+#include "AsyncSslClientTransport.hpp"
+
+#include <openssl/err.h>
+#include <openssl/ssl.h>
+
+#include "Logger.h"
+
+std::future<std::shared_ptr<MsgWrapper>> AsyncSslClientTransport::send_msg(std::vector<uint8_t> message) {
+    auto promise = std::make_shared<std::promise<std::shared_ptr<MsgWrapper>>>();
+
+    decltype(_msg_id) id;
+    {
+        std::lock_guard lock(_promises_mutex);
+        id = _msg_id++;
+        _promises.emplace(id, promise);
+    }
+
+    send_message(std::make_shared<MsgWrapper>(id, std::move(message)));
+
+    return promise->get_future();
+}
+
+std::vector<uint8_t> AsyncSslClientTransport::send_msg_and_wait(std::vector<uint8_t> message) {
+    auto future = send_msg(std::move(message));
+    return future.get()->data;
+}
+
+void AsyncSslClientTransport::before_entry() {
+    Logger::log(Logger::RemoteFs, [&](std::ostream& os) { os << "Connecting"; }, Logger::INFO);
+
+    int r;
+    while ((r = SSL_connect(_ssl.get())) <= 0) {
+        ERR_print_errors_fp(stderr);
+        int err = SSL_get_error(_ssl.get(), r);
+        if (err == SSL_ERROR_WANT_READ || err == SSL_ERROR_WANT_WRITE) {
+            Helpers::poll_wait(_fd, err == SSL_ERROR_WANT_WRITE);
+            continue;
+        }
+        throw OpenSSLException("SSL_connect() failed");
+    }
+    Logger::log(Logger::RemoteFs, [&](std::ostream& os) { os << "Connected"; }, Logger::INFO);
+}
+
+void AsyncSslClientTransport::handle_message(std::shared_ptr<MsgWrapper> msg) {
+    std::lock_guard lock(_promises_mutex);
+    auto            future_it = _promises.find(msg->id);
+
+    if (future_it == _promises.end()) {
+        Logger::log(Logger::RemoteFs, "Could not find future for msg with id " + std::to_string(msg->id),
+                    Logger::ERROR);
+        return;
+    }
+
+    future_it->second->set_value(msg);
+    _promises.erase(future_it);
+}

networking/src/AsyncSslServerTransport.cpp

@@ -0,0 +1,51 @@
+//
+// Created by Stepan Usatiuk on 15.12.2024.
+//
+
+#include "AsyncSslServerTransport.hpp"
+
+#include <openssl/err.h>
+#include <openssl/ssl.h>
+
+#include "Logger.h"
+
+void AsyncSslServerTransport::handle_message(std::shared_ptr<MsgWrapper> msg) {
+    std::lock_guard lock(_msgs_mutex);
+    _msgs.emplace_back(msg);
+    _msgs_condition.notify_all();
+}
+
+
+void AsyncSslServerTransport::handle_fail() {
+    _msgs_condition.notify_all();
+}
+
+void AsyncSslServerTransport::before_entry() {
+    int r;
+    while ((r = SSL_accept(_ssl.get())) <= 0) {
+        ERR_print_errors_fp(stderr);
+        int err = SSL_get_error(_ssl.get(), r);
+        if (err == SSL_ERROR_WANT_READ || err == SSL_ERROR_WANT_WRITE) {
+            Helpers::poll_wait(_fd, err == SSL_ERROR_WANT_WRITE);
+            continue;
+        }
+        throw OpenSSLException("SSL_accept() failed");
+    }
+    Logger::log(Logger::RemoteFs, "Client " + std::to_string(_client_id) + " connected\n", Logger::INFO);
+}
+
+std::shared_ptr<MsgWrapper> AsyncSslServerTransport::get_msg() {
+    std::unique_lock lock(_msgs_mutex);
+    _msgs_condition.wait(lock, [&] { return !_msgs.empty() || is_failed() || is_stopped(); });
+
+    if (is_failed())
+        return nullptr;
+
+    auto ret = _msgs.begin();
+
+    auto real_ret = *ret;
+
+    _msgs.erase(_msgs.begin());
+
+    return real_ret;
+}

networking/src/AsyncSslTransport.cpp

@@ -0,0 +1,249 @@
+//
+// Created by Stepan Usatiuk on 14.12.2024.
+//
+
+#include "AsyncSslTransport.hpp"
+
+#include <fcntl.h>
+#include <openssl/err.h>
+#include <openssl/ssl.h>
+#include <poll.h>
+#include <unistd.h>
+
+#include <iomanip>
+
+#include "Logger.h"
+#include "Serialize.hpp"
+#include "stuff.hpp"
+
+AsyncSslTransport::AsyncSslTransport(SSL_CTX* ssl_ctx, int fd) : _ssl(SSL_new(ssl_ctx), &SSL_free), _fd(fd) {
+    SSL_set_fd(_ssl.get(), _fd);
+    pipe(_to_send_notif_pipe);
+    Helpers::init_nonblock(_fd);
+}
+
+AsyncSslTransport::~AsyncSslTransport() {
+    stop();
+    _thread.join();
+}
+
+void AsyncSslTransport::stop() {
+    std::unique_lock lock(_stopped_mutex);
+    _stopped_condition.notify_all();
+    _stopped = true;
+    write(_to_send_notif_pipe[1], "1", 1);
+}
+
+void AsyncSslTransport::send_message(std::shared_ptr<MsgWrapper> msg) {
+    std::unique_lock lock(_to_send_mutex);
+    _to_send.push_back(std::move(msg));
+    write(_to_send_notif_pipe[1], "1", 1);
+}
+
+void AsyncSslTransport::thread_entry() {
+    try {
+        before_entry();
+
+        bool                 sending = false;
+        std::vector<uint8_t> to_send_buf{};
+        size_t               cur_sent = 0;
+
+        bool                 reading_msg = false; // False if reading header, true if message
+        std::vector<uint8_t> read_buf;
+        size_t               cur_read{};
+        uint64_t             msg_id  = 0;
+        size_t               msg_len = sizeof(MsgHeader); // Message length to read if reading message, or header len
+        read_buf.resize(msg_len);
+
+        while (!_stopped) {
+            if (!sending) {
+                std::shared_ptr<MsgWrapper> to_send_now{};
+
+                std::unique_lock lock(_to_send_mutex);
+                auto             next = _to_send.begin();
+                if (next != _to_send.end()) {
+                    to_send_now = *next;
+                    _to_send.erase(next);
+                }
+
+                if (to_send_now) {
+                    MsgHeader header{};
+                    header.id  = htobe64(to_send_now->id);
+                    header.len = htobe64(to_send_now->data.size());
+                    to_send_buf.resize(sizeof(MsgHeader) + to_send_now->data.size());
+                    memcpy(to_send_buf.data(), &header, sizeof(MsgHeader));
+                    memcpy(to_send_buf.data() + sizeof(MsgHeader), to_send_now->data.data(), to_send_now->data.size());
+                    sending = true;
+                    Logger::log(
+                            Logger::RemoteFs, [&](std::ostream& os) { os << "Started sending message " << to_send_now->id; },
+                            Logger::DEBUG);
+                }
+            }
+
+            while (sending) {
+                size_t written_now = 0;
+                int    ret;
+                if ((ret = SSL_write_ex(_ssl.get(), to_send_buf.data() + cur_sent, to_send_buf.size() - cur_sent,
+                                        &written_now)) <= 0) {
+                    int err = SSL_get_error(_ssl.get(), ret);
+                    if (err == SSL_ERROR_WANT_READ || err == SSL_ERROR_WANT_WRITE) {
+                        break;
+                    }
+
+                    throw OpenSSLException("write failed");
+                }
+                Logger::log(
+                        Logger::RemoteFs,
+                        [&](std::ostream& os) {
+                            os << "Written " << written_now;
+                            if (Logger::en_level(Logger::RemoteFs, Logger::TRACE)) {
+                                os << ": ";
+                                for (size_t i = 0; i < written_now; i++) {
+                                    os << std::setw(2) << std::setfill('0') << std::hex
+                                       << (int) to_send_buf[i + cur_sent] << " ";
+                                }
+                            }
+                        },
+                        Logger::DEBUG);
+                cur_sent += written_now;
+
+                if (cur_sent == to_send_buf.size()) {
+                    Logger::log(Logger::RemoteFs, [&](std::ostream& os) { os << "Finished sending"; }, Logger::DEBUG);
+                    to_send_buf.resize(0);
+                    cur_sent = 0;
+                    sending  = false;
+                }
+            }
+
+            while (true) {
+                int    ret;
+                size_t read_now = 0;
+                if ((ret = SSL_read_ex(_ssl.get(), read_buf.data() + cur_read, msg_len, &read_now)) <= 0) {
+                    int err = SSL_get_error(_ssl.get(), ret);
+                    if (err == SSL_ERROR_WANT_READ || err == SSL_ERROR_WANT_WRITE) {
+                        break;
+                    }
+
+                    throw OpenSSLException("read failed");
+                }
+                Logger::log(
+                        Logger::RemoteFs,
+                        [&](std::ostream& os) {
+                            os << "Read " << read_now;
+                            if (Logger::en_level(Logger::RemoteFs, Logger::TRACE)) {
+                                os << ": ";
+                                for (size_t i = 0; i < read_now; i++) {
+                                    os << std::setw(2) << std::setfill('0') << std::hex << (int) read_buf[i + cur_read]
+                                       << " ";
+                                }
+                            }
+                        },
+                        Logger::DEBUG);
+                cur_read += read_now;
+
+                if (cur_read == msg_len) {
+                    if (reading_msg) {
+                        handle_message(std::make_shared<MsgWrapper>(msg_id, std::move(read_buf)));
+                        reading_msg = false;
+                        msg_len     = sizeof(MsgHeader);
+                        Logger::log(
+                                Logger::RemoteFs,
+                                [&](std::ostream& os) { os << "Finished receiving message " << msg_id; },
+                                Logger::DEBUG);
+                    } else {
+                        MsgHeader hdr;
+                        memcpy(&hdr, read_buf.data(), sizeof(hdr));
+                        reading_msg = true;
+                        msg_len     = be64toh(hdr.len);
+                        msg_id      = be64toh(hdr.id);
+                        Logger::log(
+                                Logger::RemoteFs,
+                                [&](std::ostream& os) { os << "Started receiving message " << msg_id; }, Logger::DEBUG);
+                    }
+
+                    read_buf.clear();
+                    read_buf.resize(msg_len);
+                    cur_read = 0;
+                }
+            }
+
+            pollfd fds[2];
+
+            fds[0].fd     = _fd;
+            fds[0].events = POLLIN;
+            if (sending)
+                fds[0].events |= POLLOUT;
+            fds[0].revents = 0;
+
+            fds[1].fd      = _to_send_notif_pipe[0];
+            fds[1].events  = POLLIN;
+            fds[1].revents = 0;
+
+            Logger::log(Logger::RemoteFs, [&](std::ostream& os) { os << "Waiting"; }, Logger::DEBUG);
+
+
+            if (poll(fds, 2, checked_cast<int>(Options::get<size_t>("timeout")) * 1000) < 0) {
+                if (errno == EINTR)
+                    return;
+                throw ErrnoException("Could not poll");
+            }
+
+            if (!(fds[0].revents & POLLOUT) && !(fds[1].revents & POLLIN) && !(fds[0].revents & POLLIN)) {
+                throw ErrnoException("Could not poll (timeout?)");
+            }
+
+            if (fds[1].revents & POLLIN) {
+                char temp;
+                Logger::log(
+                        Logger::RemoteFs, [&](std::ostream& os) { os << "Received message on pipe"; }, Logger::DEBUG);
+                read(_to_send_notif_pipe[0], &temp, 1);
+            }
+
+            // Logger::log(
+            //         Logger::Server,
+            //         [&](std::ostream& os) {
+            //             os << "Sent " << (*next)->id << " with " << (*next)->data.size() << " bytes: ";
+            //             if (Logger::en_level(Logger::Server, Logger::TRACE))
+            //                 for (unsigned char i: (*next)->data)
+            //                     os << (int) i << " ";
+            //         },
+            //         Logger::DEBUG);
+            // Helpers::SSL_send_msg(_ssl, _fd, **next);
+        }
+    } catch (std::exception& e) {
+        _failed = true;
+        Logger::log(Logger::RemoteFs, e.what(), Logger::ERROR);
+        handle_fail();
+    }
+    SSL_shutdown(_ssl.get());
+    OPENSSL_thread_stop();
+}
+
+// void SSLMessagePump::receiver_entry() {
+//     try {
+//         before_receiver();
+//
+//         while (!_stopped) {
+//             auto msg = std::make_shared<MsgWrapper>(Helpers::SSL_read_msg(_ssl, _fd));
+//             Logger::log(
+//                     Logger::Server,
+//                     [&](std::ostream& os) {
+//                         os << "Received " << msg->id << " with " << msg->data.size() << " bytes: ";
+//                         if (Logger::en_level(Logger::Server, Logger::TRACE))
+//                             for (unsigned char i: msg->data)
+//                                 os << (int) i << " ";
+//                     },
+//                     Logger::DEBUG);
+//             handle_message(msg);
+//         }
+//     } catch (std::exception& e) {
+//         _failed = true;
+//         Logger::log(Logger::Server, e.what(), Logger::ERROR);
+//         handle_fail();
+//     }
+// }
+
+
+void AsyncSslTransport::run() {
+    _thread = std::thread([&]() { thread_entry(); });
+}

networking/src/Client.cpp

@@ -0,0 +1,72 @@
+//
+// Created by stepus53 on 11.12.24.
+//
+
+#include "Client.hpp"
+
+#include <arpa/inet.h>
+#include <fcntl.h>
+#include <netdb.h>
+#include <netinet/ip.h>
+#include <netinet/tcp.h>
+#include <poll.h>
+#include <signal.h>
+#include <sys/select.h>
+#include <sys/socket.h>
+#include <sys/time.h>
+#include <unistd.h>
+
+#include <openssl/err.h>
+
+#include "Exception.h"
+#include "Helpers.hpp"
+#include "Logger.h"
+
+// From https://wiki.openssl.org/index.php/Simple_TLS_Server
+static std::unique_ptr<SSL_CTX, decltype(&SSL_CTX_free)> create_context() {
+    const SSL_METHOD* method;
+    SSL_CTX*          ctx;
+
+    method = TLS_client_method();
+
+    ctx = SSL_CTX_new(method);
+    if (!ctx) {
+        throw OpenSSLException("Unable to create SSL context");
+    }
+
+    return {ctx, &SSL_CTX_free};
+}
+
+Client::Client(uint16_t port, std::string ip, std::string cert_path, std::string key_path) :
+    _port(port), _ip(ip), _cert_path(cert_path), _key_path(key_path), _ssl_ctx(create_context()) {
+    SSL_CTX_set_verify(_ssl_ctx.get(), SSL_VERIFY_PEER, nullptr);
+    if (SSL_CTX_load_verify_locations(_ssl_ctx.get(), cert_path.c_str(), nullptr) <= 0) {
+        throw OpenSSLException("Unable to read certificate file");
+    }
+}
+
+void Client::run() {
+    protoent* proto = getprotobyname("tcp");
+    if (proto == NULL) {
+        throw ErrnoException("Could not get TCP protocol info");
+    }
+
+    _sock = socket(AF_INET, SOCK_STREAM, proto->p_proto);
+    sockaddr_in addr;
+
+    addr.sin_family = AF_INET;
+    addr.sin_port   = htons(_port);
+
+    if (inet_pton(AF_INET, _ip.c_str(), &addr.sin_addr) <= 0)
+        throw ErrnoException("inet_pton()");
+
+    memset(addr.sin_zero, 0, sizeof(addr.sin_zero));
+#ifdef APPLE
+    addr.sin_len = sizeof(struct sockaddr_in),
+#endif
+
+    if (connect(_sock, reinterpret_cast<sockaddr*>(&addr), sizeof(addr)) < 0) throw ErrnoException("connect()");
+
+    _transport.emplace(_ssl_ctx.get(), _sock);
+    _transport->run();
+}

networking/src/Helpers.cpp

@@ -0,0 +1,93 @@
+//
+// Created by Stepan Usatiuk on 09.12.2024.
+//
+
+#include "Helpers.hpp"
+
+#include <fcntl.h>
+#include <poll.h>
+
+#include <openssl/err.h>
+#include <openssl/ssl.h>
+
+#include "Exception.h"
+#include "Options.h"
+#include "stuff.hpp"
+
+void Helpers::poll_wait(int fd, bool write, int timeout) {
+    pollfd p;
+    p.fd      = fd;
+    p.events  = write ? POLLOUT : POLLIN;
+    p.revents = 0;
+    if (poll(&p, 1, timeout) < 0) {
+        if (errno == EINTR)
+            return;
+        throw ErrnoException("Could not poll");
+    }
+    if (!(p.revents & (write ? POLLOUT : POLLIN))) {
+        throw ErrnoException("Could not poll (timeout?)");
+    }
+}
+
+void Helpers::init_nonblock(int fd) {
+    int r = fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK);
+    if (r < 0) {
+        throw ErrnoException("Could not set nonblocking mode");
+    }
+}
+
+bool Helpers::SSL_write(SSL* ctx, int fd, const std::vector<uint8_t>& buf) {
+    for (size_t written = 0; written < buf.size();) {
+        size_t written_now = 0;
+        int    ret;
+        while ((ret = SSL_write_ex(ctx, buf.data() + written, buf.size() - written, &written_now)) <= 0) {
+            int err = SSL_get_error(ctx, ret);
+            if (err == SSL_ERROR_WANT_WRITE) {
+                poll_wait(fd, true);
+                continue;
+            }
+
+            throw OpenSSLException("write failed");
+        }
+        written += written_now;
+    }
+    return true;
+}
+
+std::vector<uint8_t> Helpers::SSL_read_n(SSL* ctx, int fd, size_t n) {
+    size_t               nread    = 0;
+    size_t               read_now = 0;
+    std::vector<uint8_t> buf(n);
+    int                  ret;
+
+    while (nread < n) {
+        while ((ret = SSL_read_ex(ctx, buf.data() + nread, n, &read_now)) <= 0) {
+            int err = SSL_get_error(ctx, ret);
+            if (err == SSL_ERROR_WANT_READ) {
+                Helpers::poll_wait(fd, false);
+                continue;
+            }
+            throw OpenSSLException("read failed");
+        }
+        nread += read_now;
+    }
+    return buf;
+}
+
+MsgWrapper Helpers::SSL_read_msg(SSL* ctx, int fd) {
+    auto      hdrBuf = SSL_read_n(ctx, fd, sizeof(MsgHeader));
+    MsgHeader hdr;
+    memcpy(&hdr, hdrBuf.data(), sizeof(hdr));
+    auto data = SSL_read_n(ctx, fd, hdr.len);
+    return {hdr.id, std::move(data)};
+}
+
+void Helpers::SSL_send_msg(SSL* ctx, int fd, const MsgWrapper& buf) {
+    MsgHeader header{};
+    header.id  = buf.id;
+    header.len = checked_cast<uint32_t>(buf.data.size());
+    std::vector<uint8_t> hdrBuf(sizeof(MsgHeader));
+    memcpy(hdrBuf.data(), &header, sizeof(MsgHeader));
+    SSL_write(ctx, fd, hdrBuf);
+    SSL_write(ctx, fd, buf.data);
+}

networking/src/Server.cpp

@@ -0,0 +1,171 @@
+//
+// Created by Stepan Usatiuk on 09.12.2024.
+//
+
+#ifndef TCPSERVER_IPP
+#define TCPSERVER_IPP
+
+#include "Server.hpp"
+
+#include <memory>
+#include <string>
+#include <thread>
+
+#include <fcntl.h>
+#include <netdb.h>
+#include <netinet/ip.h>
+#include <netinet/tcp.h>
+#include <poll.h>
+#include <signal.h>
+#include <sys/select.h>
+#include <sys/socket.h>
+#include <sys/time.h>
+#include <unistd.h>
+
+#include <openssl/err.h>
+#include <openssl/ssl.h>
+
+#include "Exception.h"
+#include "Helpers.hpp"
+#include "Logger.h"
+
+// From https://wiki.openssl.org/index.php/Simple_TLS_Server
+static std::unique_ptr<SSL_CTX, decltype(&SSL_CTX_free)> create_context() {
+    const SSL_METHOD* method;
+    SSL_CTX*          ctx;
+
+    method = TLS_server_method();
+
+    ctx = SSL_CTX_new(method);
+    if (!ctx) {
+        throw OpenSSLException("Unable to create SSL context");
+    }
+
+    return {ctx, &SSL_CTX_free};
+}
+
+static void configure_context(SSL_CTX* ctx, const std::string& cert_path, const std::string& key_path) {
+    /* Set the key and cert */
+    if (SSL_CTX_use_certificate_file(ctx, cert_path.c_str(), SSL_FILETYPE_PEM) <= 0) {
+        throw OpenSSLException("Unable to read certificate file");
+    }
+
+    if (SSL_CTX_use_PrivateKey_file(ctx, key_path.c_str(), SSL_FILETYPE_PEM) <= 0) {
+        throw OpenSSLException("Unable to read private key file");
+    }
+}
+
+Server::Server(uint16_t port, uint32_t ip, std::string cert_path, std::string key_path) :
+    _port(port), _ip(ip), _cert_path(std::move(cert_path)), _key_path(std::move(key_path)), _ssl_ctx(create_context()) {
+    configure_context(_ssl_ctx.get(), _cert_path, _key_path);
+}
+
+void Server::process_req(int conn_fd) {
+    _req_in_progress.fetch_add(1);
+    std::thread proc([=, this] {
+        int id = _total_req.fetch_add(1);
+
+        Logger::log(Logger::RemoteFs, "Client " + std::to_string(id) + " connecting\n", Logger::INFO);
+
+        ClientCtx context{{}, {_ssl_ctx.get(), conn_fd, id}, {}};
+
+        try {
+            Helpers::init_nonblock(conn_fd);
+
+            context.transport.run();
+
+            for (;;) {
+                auto msg = context.transport.get_msg();
+
+                if (!msg)
+                    break;
+
+                std::thread msg_proc([&context, msg, this] {
+                    auto ret = this->handle_message(context, std::move(msg->data));
+                    context.transport.send_message(std::make_shared<MsgWrapper>(msg->id, std::move(ret)));
+                });
+                msg_proc.detach();
+            }
+        } catch (std::exception& e) {
+            Logger::log(Logger::RemoteFs, std::string("Error: ") + e.what(), Logger::ERROR);
+        }
+
+        close(conn_fd);
+        _req_in_progress.fetch_sub(1);
+        std::lock_guard<std::mutex> lock(_req_in_progress_mutex);
+        Logger::log(Logger::RemoteFs, "Client " + std::to_string(id) + " finished\n", Logger::INFO);
+        _req_in_progress_cond.notify_all();
+    });
+    proc.detach();
+}
+
+void Server::run() {
+    protoent* proto = getprotobyname("tcp");
+    if (proto == NULL) {
+        throw ErrnoException("Could not get TCP protocol info");
+    }
+
+    int         sock = socket(AF_INET, SOCK_STREAM, proto->p_proto);
+    sockaddr_in addr;
+
+    addr.sin_family = AF_INET;
+    addr.sin_port   = htons(_port);
+    addr.sin_addr   = {_ip};
+    memset(addr.sin_zero, 0, sizeof(addr.sin_zero));
+#ifdef APPLE
+    addr.sin_len = sizeof(struct sockaddr_in),
+#endif
+
+    Logger::log(
+            Logger::RemoteFs,
+            [&](std::ostream& os) {
+                os << "Listening on ";
+                for (int i = 0; i < 4; i++) {
+                    os << static_cast<int>(reinterpret_cast<uint8_t*>(&_ip)[i]);
+                    if (i != 3)
+                        os << ".";
+                }
+                os << ":" << _port;
+            },
+            Logger::INFO);
+
+    if (bind(sock, reinterpret_cast<sockaddr*>(&addr), sizeof(addr)) < 0) {
+        throw ErrnoException("Could not bind");
+    }
+
+    if (listen(sock, 1) < 0) {
+        throw ErrnoException("Could not listen");
+    }
+
+    Helpers::init_nonblock(sock);
+
+    try {
+        // while (!Signals::is_stopped()) {
+        while (true) {
+            try {
+                Helpers::poll_wait(sock, false, -1);
+                int conn = accept(sock, nullptr, nullptr);
+                if (conn == -1) {
+                    throw ErrnoException("accept");
+                    continue;
+                }
+                process_req(conn);
+            } catch (std::exception& e) {
+                std::cerr << "Error: " << e.what() << std::endl;
+            }
+        }
+    } catch (std::exception& e) {
+        std::cerr << "Error: " << e.what() << std::endl;
+    }
+
+    Logger::log(Logger::RemoteFs, "Exiting", Logger::INFO);
+    {
+        std::unique_lock lock(_req_in_progress_mutex);
+        _req_in_progress_cond.wait(lock, [&] { return _req_in_progress == 0; });
+    }
+
+    close(sock);
+}
+
+
+#endif // TCPSERVER_IPP

remotefs/CMakeLists.txt

@@ -0,0 +1,31 @@
+cmake_minimum_required(VERSION 3.15)
+
+add_library(remotefs_lib
+        include/FsClient.hpp
+        src/FsClient.cpp
+        include/FsServer.hpp
+        src/FsServer.cpp
+        include/Messages.hpp
+        src/Acl.cpp
+        include/Acl.hpp
+)
+
+target_include_directories(remotefs_lib PUBLIC include)
+
+find_package(OpenSSL REQUIRED)
+find_package(PkgConfig REQUIRED)
+pkg_check_modules(FUSE REQUIRED IMPORTED_TARGET fuse)
+
+target_link_libraries(remotefs_lib PUBLIC
+        utils
+        networking
+        OpenSSL::SSL
+        OpenSSL::Crypto
+        PkgConfig::FUSE
+)
+
+add_executable(remotefs src/main.cpp)
+
+target_link_libraries(remotefs PRIVATE remotefs_lib)
+
+add_subdirectory(tests)

remotefs/include/Acl.hpp

@@ -0,0 +1,24 @@
+//
+// Created by Stepan Usatiuk on 04.01.2025.
+//
+
+#ifndef ACL_HPP
+#define ACL_HPP
+
+#include <map>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+
+class ACL {
+public:
+    void load(std::string acl, std::string users);
+    bool authorize(std::string username, std::string password);
+    bool authorize_path(std::string username, std::string path);
+
+private:
+    std::unordered_map<std::string, std::string>    _users;
+    std::map<std::string, std::unordered_set<std::string>> _filter;
+};
+
+#endif // ACL_HPP

remotefs/include/FsClient.hpp

@@ -0,0 +1,16 @@
+//
+// Created by Stepan Usatiuk on 12.12.2024.
+//
+
+#ifndef FSCLIENT_HPP
+#define FSCLIENT_HPP
+
+#define FUSE_USE_VERSION 26
+
+class FsClient {
+public:
+    void run();
+};
+
+
+#endif // FSCLIENT_HPP

remotefs/include/FsServer.hpp

@@ -0,0 +1,15 @@
+//
+// Created by Stepan Usatiuk on 12.12.2024.
+//
+
+#ifndef FSSERVER_HPP
+#define FSSERVER_HPP
+
+
+class FsServer {
+public:
+    void run();
+};
+
+
+#endif // FSSERVER_HPP

remotefs/include/Messages.hpp

@@ -0,0 +1,225 @@
+//
+// Created by Stepan Usatiuk on 12.12.2024.
+//
+
+#ifndef MESSAGES_HPP
+#define MESSAGES_HPP
+
+#include <cstdint>
+#include <variant>
+
+#include "SerializableStruct.hpp"
+#include "Serialize.hpp"
+
+enum class FileType { NONE, DIRECTORY, REG_FILE, SYMLINK, END };
+
+#define LOGIN_REQ(FIELD)                                                                                               \
+    FIELD(std::string, username)                                                                                       \
+    FIELD(std::string, password)
+DECLARE_SERIALIZABLE(LoginReq, LOGIN_REQ)
+DECLARE_SERIALIZABLE_END
+#undef LOGIN_REQ
+
+#define LOGIN_REPLY(FIELD)
+DECLARE_SERIALIZABLE(LoginReply, LOGIN_REPLY)
+DECLARE_SERIALIZABLE_END
+#undef LOGIN_REPLY
+
+#define GETATTR_REQ(FIELD) FIELD(std::string, path)
+DECLARE_SERIALIZABLE(GetattrReq, GETATTR_REQ)
+DECLARE_SERIALIZABLE_END
+#undef GETATTR_REQ
+
+#define GETATTR_REPLY(FIELD)                                                                                           \
+    FIELD(FileType, type)                                                                                              \
+    FIELD(uint64_t, mode)                                                                                              \
+    FIELD(uint64_t, links)                                                                                             \
+    FIELD(uint64_t, size)
+DECLARE_SERIALIZABLE(GetattrReply, GETATTR_REPLY)
+DECLARE_SERIALIZABLE_END
+#undef GETATTR_REPLY
+
+#define READDIR_REQ(FIELD) FIELD(std::string, path)
+DECLARE_SERIALIZABLE(ReaddirReq, READDIR_REQ)
+DECLARE_SERIALIZABLE_END
+#undef READDIR_REQ
+
+#define READDIR_REPLY(FIELD) FIELD(std::vector<std::string>, path)
+DECLARE_SERIALIZABLE(ReaddirReply, READDIR_REPLY)
+DECLARE_SERIALIZABLE_END
+#undef READDIR_REPLY
+
+#define OPEN_REQ(FIELD) FIELD(std::string, path)
+DECLARE_SERIALIZABLE(OpenReq, OPEN_REQ)
+DECLARE_SERIALIZABLE_END
+#undef OPEN_REQ
+
+#define OPEN_REPLY(FIELD) FIELD(int, ok)
+DECLARE_SERIALIZABLE(OpenReply, OPEN_REPLY)
+DECLARE_SERIALIZABLE_END
+#undef OPEN_REPLY
+
+#define READ_REQ(FIELD)                                                                                                \
+    FIELD(std::string, path)                                                                                           \
+    FIELD(int64_t, off)                                                                                                \
+    FIELD(uint64_t, len)
+DECLARE_SERIALIZABLE(ReadReq, READ_REQ)
+DECLARE_SERIALIZABLE_END
+#undef READ_REQ
+
+#define READ_REPLY(FIELD) FIELD(std::vector<uint8_t>, data)
+DECLARE_SERIALIZABLE(ReadReply, READ_REPLY)
+DECLARE_SERIALIZABLE_END
+#undef READ_REPLY
+
+#define WRITE_REQ(FIELD)                                                                                               \
+    FIELD(std::string, path)                                                                                           \
+    FIELD(int64_t, off)                                                                                                \
+    FIELD(uint64_t, len)                                                                                               \
+    FIELD(std::vector<uint8_t>, data)
+DECLARE_SERIALIZABLE(WriteReq, WRITE_REQ)
+DECLARE_SERIALIZABLE_END
+#undef WRITE_REQ
+
+#define WRITE_REPLY(FIELD) FIELD(int, len)
+DECLARE_SERIALIZABLE(WriteReply, WRITE_REPLY)
+DECLARE_SERIALIZABLE_END
+#undef WRITE_REPLY
+
+#define CREATE_REQ(FIELD)                                                                                              \
+    FIELD(std::string, path)                                                                                           \
+    FIELD(int, mode)
+DECLARE_SERIALIZABLE(CreateReq, CREATE_REQ)
+DECLARE_SERIALIZABLE_END
+#undef CREATE_REQ
+
+#define CREATE_REPLY(FIELD) FIELD(int, ok)
+DECLARE_SERIALIZABLE(CreateReply, CREATE_REPLY)
+DECLARE_SERIALIZABLE_END
+#undef CREATE_REPLY
+
+#define CHMOD_REQ(FIELD)                                                                                               \
+    FIELD(std::string, path)                                                                                           \
+    FIELD(int, mode)
+DECLARE_SERIALIZABLE(ChmodReq, CHMOD_REQ)
+DECLARE_SERIALIZABLE_END
+#undef CHMOD_REQ
+
+#define CHMOD_REPLY(FIELD) FIELD(int, ok)
+DECLARE_SERIALIZABLE(ChmodReply, CHMOD_REPLY)
+DECLARE_SERIALIZABLE_END
+#undef CHMOD_REPLY
+
+#define MKDIR_REQ(FIELD)                                                                                               \
+    FIELD(std::string, path)                                                                                           \
+    FIELD(int, mode)
+DECLARE_SERIALIZABLE(MkdirReq, MKDIR_REQ)
+DECLARE_SERIALIZABLE_END
+#undef MKDIR_REQ
+
+#define MKDIR_REPLY(FIELD) FIELD(int, ok)
+DECLARE_SERIALIZABLE(MkdirReply, MKDIR_REPLY)
+DECLARE_SERIALIZABLE_END
+#undef MKDIR_REPLY
+
+#define RMDIR_REQ(FIELD) FIELD(std::string, path)
+DECLARE_SERIALIZABLE(RmdirReq, RMDIR_REQ)
+DECLARE_SERIALIZABLE_END
+#undef RMDIR_REQ
+
+#define RMDIR_REPLY(FIELD) FIELD(int, ok)
+DECLARE_SERIALIZABLE(RmdirReply, RMDIR_REPLY)
+DECLARE_SERIALIZABLE_END
+#undef RMDIR_REPLY
+
+#define UNLINK_REQ(FIELD) FIELD(std::string, path)
+DECLARE_SERIALIZABLE(UnlinkReq, UNLINK_REQ)
+DECLARE_SERIALIZABLE_END
+#undef UNLINK_REQ
+
+#define UNLINK_REPLY(FIELD) FIELD(int, ok)
+DECLARE_SERIALIZABLE(UnlinkReply, UNLINK_REPLY)
+DECLARE_SERIALIZABLE_END
+#undef UNLINK_REPLY
+
+#define TRUNCATE_REQ(FIELD)                                                                                            \
+    FIELD(std::string, path)                                                                                           \
+    FIELD(long, size)
+DECLARE_SERIALIZABLE(TruncateReq, TRUNCATE_REQ)
+DECLARE_SERIALIZABLE_END
+#undef TRUNCATE_REQ
+
+#define TRUNCATE_REPLY(FIELD) FIELD(int, res)
+DECLARE_SERIALIZABLE(TruncateReply, TRUNCATE_REPLY)
+DECLARE_SERIALIZABLE_END
+#undef TRUNCATE_REPLY
+
+#define RENAME_REQ(FIELD)                                                                                              \
+    FIELD(std::string, path)                                                                                           \
+    FIELD(std::string, newPath)
+DECLARE_SERIALIZABLE(RenameReq, RENAME_REQ)
+DECLARE_SERIALIZABLE_END
+#undef RENAME_REQ
+
+#define RENAME_REPLY(FIELD) FIELD(int, ok)
+DECLARE_SERIALIZABLE(RenameReply, RENAME_REPLY)
+DECLARE_SERIALIZABLE_END
+#undef RENAME_REPLY
+
+#define UTIMENS_REQ(FIELD)                                                                                             \
+    FIELD(std::string, path)                                                                                           \
+    FIELD(int64_t, asecs)                                                                                              \
+    FIELD(int64_t, ans)                                                                                                \
+    FIELD(int64_t, msecs)                                                                                              \
+    FIELD(int64_t, mns)
+DECLARE_SERIALIZABLE(UTimensReq, UTIMENS_REQ)
+DECLARE_SERIALIZABLE_END
+#undef UTIMENS_REQ
+
+#define UTIMENS_REPLY(FIELD) FIELD(int, ok)
+DECLARE_SERIALIZABLE(UTimensReply, UTIMENS_REPLY)
+DECLARE_SERIALIZABLE_END
+#undef UTIMENS_REPLY
+
+#define STATFS_REQ(FIELD) FIELD(std::string, path)
+DECLARE_SERIALIZABLE(StatfsReq, STATFS_REQ)
+DECLARE_SERIALIZABLE_END
+#undef STATFS_REQ
+
+#define STATFS_REPLY(FIELD)                                                                                            \
+    FIELD(int, ok)                                                                                                     \
+    FIELD(uint64_t, frsize)                                                                                            \
+    FIELD(uint64_t, blksize)                                                                                           \
+    FIELD(uint64_t, blocks)                                                                                            \
+    FIELD(uint64_t, bfree)                                                                                             \
+    FIELD(uint64_t, bavail)                                                                                            \
+    FIELD(uint64_t, files)                                                                                             \
+    FIELD(uint64_t, ffree)                                                                                             \
+    FIELD(uint64_t, favail)                                                                                            \
+    FIELD(uint64_t, namemax)
+DECLARE_SERIALIZABLE(StatfsReply, STATFS_REPLY)
+DECLARE_SERIALIZABLE_END
+#undef STATFS_REPLY
+
+#define KEEPALIVE_REQ(FIELD)
+DECLARE_SERIALIZABLE(KeepAliveReq, KEEPALIVE_REQ)
+DECLARE_SERIALIZABLE_END
+#undef RENAME_REQ
+
+#define KEEPALIVE_REPLY(FIELD)
+DECLARE_SERIALIZABLE(KeepAliveReply, KEEPALIVE_REPLY)
+DECLARE_SERIALIZABLE_END
+#undef KEEPALIVE_REPLY
+
+#define ERROR_REPLY(FIELD) FIELD(std::string, error)
+DECLARE_SERIALIZABLE(ErrorReply, ERROR_REPLY)
+DECLARE_SERIALIZABLE_END
+#undef ERROR_REPLY
+
+using AnyMsgT = std::variant<ErrorReply, KeepAliveReq, KeepAliveReply, LoginReq, LoginReply, GetattrReq, GetattrReply,
+                             ReaddirReq, ReaddirReply, OpenReq, OpenReply, ReadReq, ReadReply, WriteReq, WriteReply,
+                             CreateReq, CreateReply, MkdirReq, MkdirReply, RmdirReq, RmdirReply, UnlinkReq, UnlinkReply,
+                             TruncateReq, TruncateReply, RenameReq, RenameReply, UTimensReq, UTimensReply, StatfsReply,
+                             StatfsReq, ChmodReq, ChmodReply>;
+
+#endif // MESSAGES_HPP

remotefs/src/Acl.cpp

@@ -0,0 +1,114 @@
+//
+// Created by Stepan Usatiuk on 04.01.2025.
+//
+
+#include "Acl.hpp"
+
+#include <sstream>
+#include <unordered_set>
+
+#include "Exception.h"
+#include "Logger.h"
+#include "SHA.h"
+#include "stuff.hpp"
+
+static const char int2hex[] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};
+
+template<typename T>
+std::string chars_to_hex(const T& v) {
+    std::string out;
+    out.resize(v.size() * 2);
+    for (size_t i = 0; i < v.size(); i++) {
+        out[2 * i]     = int2hex[(v[i] & 0xF0) >> 4];
+        out[2 * i + 1] = int2hex[(v[i] & 0x0F)];
+    }
+    return out;
+}
+
+bool ACL::authorize(std::string username, std::string password) {
+    auto hash = chars_to_hex(SHA::calculate(password));
+
+    auto found_user = _users.find(username);
+
+    if (found_user == _users.end()) {
+        return false;
+    }
+
+    if (found_user->second != hash) {
+        return false;
+    }
+
+    return true;
+}
+
+bool ACL::authorize_path(std::string username, std::string path) {
+    // FIXME: could be faster with e.g. trie
+
+    std::pair<std::string, std::unordered_set<std::string>> best_match{"", {}};
+
+    for (const auto& filter: _filter) {
+        if (path.starts_with(filter.first)) {
+            if (filter.first.length() >= best_match.first.length()) {
+                best_match = filter;
+            }
+        }
+    }
+
+    if (best_match.second.empty()) {
+        return true;
+    }
+
+    if (best_match.second.contains(username)) {
+        return true;
+    }
+
+    return false;
+}
+
+void ACL::load(std::string acl, std::string users) {
+    auto users_lines = split(users, '\n');
+
+    for (const auto& line: users_lines) {
+        auto tokens = split(line, ' ');
+        if (tokens.size() != 2) {
+            throw Exception("Could not parse user definition: " + line);
+        }
+
+        if (tokens.at(1).length() != 64) {
+            throw Exception("Could not parse user definition (wrong hash length): " + line);
+        }
+
+        _users.emplace(tokens.at(0), tokens.at(1));
+    }
+
+    auto acl_lines = split(acl, '\n');
+
+    for (const auto& line: acl_lines) {
+        auto tokens = split(line, ' ');
+        if (tokens.empty())
+            continue;
+
+        std::unordered_set<std::string> usernames(tokens.begin() + 1, tokens.end());
+
+        _filter.emplace(tokens.at(0), std::move(usernames));
+    }
+
+    Logger::log(
+            Logger::RemoteFs,
+            [&](std::ostream& os) {
+                os << "Loaded users and ACL:\n";
+                os << "Users:\n";
+                for (const auto& user: _users) {
+                    os << user.first << " " << user.second << '\n';
+                }
+                os << "ACL:\n";
+                for (const auto& filter: _filter) {
+                    os << filter.first << " ";
+                    for (const auto& user: filter.second) {
+                        os << " " << user;
+                    }
+                    os << '\n';
+                }
+            },
+            Logger::INFO);
+}

remotefs/src/FsClient.cpp

@@ -0,0 +1,311 @@
+//
+// Created by Stepan Usatiuk on 12.12.2024.
+//
+
+#include "FsClient.hpp"
+
+#include "Client.hpp"
+#include "Options.h"
+#include "stuff.hpp"
+
+#include <fuse.h>
+#include <iostream>
+
+#include <sys/statvfs.h>
+#include <unistd.h>
+
+#include "Logger.h"
+#include "Messages.hpp"
+#include "Serialize.hpp"
+
+static Client*         client;
+static AsyncSslClientTransport* asyncTransport;
+
+template<typename R, typename M>
+R call(M msg) {
+    auto ret = asyncTransport->send_msg_and_wait(Serialize::serialize(AnyMsgT{msg}));
+
+    auto deserialized = Serialize::deserialize<AnyMsgT>(ret);
+    if (!std::holds_alternative<R>(deserialized)) {
+        if (std::holds_alternative<ErrorReply>(deserialized)) {
+            throw Exception("Error when reading: " + std::get<ErrorReply>(deserialized).error);
+        } else {
+            throw Exception("Unexpected reply from server");
+        }
+    }
+    return std::get<R>(deserialized);
+}
+
+static int rfsGetattr(const char* path, struct stat* stbuf) {
+    try {
+        memset(stbuf, 0, sizeof(struct stat));
+        if (strcmp(path, "/") == 0) {
+            stbuf->st_mode  = S_IFDIR | 0755;
+            stbuf->st_nlink = 2;
+            return 0;
+        }
+
+        auto ret = call<GetattrReply>(GetattrReq{path});
+        switch (ret.type) {
+            case FileType::NONE:
+                return -ENOENT;
+                break;
+            case FileType::DIRECTORY:
+                stbuf->st_mode = S_IFDIR;
+                break;
+            case FileType::REG_FILE:
+                stbuf->st_mode = S_IFREG;
+                break;
+            case FileType::SYMLINK:
+                stbuf->st_mode = S_IFLNK;
+                break;
+            default:
+                return -ENOENT;
+        }
+
+        stbuf->st_mode |= checked_cast<mode_t>(ret.mode);
+        stbuf->st_size  = checked_cast<off_t>(ret.size);
+        stbuf->st_nlink = checked_cast<nlink_t>(ret.links);
+
+        return 0;
+    } catch (std::exception& e) {
+        Logger::log(Logger::RemoteFs, e.what(), Logger::ERROR);
+        return -EIO;
+    }
+}
+
+static int rfsReaddir(const char* path, void* buf, fuse_fill_dir_t filler, off_t offset, struct fuse_file_info* fi) {
+    try {
+        filler(buf, ".", NULL, 0);
+        filler(buf, "..", NULL, 0);
+        auto ret = call<ReaddirReply>(ReaddirReq{path});
+
+        for (auto const& e: ret.path) {
+            filler(buf, e.c_str(), NULL, 0);
+        }
+
+        return 0;
+    } catch (std::exception& e) {
+        Logger::log(Logger::RemoteFs, e.what(), Logger::ERROR);
+        return -EIO;
+    }
+}
+
+static int rfsOpen(const char* path, struct fuse_file_info* fi) {
+    try {
+        auto ret = call<OpenReply>(OpenReq{path});
+
+        if (ret.ok != 1) {
+            return -ENOENT;
+        }
+
+        return 0;
+    } catch (std::exception& e) {
+        Logger::log(Logger::RemoteFs, e.what(), Logger::ERROR);
+        return -EIO;
+    }
+}
+
+static int rfsRead(const char* path, char* buf, size_t size, off_t offset, struct fuse_file_info* fi) {
+    try {
+        auto   ret        = call<ReadReply>(ReadReq{path, offset, size});
+        size_t reallyRead = std::min(ret.data.size(), size);
+        std::memcpy(buf, ret.data.data(), reallyRead);
+        return checked_cast<int>(reallyRead);
+    } catch (std::exception& e) {
+        Logger::log(Logger::RemoteFs, e.what(), Logger::ERROR);
+        return -EIO;
+    }
+}
+
+static int rfsWrite(const char* path, const char* buf, size_t size, off_t offset, struct fuse_file_info* fi) {
+    try {
+        auto ret = call<WriteReply>(WriteReq{path, offset, size, std::vector<uint8_t>(buf, buf + size)});
+        return ret.len;
+    } catch (std::exception& e) {
+        Logger::log(Logger::RemoteFs, e.what(), Logger::ERROR);
+        return -EIO;
+    }
+}
+
+static int rfsCreate(const char* path, mode_t mode, struct fuse_file_info* fi) {
+    try {
+        auto ret = call<CreateReply>(CreateReq{std::string(path), static_cast<int>(mode)});
+        return ret.ok;
+    } catch (std::exception& e) {
+        Logger::log(Logger::RemoteFs, e.what(), Logger::ERROR);
+        return -EIO;
+    }
+}
+
+static int rfsMkdir(const char* path, mode_t mode) {
+    try {
+        auto ret = call<MkdirReply>(MkdirReq{std::string(path), static_cast<int>(mode)});
+        return ret.ok;
+    } catch (std::exception& e) {
+        Logger::log(Logger::RemoteFs, e.what(), Logger::ERROR);
+        return -EIO;
+    }
+}
+
+static int rfsRmdir(const char* path) {
+    try {
+        auto ret = call<RmdirReply>(RmdirReq{std::string(path)});
+        return ret.ok;
+    } catch (std::exception& e) {
+        Logger::log(Logger::RemoteFs, e.what(), Logger::ERROR);
+        return -EIO;
+    }
+}
+
+static int rfsUnlink(const char* path) {
+    try {
+        auto ret = call<UnlinkReply>(UnlinkReq{std::string(path)});
+        return ret.ok;
+    } catch (std::exception& e) {
+        Logger::log(Logger::RemoteFs, e.what(), Logger::ERROR);
+        return -EIO;
+    }
+}
+
+static int rfsTruncate(const char* path, off_t size) {
+    try {
+        auto ret = call<TruncateReply>(TruncateReq{std::string(path), size});
+        return ret.res;
+    } catch (std::exception& e) {
+        Logger::log(Logger::RemoteFs, e.what(), Logger::ERROR);
+        return -EIO;
+    }
+}
+
+static int rfsRename(const char* path, const char* newPath) {
+    try {
+        auto ret = call<RenameReply>(RenameReq{std::string(path), newPath});
+        return ret.ok;
+    } catch (std::exception& e) {
+        Logger::log(Logger::RemoteFs, e.what(), Logger::ERROR);
+        return -EIO;
+    }
+}
+
+static int rfsUtimens(const char* path, const struct timespec time[2]) {
+    try {
+        auto ret =
+                call<UTimensReply>(UTimensReq{path, time[0].tv_sec, time[0].tv_nsec, time[1].tv_sec, time[1].tv_nsec});
+        return ret.ok;
+    } catch (std::exception& e) {
+        Logger::log(Logger::RemoteFs, e.what(), Logger::ERROR);
+        return -EIO;
+    }
+}
+
+static int rfsUtime(const char* path, struct utimbuf* time) {
+    timespec timens[2] = {
+            {
+                    time->actime,
+                    0,
+            },
+            {
+                    time->modtime,
+                    0,
+            },
+    };
+
+    return rfsUtimens(path, timens);
+}
+
+static int rfsStatfs(const char* path, struct statvfs* stats) {
+    try {
+        auto ret         = call<StatfsReply>(StatfsReq{path});
+        stats->f_frsize  = checked_cast<decltype(stats->f_frsize)>(ret.frsize);
+        stats->f_bsize   = checked_cast<decltype(stats->f_bsize)>(ret.blksize);
+        stats->f_blocks  = checked_cast<decltype(stats->f_blocks)>(ret.blocks);
+        stats->f_bfree   = checked_cast<decltype(stats->f_bfree)>(ret.bfree);
+        stats->f_bavail  = checked_cast<decltype(stats->f_bavail)>(ret.bavail);
+        stats->f_files   = checked_cast<decltype(stats->f_files)>(ret.files);
+        stats->f_ffree   = checked_cast<decltype(stats->f_ffree)>(ret.ffree);
+        stats->f_favail  = checked_cast<decltype(stats->f_favail)>(ret.favail);
+        stats->f_namemax = checked_cast<decltype(stats->f_namemax)>(ret.namemax);
+        return ret.ok;
+    } catch (std::exception& e) {
+        Logger::log(Logger::RemoteFs, e.what(), Logger::ERROR);
+        return -EIO;
+    }
+}
+
+static int rfsChmod(const char* path, mode_t mode) {
+    try {
+        auto ret = call<ChmodReply>(ChmodReq{std::string(path), static_cast<int>(mode)});
+        return ret.ok;
+    } catch (std::exception& e) {
+        Logger::log(Logger::RemoteFs, e.what(), Logger::ERROR);
+        return -EIO;
+    }
+}
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wmissing-field-initializers"
+
+static struct fuse_operations ops = {
+        .getattr  = rfsGetattr,
+        .mkdir    = rfsMkdir,
+        .unlink   = rfsUnlink,
+        .rmdir    = rfsRmdir,
+        .rename   = rfsRename,
+        .chmod    = rfsChmod,
+        .truncate = rfsTruncate,
+        .utime    = rfsUtime,
+        .open     = rfsOpen,
+        .read     = rfsRead,
+        .write    = rfsWrite,
+        .statfs   = rfsStatfs,
+        .readdir  = rfsReaddir,
+        .create   = rfsCreate,
+        .utimens  = rfsUtimens,
+};
+
+#pragma GCC diagnostic pop
+
+std::thread keep_alive_thread;
+
+static void keep_alive() {
+    while (!asyncTransport->is_stopped()) {
+        try {
+            std::this_thread::sleep_for(std::chrono::seconds(1));
+            call<KeepAliveReply>(KeepAliveReq{});
+        } catch (std::exception& e) {
+            Logger::log(Logger::RemoteFs, std::string("Keepalive error: ") + e.what(), Logger::ERROR);
+        }
+    }
+}
+
+void FsClient::run() {
+    client = new Client(checked_cast<uint16_t>(Options::get<size_t>("port")), Options::get<std::string>("ip"),
+                           Options::get<std::string>("ca_path"), Options::get<std::string>("pk_path"));
+
+    client->run();
+    asyncTransport  = &client->transport();
+    keep_alive_thread = std::thread(keep_alive);
+
+    Logger::log(
+            Logger::RemoteFs,
+            [&](std::ostream& os) {
+                os << "Trying to log in with " << Options::get<std::string>("username") << " "
+                   << Options::get<std::string>("password");
+            },
+            Logger::INFO);
+
+    call<LoginReply>(LoginReq{Options::get<std::string>("username"), Options::get<std::string>("password")});
+
+    char        arg1[] = "";
+    char        arg2[] = "-o";
+    std::string arg3   = "uid=" + std::to_string(getuid());
+    char        arg4[] = "-o";
+    std::string arg5   = "gid=" + std::to_string(getgid());
+    auto        arg6   = Options::get<std::string>("path");
+    char        arg8[] = "-f";
+
+    int   argc   = 7;
+    char* argv[] = {arg1, arg2, arg3.data(), arg4, arg5.data(), arg6.data(), arg8};
+    std::cout << static_cast<int>(fuse_main(argc, argv, &ops, nullptr));
+}

remotefs/src/FsServer.cpp

@@ -0,0 +1,325 @@
+//
+// Created by Stepan Usatiuk on 12.12.2024.
+//
+
+#include "FsServer.hpp"
+
+#include <fcntl.h>
+#include <filesystem>
+#include <fstream>
+#include <sstream>
+#include <sys/stat.h>
+#include <sys/statvfs.h>
+
+#include "Acl.hpp"
+#include "Exception.h"
+#include "Logger.h"
+#include "Messages.hpp"
+#include "Options.h"
+#include "Serialize.hpp"
+#include "Server.hpp"
+#include "stuff.hpp"
+
+static uint32_t parse_ip(const std::string& ip_str) {
+    std::istringstream iss(ip_str);
+    std::string        part;
+    int                ctr = 0;
+
+    uint8_t ip_res[4];
+
+    while (std::getline(iss, part, '.')) {
+        if (ctr == 4) {
+            throw Exception("Unknown address format");
+        }
+        if (!part.empty())
+            ip_res[ctr++] = (unsigned char) strtol(part.c_str(), NULL, 10);
+    }
+
+    uint32_t res;
+    memcpy(&res, ip_res, sizeof(res));
+    return res;
+}
+
+static ACL acl;
+
+class RemoteFsServer : public Server {
+
+private:
+    std::vector<uint8_t> handle_auth(ClientCtx& context, AnyMsgT msg) {
+        return Serialize::serialize(std::visit(
+                [&](auto&& arg) -> AnyMsgT {
+                    using T = std::decay_t<decltype(arg)>;
+                    if constexpr (std::is_same_v<T, LoginReq>) {
+                        Logger::log(Logger::RemoteFs, "Authenticating " + arg.username, Logger::INFO);
+                        if (acl.authorize(arg.username, arg.password)) {
+                            context.client_name = arg.username;
+                            return LoginReply{};
+                        } else {
+                            throw Exception("Invalid username or password");
+                        }
+                    } else
+                        throw Exception(std::string("Unexpected message type: ") + typeid(T).name());
+                },
+                msg));
+    }
+
+public:
+    RemoteFsServer(uint16_t port, uint32_t ip, const std::string& cert_path, const std::string& key_path) :
+        Server(port, ip, cert_path, key_path) {}
+
+    std::vector<uint8_t> handle_message(ClientCtx& context, std::vector<uint8_t> data) override {
+        try {
+            auto msg = Serialize::deserialize<AnyMsgT>(data);
+            if (!context.client_name) {
+                std::lock_guard lock(context.ctx_mutex);
+                if (!context.client_name) {
+                    return handle_auth(context, msg);
+                }
+            }
+            return Serialize::serialize(std::visit(
+                    [&](auto&& arg) -> AnyMsgT {
+                        auto root = std::filesystem::path(Options::get<std::string>("path"));
+                        using T   = std::decay_t<decltype(arg)>;
+                        if constexpr (std::is_same_v<T, GetattrReq>) {
+                            auto path = root.concat(arg.path);
+
+                            if (!std::filesystem::exists(path)) {
+                                return GetattrReply{FileType::NONE, 0, 0, 0};
+                            } else if (std::filesystem::is_directory(path)) {
+                                struct stat buf;
+                                stat(path.c_str(), &buf);
+                                return GetattrReply{FileType::DIRECTORY, buf.st_mode, buf.st_nlink,
+                                                    checked_cast<uint64_t>(buf.st_size)};
+                            } else if (std::filesystem::is_regular_file(path)) {
+                                struct stat buf;
+                                stat(path.c_str(), &buf);
+                                return GetattrReply{FileType::REG_FILE, buf.st_mode, buf.st_nlink,
+                                                    checked_cast<uint64_t>(buf.st_size)};
+                            } else {
+                                return GetattrReply{FileType::NONE, 0, 0, 0};
+                            }
+                        } else if constexpr (std::is_same_v<T, ReaddirReq>) {
+                            auto path = root.concat(arg.path);
+
+                            std::vector<std::string> results;
+                            for (const auto& entry: std::filesystem::directory_iterator(path)) {
+                                results.push_back(entry.path().lexically_relative(root).string());
+                            }
+                            return ReaddirReply{results};
+                        } else if constexpr (std::is_same_v<T, OpenReq>) {
+                            auto path = root.concat(arg.path);
+
+                            if (!acl.authorize_path(*context.client_name, arg.path)) {
+                                return ErrorReply("Unauthorized path");
+                            }
+
+                            if (std::filesystem::exists(path)) {
+                                return OpenReply{1};
+                            }
+                            return OpenReply{0};
+                        } else if constexpr (std::is_same_v<T, ReadReq>) {
+                            auto path = root.concat(arg.path);
+
+                            if (!acl.authorize_path(*context.client_name, arg.path)) {
+                                return ErrorReply("Unauthorized path");
+                            }
+
+                            if (std::filesystem::is_regular_file(path)) {
+                                std::ifstream        ifs(path, std::ios::binary);
+                                std::vector<uint8_t> buf(arg.len);
+
+                                ifs.seekg(arg.off, std::ios::beg);
+                                ifs.read(reinterpret_cast<char*>(buf.data()), checked_cast<ssize_t>(arg.len));
+                                buf.resize(checked_cast<size_t>(ifs.tellg() - arg.off));
+
+                                return ReadReply{buf};
+                            } else {
+                                return ReadReply{{}};
+                            }
+                        } else if constexpr (std::is_same_v<T, WriteReq>) {
+                            auto path = root.concat(arg.path);
+
+                            if (!acl.authorize_path(*context.client_name, arg.path)) {
+                                return ErrorReply("Unauthorized path");
+                            }
+
+                            if (std::filesystem::is_regular_file(path)) {
+                                std::fstream ofs(path, std::ios::binary | std::ios::out | std::ios::in);
+
+                                size_t real_write = std::min(checked_cast<size_t>(arg.len), arg.data.size());
+
+                                ofs.seekg(arg.off, std::ios::beg);
+                                ofs.write(reinterpret_cast<char*>(arg.data.data()), checked_cast<ssize_t>(real_write));
+
+                                return WriteReply{checked_cast<int>(ofs.tellg() - arg.off)};
+                            } else {
+                                return WriteReply{-1};
+                            }
+                        } else if constexpr (std::is_same_v<T, CreateReq>) {
+                            auto path = root.concat(arg.path);
+
+                            if (!acl.authorize_path(*context.client_name, arg.path)) {
+                                return ErrorReply("Unauthorized path");
+                            }
+
+                            if (std::filesystem::exists(path)) {
+                                return CreateReply{-1};
+                            } else {
+                                {
+                                    std::ofstream output(path);
+                                }
+                                chmod(path.c_str(), checked_cast<mode_t>(arg.mode));
+                                return CreateReply{0};
+                            }
+                        } else if constexpr (std::is_same_v<T, ChmodReq>) {
+                            auto path = root.concat(arg.path);
+
+                            if (!acl.authorize_path(*context.client_name, arg.path)) {
+                                return ErrorReply("Unauthorized path");
+                            }
+
+                            if (std::filesystem::exists(path)) {
+                                return ChmodReply{-1};
+                            } else {
+                                int ret = chmod(path.c_str(), checked_cast<mode_t>(arg.mode));
+                                return ChmodReply{ret};
+                            }
+                        } else if constexpr (std::is_same_v<T, MkdirReq>) {
+                            auto path = root.concat(arg.path);
+
+                            if (!acl.authorize_path(*context.client_name, arg.path)) {
+                                return ErrorReply("Unauthorized path");
+                            }
+
+                            if (std::filesystem::exists(path)) {
+                                return MkdirReply{-1};
+                            } else {
+                                std::filesystem::create_directory(path);
+                                return MkdirReply{0};
+                            }
+                        } else if constexpr (std::is_same_v<T, RmdirReq>) {
+                            auto path = root.concat(arg.path);
+
+                            if (!acl.authorize_path(*context.client_name, arg.path)) {
+                                return ErrorReply("Unauthorized path");
+                            }
+
+                            if (!std::filesystem::is_directory(path)) {
+                                return RmdirReply{-1};
+                            } else {
+                                std::filesystem::remove(path);
+                                return RmdirReply{0};
+                            }
+                        } else if constexpr (std::is_same_v<T, UnlinkReq>) {
+                            auto path = root.concat(arg.path);
+
+                            if (!acl.authorize_path(*context.client_name, arg.path)) {
+                                return ErrorReply("Unauthorized path");
+                            }
+
+                            if (!std::filesystem::is_regular_file(path)) {
+                                return UnlinkReply{-1};
+                            } else {
+                                std::filesystem::remove(path);
+                                return UnlinkReply{0};
+                            }
+                        } else if constexpr (std::is_same_v<T, TruncateReq>) {
+                            auto path = root.concat(arg.path);
+
+                            if (!acl.authorize_path(*context.client_name, arg.path)) {
+                                return ErrorReply("Unauthorized path");
+                            }
+
+                            if (!std::filesystem::is_regular_file(path)) {
+                                return TruncateReply{-1};
+                            } else {
+                                std::filesystem::resize_file(path, static_cast<uintmax_t>(arg.size));
+                                return TruncateReply{0};
+                            }
+                        } else if constexpr (std::is_same_v<T, RenameReq>) {
+                            auto path    = root.concat(arg.path);
+                            auto newPath = root.concat(arg.newPath);
+
+                            if (!acl.authorize_path(*context.client_name, arg.path) ||
+                                !acl.authorize_path(*context.client_name, arg.newPath)) {
+                                return ErrorReply("Unauthorized path");
+                            }
+
+                            if (!std::filesystem::is_regular_file(path)) {
+                                return RenameReply{-1};
+                            } else {
+                                std::filesystem::rename(path, newPath);
+                                return RenameReply{0};
+                            }
+                        } else if constexpr (std::is_same_v<T, UTimensReq>) {
+                            auto path = root.concat(arg.path);
+
+                            if (!acl.authorize_path(*context.client_name, arg.path)) {
+                                return ErrorReply("Unauthorized path");
+                            }
+
+                            timespec time[2] = {
+                                    {
+                                            arg.asecs,
+                                            arg.ans,
+                                    },
+                                    {
+                                            arg.msecs,
+                                            arg.mns,
+                                    },
+                            };
+                            int ret = utimensat(AT_FDCWD, path.c_str(), time, 0);
+                            return UTimensReply{ret};
+                        } else if constexpr (std::is_same_v<T, StatfsReq>) {
+                            auto path = root.concat(arg.path);
+
+                            struct statvfs res{};
+                            int            ret = statvfs(path.c_str(), &res);
+
+                            return StatfsReply{ret,          res.f_frsize, res.f_bsize, res.f_blocks, res.f_bfree,
+                                               res.f_bavail, res.f_files,  res.f_ffree, res.f_favail, res.f_namemax};
+                        } else if constexpr (std::is_same_v<T, KeepAliveReq>) {
+                            return KeepAliveReply{};
+                        } else
+                            throw Exception(std::string("Unexpected message type: ") + typeid(T).name());
+                    },
+                    msg));
+        } catch (const std::exception& e) {
+            return Serialize::serialize(AnyMsgT{ErrorReply(std::string("Error: ") + e.what())});
+        }
+    }
+};
+
+static std::string read_file(std::string path) {
+    std::ifstream ifs(path);
+
+    if (!ifs.is_open()) {
+        throw Exception(std::string("Unable to open file ") + path);
+    }
+
+    std::stringstream buffer;
+    buffer << ifs.rdbuf();
+
+    return buffer.str();
+}
+
+void FsServer::run() {
+    RemoteFsServer server(checked_cast<uint16_t>(Options::get<size_t>("port")),
+                          parse_ip(Options::get<std::string>("ip")), Options::get<std::string>("ca_path"),
+                          Options::get<std::string>("pk_path"));
+
+    if (Options::get<std::string>("users_path").empty()) {
+        throw Exception("Please specify a file with user config: --users_path:<file>");
+    }
+
+    auto        users_file = read_file(Options::get<std::string>("users_path"));
+    std::string acl_file;
+
+    if (!Options::get<std::string>("acl_path").empty()) {
+        acl_file = read_file(Options::get<std::string>("acl_path"));
+    }
+
+    acl.load(acl_file, users_file);
+
+    server.run();
+}

remotefs/src/main.cpp

@@ -0,0 +1,45 @@
+#include <array>
+#include <atomic>
+#include <condition_variable>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <deque>
+#include <functional>
+#include <iostream>
+#include <mutex>
+#include <sstream>
+#include <thread>
+#include <vector>
+
+#include <netdb.h>
+#include <unistd.h>
+
+#include "FsClient.hpp"
+#include "FsServer.hpp"
+#include "Logger.h"
+#include "Options.h"
+#include "stuff.hpp"
+
+int main(int argc, char* argv[]) {
+    try {
+        Options::reset(argc, argv);
+        Logger::reset();
+
+        if (Options::get<std::string>("mode") == "server") {
+            FsServer().run();
+        } else if (Options::get<std::string>("mode") == "client") {
+            FsClient().run();
+        } else {
+            throw Exception("Unknown mode");
+        }
+
+    } catch (const std::exception& e) {
+        std::cerr << "\nError: " << e.what() << std::endl;
+        return -1;
+    } catch (...) {
+        std::cerr << "Crash!" << std::endl;
+        return -1;
+    }
+    return 0;
+}

remotefs/tests/CMakeLists.txt

@@ -0,0 +1,13 @@
+include(gTest)
+
+add_executable(
+        AclTest
+        src/AclTest.cpp
+)
+
+target_link_libraries(
+        AclTest PRIVATE
+        GTest::gtest_main remotefs_lib
+)
+
+gtest_discover_tests(AclTest DISCOVERY_TIMEOUT 600)

remotefs/tests/src/AclTest.cpp

@@ -0,0 +1,92 @@
+//
+// Created by Stepan Usatiuk on 08.12.2024.
+//
+
+#include <gtest/gtest.h>
+
+#include "Acl.hpp"
+
+// sha256 hash of string "password"
+static const std::string pass_hash = "5e884898da28047151d0e56f8dc6292773603d0d6aabbdd62a11ef721d1542d8";
+
+TEST(AclTest, Empty) {
+    ACL acl;
+    acl.load("", "username " + pass_hash);
+
+    ASSERT_TRUE(acl.authorize("username", "password"));
+    ASSERT_FALSE(acl.authorize("username", "1234"));
+    ASSERT_TRUE(acl.authorize_path("username", "/asdf"));
+    ASSERT_TRUE(acl.authorize_path("username", "/asdf/1234"));
+    ASSERT_TRUE(acl.authorize_path("username34", "/asdf/1234"));
+}
+
+TEST(AclTest, SimplePaths) {
+    ACL acl;
+    acl.load("/ username\n/public\n", "username " + pass_hash);
+
+    ASSERT_TRUE(acl.authorize_path("username", "/"));
+    ASSERT_FALSE(acl.authorize_path("no_username", "/"));
+    ASSERT_TRUE(acl.authorize_path("username", "/asdf"));
+    ASSERT_FALSE(acl.authorize_path("no_username", "/asdf"));
+
+    ASSERT_TRUE(acl.authorize_path("username", "/public"));
+    ASSERT_TRUE(acl.authorize_path("no_username", "/public"));
+    ASSERT_TRUE(acl.authorize_path("username", "/public/1234"));
+    ASSERT_TRUE(acl.authorize_path("no_username", "/public/1234"));
+}
+
+TEST(AclTest, MultipleUsers) {
+    ACL acl;
+    acl.load("/ user1\n/A user1 user2\n/B user2", "username " + pass_hash);
+
+    ASSERT_TRUE(acl.authorize_path("user1", "/"));
+    ASSERT_TRUE(acl.authorize_path("user1", "/asdf"));
+    ASSERT_FALSE(acl.authorize_path("user2", "/"));
+    ASSERT_FALSE(acl.authorize_path("user2", "/asdf"));
+
+    ASSERT_TRUE(acl.authorize_path("user1", "/A"));
+    ASSERT_TRUE(acl.authorize_path("user1", "/A/a"));
+    ASSERT_TRUE(acl.authorize_path("user2", "/A"));
+    ASSERT_TRUE(acl.authorize_path("user2", "/A/a"));
+
+    ASSERT_TRUE(acl.authorize_path("user2", "/B/a"));
+    ASSERT_TRUE(acl.authorize_path("user2", "/B"));
+    ASSERT_FALSE(acl.authorize_path("user1", "/B"));
+    ASSERT_FALSE(acl.authorize_path("user1", "/B/a"));
+
+    ASSERT_FALSE(acl.authorize_path("user3", "/B"));
+    ASSERT_FALSE(acl.authorize_path("user3", "/A"));
+    ASSERT_FALSE(acl.authorize_path("user3", "/"));
+}
+
+TEST(AclTest, MultipleUsers3) {
+    ACL acl;
+    acl.load("/ user1\n/A user1 user2\n/B user2\n/C", "username " + pass_hash);
+
+    ASSERT_TRUE(acl.authorize_path("user1", "/"));
+    ASSERT_TRUE(acl.authorize_path("user1", "/asdf"));
+    ASSERT_FALSE(acl.authorize_path("user2", "/"));
+    ASSERT_FALSE(acl.authorize_path("user2", "/asdf"));
+
+    ASSERT_TRUE(acl.authorize_path("user1", "/A"));
+    ASSERT_TRUE(acl.authorize_path("user1", "/A/a"));
+    ASSERT_TRUE(acl.authorize_path("user2", "/A"));
+    ASSERT_TRUE(acl.authorize_path("user2", "/A/a"));
+
+    ASSERT_TRUE(acl.authorize_path("user2", "/B/a"));
+    ASSERT_TRUE(acl.authorize_path("user2", "/B"));
+    ASSERT_FALSE(acl.authorize_path("user1", "/B"));
+    ASSERT_FALSE(acl.authorize_path("user1", "/B/a"));
+
+    ASSERT_FALSE(acl.authorize_path("user3", "/B"));
+    ASSERT_FALSE(acl.authorize_path("user3", "/A"));
+    ASSERT_FALSE(acl.authorize_path("user3", "/"));
+
+    ASSERT_TRUE(acl.authorize_path("user1", "/C"));
+    ASSERT_TRUE(acl.authorize_path("user2", "/C"));
+    ASSERT_TRUE(acl.authorize_path("user3", "/C"));
+
+    ASSERT_TRUE(acl.authorize_path("user1", "/C/a"));
+    ASSERT_TRUE(acl.authorize_path("user2", "/C/a"));
+    ASSERT_TRUE(acl.authorize_path("user3", "/C/a"));
+}

utils/CMakeLists.txt

@@ -0,0 +1,22 @@
+cmake_minimum_required(VERSION 3.15)
+
+add_library(utils
+        include/Serialize.hpp
+        src/Exception.cpp
+        include/Exception.h
+        include/stuff.hpp
+        src/stuff.cpp
+        include/Logger.h
+        src/Logger.cpp
+        src/Options.cpp
+        include/Options.h
+        include/SHA.h
+        src/SHA.cpp
+)
+
+find_package(OpenSSL REQUIRED)
+target_link_libraries(utils PRIVATE OpenSSL::SSL OpenSSL::Crypto)
+
+target_include_directories(utils PUBLIC include)
+
+add_subdirectory(tests)

utils/include/Exception.h

@@ -0,0 +1,37 @@
+//
+// Created by Stepan Usatiuk on 01.05.2023.
+//
+
+#ifndef SEMBACKUP_EXCEPTION_H
+#define SEMBACKUP_EXCEPTION_H
+
+#include <cstring>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+/// Custom exception class that uses execinfo to append a stacktrace to the exception message
+class Exception : public std::runtime_error {
+public:
+    Exception(const std::string& text);
+    Exception(const char* text);
+
+private:
+    /// Static function to get the current stacktrace
+    static std::string getStacktrace();
+};
+
+class ErrnoException : public Exception {
+public:
+    ErrnoException(const std::string& text) : Exception(text + " " + std::strerror(errno)) {}
+    ErrnoException(const char* text) : Exception(std::string(text) + " " + std::strerror(errno)) {}
+};
+
+class OpenSSLException : public Exception {
+public:
+    OpenSSLException(const std::string& text);
+    OpenSSLException(const char* text);
+};
+
+
+#endif // SEMBACKUP_EXCEPTION_H

utils/include/Logger.h

@@ -0,0 +1,65 @@
+//
+// Created by stepus53 on 3.1.24.
+//
+
+#ifndef PSIL_LOGGER_H
+#define PSIL_LOGGER_H
+
+#include <chrono>
+#include <functional>
+#include <iostream>
+#include <mutex>
+#include <ostream>
+#include <shared_mutex>
+#include <string>
+#include <unordered_map>
+
+class Logger {
+public:
+    Logger();
+
+    enum LogLevel { ALWAYS = 0, ERROR = 1, INFO = 2, DEBUG = 3, TRACE = 4 };
+
+    enum LogTag { RemoteFs, LogTagMax };
+
+    static void log(LogTag tag, const std::string& what, int level = INFO);
+    static void log(LogTag tag, const std::function<void(std::ostream&)>& fn, int level = INFO);
+
+    // 0 - disabled
+    // 1 - error
+    // 2 - info
+    // 3 - debug
+    // 4 - trace
+    static void set_level(LogTag tag, int level);
+    static int  get_level(LogTag tag);
+    static bool en_level(LogTag tag, int level);
+
+    static void set_out(std::ostream& out);
+    static void set_out_err(std::ostream& out_err);
+
+    static void reset();
+
+    static Logger& get();
+
+    static LogTag             str_to_tag(const std::string& str) { return _str_to_tag.at(str); }
+    static const std::string& tag_to_str(LogTag tag) { return _tag_to_str.at(tag); }
+
+private:
+    std::array<LogLevel, LogTag::LogTagMax>            _levels{};
+    static inline std::unordered_map<int, std::string> _level_names{
+            {ALWAYS, "ALWAYS"}, {ERROR, "ERROR"}, {INFO, "INFO"}, {DEBUG, "DEBUG"}, {TRACE, "TRACE"},
+    };
+    static inline std::unordered_map<std::string, LogTag> _str_to_tag{
+            {"Server", RemoteFs},
+    };
+    static inline std::unordered_map<LogTag, std::string> _tag_to_str{
+            {RemoteFs, "Server"},
+    };
+
+    std::chrono::time_point<std::chrono::high_resolution_clock> _start_time = std::chrono::high_resolution_clock::now();
+    std::reference_wrapper<std::ostream>                        _out        = std::cout;
+    std::reference_wrapper<std::ostream>                        _out_err    = std::cerr;
+};
+
+
+#endif // PSIL_LOGGER_H

utils/include/Options.h

@@ -0,0 +1,66 @@
+//
+// Created by stepus53 on 3.1.24.
+//
+
+#ifndef PSIL_OPTIONS_H
+#define PSIL_OPTIONS_H
+
+
+#include <cstddef>
+#include <cstdint>
+#include <mutex>
+#include <shared_mutex>
+#include <stdexcept>
+#include <string>
+#include <unordered_map>
+#include <variant>
+
+#include "Exception.h"
+
+class Options {
+public:
+    template<typename T>
+    static T get(const std::string& opt) {
+        Options& o = get();
+        if (_defaults.find(opt) == _defaults.end())
+            throw Exception("Unknown option " + opt);
+        if (!std::holds_alternative<T>(_defaults.at(opt)))
+            throw Exception("Bad option type " + opt);
+        return std::get<T>(o._current.at(opt));
+    }
+
+    template<typename T>
+    static void set(const std::string& opt, const T& val) {
+        Options& o = get();
+        if (_defaults.find(opt) == _defaults.end())
+            throw Exception("Unknown option " + opt);
+        if (!std::holds_alternative<T>(_defaults.at(opt)))
+            throw Exception("Bad option type " + opt);
+        o._current[opt] = val;
+    }
+
+    static void     reset();
+    static void     reset(int argc, char* argv[]);
+    static Options& get();
+
+private:
+    using OptionType = std::variant<size_t, std::string, bool>;
+
+    const static inline std::unordered_map<std::string, OptionType> _defaults{{"ip", "127.0.0.1"},
+                                                                              {"port", 42069U},
+                                                                              {"default_log_level", 2U},
+                                                                              {"timeout", 30U},
+                                                                              {"ca_path", "cert.pem"},
+                                                                              {"pk_path", "key.pem"},
+                                                                              {"mode", "server"},
+                                                                              {"path", ""},
+                                                                              {"acl_path", ""},
+                                                                              {"users_path", ""},
+                                                                              {"username", ""},
+                                                                              {"password", ""}};
+
+    std::unordered_map<std::string, OptionType> _current = _defaults;
+};
+
+
+#endif // PSIL_OPTIONS_H

utils/include/SHA.h

@@ -0,0 +1,49 @@
+//
+// Created by Stepan Usatiuk on 15.04.2023.
+//
+
+#ifndef SEMBACKUP_SHA_H
+#define SEMBACKUP_SHA_H
+
+#include <array>
+#include <memory>
+#include <vector>
+
+#include <openssl/evp.h>
+
+/// Class to handle SHA hashing
+/**
+ * Based on: https://wiki.openssl.org/index.php/EVP_Message_Digests
+ */
+class SHA {
+public:
+    /// Constructs an empty SHA hasher instance
+    /// \throws     Exception on initialization error
+    SHA();
+
+    /// Calculates the hash for a given \p in char vector
+    /// \param in   Constant reference to an input vector
+    /// \return     SHA hash of \p in
+    static std::string calculate(const std::vector<char> &in);
+
+    /// Calculates the hash for a given \p in string
+    /// \param in   Constant reference to an input string
+    /// \return     SHA hash of \p in
+    static std::string calculate(const std::string &in);
+
+    /// Append a vector of chars to the current hash
+    /// \param in   Constant reference to an input vector
+    /// \throws     Exception on any error
+    void feedData(const std::vector<char> &in);
+
+    /// Returns the hash, resets the hashing context
+    /// \throws     Exception on any error
+    std::string getHash();
+
+private:
+    const std::unique_ptr<EVP_MD_CTX, decltype(&EVP_MD_CTX_free)> mdctx{EVP_MD_CTX_new(),
+                                                                        &EVP_MD_CTX_free};///< Current hashing context
+};
+
+
+#endif//SEMBACKUP_SHA_H

utils/include/SerializableStruct.hpp

@@ -0,0 +1,50 @@
+//
+// Created by Stepan Usatiuk on 08.12.2024.
+//
+
+#ifndef SERIALIZABLESTRUCT_HPP
+#define SERIALIZABLESTRUCT_HPP
+
+#include <variant>
+#include <vector>
+
+#include "Serialize.hpp"
+
+#define MAKE_FIELD(type, name) type name;
+
+#define READ_FIELD(type, name) name = Serialize::deserialize<type>(in, end);
+
+#define SERIALIZE_FIELD(type, name) Serialize::serialize(name, out);
+
+#define COMPARE_FIELD(type, name) name == rhs.name&&
+
+#define ARG_FIELD(type, name) type _##name,
+
+#define CONSTRUCT_FIELD(type, name) name(std::move(_##name)),
+
+#define DECLARE_SERIALIZABLE(name, FIELDS)                                                                             \
+    class name {                                                                                                       \
+    public:                                                                                                            \
+        using serializable = std::true_type;                                                                           \
+        FIELDS(MAKE_FIELD)                                                                                             \
+                                                                                                                       \
+        std::monostate _dummy;                                                                                         \
+                                                                                                                       \
+        explicit name(FIELDS(ARG_FIELD) std::monostate dummy = std::monostate{}) :                                     \
+            FIELDS(CONSTRUCT_FIELD) _dummy(dummy) {}                                                                   \
+                                                                                                                       \
+        explicit name(std::vector<uint8_t>::const_iterator& in, const std::vector<uint8_t>::const_iterator& end) {     \
+            FIELDS(READ_FIELD)                                                                                         \
+        }                                                                                                              \
+        void serialize(std::vector<uint8_t>& out) const { FIELDS(SERIALIZE_FIELD) }                                    \
+                                                                                                                       \
+        bool operator==(name const& rhs) const { return FIELDS(COMPARE_FIELD) true; }                                  \
+                                                                                                                       \
+    private:
+
+#define DECLARE_SERIALIZABLE_END                                                                                       \
+    }                                                                                                                  \
+    ;
+
+
+#endif // SERIALIZABLESTRUCT_HPP

utils/include/Serialize.hpp

@@ -0,0 +1,286 @@
+//
+// Created by Stepan Usatiuk on 15.04.2023.
+//
+
+#ifndef SEMBACKUP_SERIALIZE_H
+#define SEMBACKUP_SERIALIZE_H
+
+#include <cstddef>
+#include <memory>
+#include <optional>
+#include <type_traits>
+#include <variant>
+
+#include "Exception.h"
+#include "stuff.hpp"
+
+/// Serialization library
+/**
+ *  To serialize the objects in Repository, we have to handle a couple of cases:
+ *  1. Serializing integers (object ids, etc...)
+ *  2. Serializing enums (object types)
+ *  3. Serializing char vectors and strings
+ *  4. Serializing other STL containers (which also requires serializing pairs)
+ *  5. Serializing custom structs (including the objects themselves)
+ *
+ *  With this library it is possible to do all of that.
+ *
+ */
+namespace Serialize {
+template<typename, typename = void, typename = void>
+struct is_pair : std::false_type {};
+
+template<typename P>
+struct is_pair<P, std::void_t<decltype(std::declval<P>().first)>, std::void_t<decltype(std::declval<P>().second)>>
+    : std::true_type {};
+
+template<typename Args, template<typename...> class T>
+struct is_template_of : std::false_type {};
+
+template<typename... Args, template<typename...> class T>
+struct is_template_of<T<Args...>, T> : std::true_type {};
+
+template<typename, typename, typename = void>
+struct has_emplace_back : std::false_type {};
+
+template<typename T, typename V>
+struct has_emplace_back<T, V, std::void_t<decltype(T().emplace_back(std::declval<V>()))>> : std::true_type {};
+
+template<typename, typename = void, typename = void>
+struct serializable : std::false_type {};
+
+/// Checks if the object has the `serializable` type
+/// In that case, its serialization will be delegated to its .serialize() parameter,
+/// and deserialization to its T(char vector iterator in, const char vector iterator end) constructor,
+/// similar to Serialize::deserialize
+template<typename T>
+struct serializable<T, std::void_t<decltype(T::serializable::value)>> : std::true_type {};
+
+/// Deserializes object of type \p T starting from fist byte \p in, advances the iterator past the end of object
+/// \tparam T   Type to deserialize
+/// \param in   Iterator to the first byte of the object
+/// \param end  End iterator of source container
+/// \return     Deserialized value
+template<typename T, typename C = std::vector<uint8_t>>
+static std::optional<T> deserializeOpt(typename C::const_iterator& in, const typename C::const_iterator& end);
+
+/// Deserializes object of type \p T starting from fist byte \p in, advances the iterator past the end of object
+/// \tparam T   Type to deserialize
+/// \param in   Iterator to the first byte of the object
+/// \param end  End iterator of source container
+/// \return     Deserialized value
+template<typename T, typename C = std::vector<uint8_t>>
+static T deserialize(typename C::const_iterator& in, const typename C::const_iterator& end);
+
+/// Serializes object of type \p T into vector \p out
+/// \tparam T   Type to serialize
+/// \param what Constant reference to the serialized object
+/// \param out  Reference to output vector
+template<typename T, typename C = std::vector<uint8_t>>
+static void serialize(const T& what, C& out);
+
+/// Serializes the object of type \p T and returns the resulting vector
+/// \tparam T   Type to serialize
+/// \param o    Constant reference to the serialized object
+/// \return     Serialized data
+template<typename T, typename C = std::vector<uint8_t>>
+static C serialize(const T& o);
+
+/// Deserializes object of type \p T from input vector \p from
+/// \tparam T   Type to deserialize
+/// \param from Constant reference to the serialized object
+/// \return     Deserialized value
+template<typename T, typename C = std::vector<uint8_t>>
+static std::optional<T> deserializeOpt(const C& from);
+
+/// Deserializes object of type \p T from input vector \p from
+/// \tparam T   Type to deserialize
+/// \param from Constant reference to the serialized object
+/// \return     Deserialized value
+template<typename T, typename C = std::vector<uint8_t>>
+static T deserialize(const C& from);
+
+// Deserializes an std::variant instance, with readIdx serving as an index of the alternative on the wire
+template<std::size_t I = 0, typename C, typename V>
+std::optional<V> deserializeVar(size_t readIdx, typename C::const_iterator& in, const typename C::const_iterator& end)
+    requires is_template_of<V, std::variant>::value
+{
+    if constexpr (I < std::variant_size_v<V>) {
+        if (readIdx == I) {
+            auto res = deserializeOpt<std::variant_alternative_t<I, V>>(in, end);
+
+            if (res)
+                return std::make_optional(V{std::move(*res)});
+
+            return std::nullopt;
+        }
+        return deserializeVar<I + 1, C, V>(readIdx, in, end);
+    } else {
+        return std::nullopt;
+    }
+}
+
+template<typename T, typename C>
+std::optional<T> deserializeOpt(typename C::const_iterator& in, const typename C::const_iterator& end) {
+    if constexpr (serializable<T>::value) {
+        // If the object declares itself as serializable, call its constructor with in and end
+        return T(in, end);
+    } else if constexpr (std::is_same_v<T, std::monostate>) {
+        // No-op
+        return std::make_optional(std::monostate{});
+    } else {
+        if (in >= end)
+            return std::nullopt;
+
+        if constexpr (is_pair<T>::value) {
+            // If the object is pair, deserialize the first and second element and return the pair
+            using KT = typename std::remove_const<decltype(T::first)>::type;
+            using VT = typename std::remove_const<decltype(T::second)>::type;
+            auto K   = deserialize<KT>(in, end);
+            auto V   = deserialize<VT>(in, end);
+            return T(std::move(K), std::move(V));
+        } else if constexpr (std::is_enum<T>::value) {
+            // If the object is an enum, deserialize an int and cast it to the enum
+            auto tmp = deserialize<uint32_t>(in, end);
+            if (tmp >= 0 && tmp < static_cast<uint32_t>(T::END))
+                return static_cast<T>(tmp);
+            else
+                return std::nullopt;
+        } else if constexpr (sizeof(T) == 1) {
+            // If it's a single byte, just copy it
+            if (std::distance(in, end) < checked_cast<ssize_t>(sizeof(T)))
+                return std::nullopt;
+            return *(in++);
+        } else if constexpr (std::is_integral<T>::value) {
+            static_assert(sizeof(T) <= sizeof(uint64_t));
+            uint64_t tmp;
+            // If the object is a number, copy it byte-by-byte
+            if (std::distance(in, end) < checked_cast<ssize_t>(sizeof(tmp)))
+                return std::nullopt;
+
+            std::copy(in, in + sizeof(tmp), reinterpret_cast<char*>(&tmp));
+            in += sizeof(tmp);
+            return static_cast<T>(be64toh(tmp));
+        } else if constexpr (is_template_of<T, std::variant>::value) {
+            auto index = deserializeOpt<uint64_t>(in, end);
+            if (!index.has_value())
+                return std::nullopt;
+            return deserializeVar<0, C, T>(*index - 1, in, end);
+        } else {
+            // Otherwise we treat it as a container, in format of <number of elements>b<elements>e
+            auto size = deserializeOpt<size_t>(in, end);
+            if (!size)
+                return std::nullopt;
+
+            auto b = deserialize<char>(in, end);
+            if (!b || b != 'b')
+                return std::nullopt;
+
+            T out;
+            if constexpr (sizeof(typename T::value_type) == 1) {
+                // Optimization for char vectors
+                if (std::distance(in, end) < checked_cast<ssize_t>(*size))
+                    return std::nullopt;
+                out.insert(out.end(), in, in + checked_cast<ssize_t>(*size));
+                in += checked_cast<ssize_t>(*size);
+            } else
+                for (size_t i = 0; i < *size; i++) {
+                    using V = typename T::value_type;
+                    V v     = deserialize<V>(in, end);
+                    // Try either emplace_back or emplace if it doesn't exist
+                    if constexpr (has_emplace_back<T, V>::value)
+                        out.emplace_back(std::move(v));
+                    else
+                        out.emplace(std::move(v));
+                }
+
+            b = deserialize<char>(in, end);
+            if (!b || b != 'e')
+                return std::nullopt;
+
+            return out;
+        }
+    }
+}
+
+template<typename T, typename C>
+static T deserialize(typename C::const_iterator& in, const typename C::const_iterator& end) {
+    std::optional<T> out = deserializeOpt<T>(in, end);
+    if (!out)
+        throw Exception("deserialize failed");
+    return out.value();
+}
+
+template<typename T, typename C>
+static T deserialize(const C& from) {
+    std::optional<T> out = deserializeOpt<T>(from);
+    if (!out)
+        throw Exception("deserialize failed");
+    return out.value();
+}
+
+template<typename... T, typename C>
+void serialize(const std::variant<T...>& what, C& out) {
+    serialize<uint64_t>(what.index() + 1, out);
+    std::visit(
+            [&out](auto&& arg) -> void {
+                using I = std::decay_t<decltype(arg)>;
+                serialize<I>(arg, out);
+            },
+            what);
+}
+
+template<typename T, typename C>
+void serialize(const T& what, C& out) {
+    if constexpr (serializable<T>::value) {
+        // If the object declares itself as serializable, call its serialize method
+        what.serialize(out);
+    } else if constexpr (std::is_same_v<T, std::monostate>) {
+        // No-op
+    } else if constexpr (is_pair<T>::value) {
+        // If the object is pair, serialize the first and second element
+        serialize(what.first, out);
+        serialize(what.second, out);
+    } else if constexpr (std::is_enum<T>::value) {
+        // If the object is an enum, cast it to an int and serialize that
+        serialize(static_cast<uint32_t>(what), out);
+    } else if constexpr (sizeof(T) == 1) {
+        // If it's a single byte, just copy it
+        out.push_back(static_cast<typename C::value_type>(what));
+    } else if constexpr (std::is_integral<T>::value) {
+        static_assert(sizeof(T) <= sizeof(uint64_t));
+        // If the object is a number, copy it byte-by-byte
+        uint64_t tmp = htobe64(static_cast<uint64_t>(what));
+        static_assert(sizeof(tmp) == 8);
+        out.insert(out.end(), (reinterpret_cast<const char*>(&tmp)),
+                   (reinterpret_cast<const char*>(&tmp) + sizeof(tmp)));
+    } else {
+        // Otherwise we treat it as a container, in format of <number of elements>b<elements>e
+        serialize(what.size(), out);
+        serialize('b', out);
+        if constexpr (sizeof(typename T::value_type) == 1) {
+            // Optimization for char vectors
+            out.insert(out.end(), what.begin(), what.end());
+        } else
+            for (auto const& i: what) {
+                serialize(i, out);
+            }
+        serialize('e', out);
+    }
+}
+
+template<typename T, typename C>
+C serialize(const T& o) {
+    C out;
+    serialize(o, out);
+    return out;
+}
+
+template<typename T, typename C>
+std::optional<T> deserializeOpt(const C& from) {
+    auto bgwr = from.begin();
+    return deserialize<T>(bgwr, from.end());
+}
+} // namespace Serialize
+
+#endif // SEMBACKUP_SERIALIZE_H

utils/include/stuff.hpp

@@ -0,0 +1,37 @@
+//
+// Created by stepus53 on 24.8.24.
+//
+
+#ifndef STUFF_H
+#define STUFF_H
+
+#include <cassert>
+#include "Exception.h"
+
+
+#ifdef __APPLE__
+#include <machine/endian.h>
+#define htobe64(x) htonll(x)
+#define be64toh(x) ntohll(x)
+#else
+#include <endian.h>
+#endif
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wsign-conversion"
+#pragma GCC diagnostic ignored "-Wsign-compare"
+
+template<typename To, typename From>
+constexpr To checked_cast(const From& f) {
+    To result = static_cast<To>(f);
+    if (f != result) {
+        throw Exception("Value out of bounds");
+    }
+    return result;
+}
+
+#pragma GCC diagnostic pop
+
+std::vector<std::string> split(const std::string& s, char delim);
+
+#endif // STUFF_H

utils/src/Exception.cpp

@@ -0,0 +1,51 @@
+//
+// Created by Stepan Usatiuk on 01.05.2023.
+//
+
+#include "Exception.h"
+
+#include "stuff.hpp"
+
+#include <execinfo.h>
+#include <sstream>
+
+#include <openssl/err.h>
+
+Exception::Exception(const std::string& text) : runtime_error(text + "\n" + getStacktrace()) {}
+
+Exception::Exception(const char* text) : runtime_error(std::string(text) + "\n" + getStacktrace()) {}
+
+// Based on: https://www.gnu.org/software/libc/manual/html_node/Backtraces.html
+std::string Exception::getStacktrace() {
+    std::vector<void*> functions(50);
+    char**             strings;
+    int                n;
+
+    n       = backtrace(functions.data(), 50);
+    strings = backtrace_symbols(functions.data(), n);
+
+    std::stringstream out;
+
+    if (strings != nullptr) {
+        out << "Stacktrace:" << std::endl;
+        for (int i = 0; i < n; i++)
+            out << strings[i] << std::endl;
+    }
+
+    free(strings);
+    return out.str();
+}
+
+// https://stackoverflow.com/questions/42106339/how-to-get-the-error-string-in-openssl
+static std::string OpenSSLErrorToString() {
+    BIO* bio = BIO_new(BIO_s_mem());
+    ERR_print_errors(bio);
+    char*       buf;
+    size_t      len = checked_cast<size_t>(BIO_get_mem_data(bio, &buf));
+    std::string ret(buf, len);
+    BIO_free(bio);
+    return ret;
+}
+
+OpenSSLException::OpenSSLException(const std::string& text) : Exception(text + "\n" + OpenSSLErrorToString()) {}
+OpenSSLException::OpenSSLException(const char* text) : Exception(std::string(text) + "\n" + OpenSSLErrorToString()) {}

utils/src/Logger.cpp

@@ -0,0 +1,60 @@
+//
+// Created by stepus53 on 3.1.24.
+//
+
+#include "Logger.h"
+
+#include <iomanip>
+#include <sstream>
+
+#include "Options.h"
+
+Logger& Logger::get() {
+    static Logger logger;
+    return logger;
+}
+
+void Logger::log(LogTag tag, const std::string& what, int level) {
+    if (!en_level(tag, level))
+        return;
+    {
+        auto              now = std::chrono::high_resolution_clock::now();
+        std::stringstream out;
+        out << std::setprecision(3) << std::fixed << "["
+            << static_cast<double>(
+                       std::chrono::duration_cast<std::chrono::milliseconds>(now - get()._start_time).count()) /
+                        1000.0
+            << "s]"
+            << "[" << tag_to_str(tag) << "][" << get()._level_names.at(level) << "] " << what << '\n';
+
+        if (level == 1)
+            get()._out_err.get() << out.str();
+        else
+            get()._out.get() << out.str();
+    }
+}
+
+void Logger::log(LogTag tag, const std::function<void(std::ostream&)>& fn, int level) {
+    if (!en_level(tag, level))
+        return;
+
+    std::stringstream out;
+    fn(out);
+    log(tag, out.str(), level);
+}
+
+void Logger::set_level(LogTag tag, int level) { get()._levels[tag] = static_cast<LogLevel>(level); }
+void Logger::set_out(std::ostream& out) { get()._out = out; }
+void Logger::set_out_err(std::ostream& out_err) { get()._out_err = out_err; }
+void Logger::reset() { get()._levels.fill(static_cast<LogLevel>(Options::get<size_t>("default_log_level"))); }
+
+int Logger::get_level(LogTag tag) { return get()._levels.at(tag); }
+
+bool Logger::en_level(LogTag tag, int level) {
+    int en_level = get_level(tag);
+    if (en_level < level)
+        return false;
+    return true;
+}
+
+Logger::Logger() { _levels.fill(static_cast<LogLevel>(Options::get<size_t>("default_log_level"))); }

utils/src/Options.cpp

@@ -0,0 +1,71 @@
+//
+// Created by stepus53 on 3.1.24.
+//
+
+#include "Options.h"
+
+#include "Exception.h"
+
+#include "Logger.h"
+
+#include <fstream>
+#include <iostream>
+#include <optional>
+#include <sstream>
+#include <stdexcept>
+
+Options& Options::get() {
+    static Options opts;
+    return opts;
+}
+
+void Options::reset() { get()._current = _defaults; }
+
+void Options::reset(int argc, char* argv[]) {
+    Options::reset();
+
+    for (int i = 1; i < argc; i++) {
+        std::string arg = argv[i];
+
+        if (arg.length() < 2 || arg.substr(0, 2) != "--") {
+            throw Exception("Can't parse argument " + arg);
+        }
+        std::string rest = arg.substr(2);
+
+        std::vector<std::string> split;
+        {
+            std::istringstream ins(rest);
+            std::string        cur;
+            while (std::getline(ins, cur, ':')) {
+                split.emplace_back(cur);
+            }
+        }
+
+        if (split.empty())
+            throw Exception("Can't parse argument " + arg);
+
+        if (split.at(0) == "log") {
+            if (split.size() != 3)
+                throw Exception("Log options must be in format --log:TAG:LEVEL");
+            try {
+                Logger::set_level(Logger::str_to_tag(split.at(1)), std::stoi(split.at(2)));
+            } catch (...) {
+                throw Exception("Log options must be in format --log:TAG:LEVEL");
+            }
+        } else if (split.size() == 1) {
+            std::string str = split.at(0);
+            if (str.back() != '+' && str.back() != '-') {
+                throw Exception("Bool options must be in format --option[+/-], instead have" + arg);
+            }
+            Options::set<bool>(str.substr(0, str.length() - 1), str.back() == '+' ? true : false);
+        } else if (split.size() == 2) {
+            try {
+                Options::set<size_t>(split.at(0), std::stoul(split.at(1)));
+            } catch (...) {
+                Options::set<std::string>(split.at(0), split.at(1));
+            }
+        } else {
+            throw Exception("Can't parse argument " + arg);
+        }
+    }
+}

utils/src/SHA.cpp

@@ -0,0 +1,43 @@
+//
+// Created by Stepan Usatiuk on 15.04.2023.
+//
+
+#include "SHA.h"
+
+#include "Exception.h"
+
+std::string SHA::calculate(const std::vector<char> &in) {
+    SHA hasher;
+    hasher.feedData(in);
+    return hasher.getHash();
+}
+
+SHA::SHA() {
+    if (!mdctx) throw Exception("Can't create hashing context!");
+
+    if (!EVP_DigestInit_ex(mdctx.get(), EVP_sha256(), nullptr)) throw Exception("Can't create hashing context!");
+}
+
+void SHA::feedData(const std::vector<char> &in) {
+    if (in.empty()) return;
+    if (!EVP_DigestUpdate(mdctx.get(), in.data(), in.size())) throw Exception("Error hashing!");
+}
+
+std::string SHA::getHash() {
+    std::array<char, 32> out;
+    unsigned int s = 0;
+
+    if (!EVP_DigestFinal_ex(mdctx.get(), reinterpret_cast<unsigned char *>(out.data()), &s))
+        throw Exception("Error hashing!");
+
+    if (s != out.size()) throw Exception("Error hashing!");
+
+    if (!EVP_MD_CTX_reset(mdctx.get())) throw Exception("Error hashing!");
+
+    return {out.begin(), out.end()};
+}
+
+std::string SHA::calculate(const std::string &in) {
+    std::vector<char> tmp(in.begin(), in.end());
+    return SHA::calculate(tmp);
+}

utils/src/stuff.cpp

@@ -0,0 +1,19 @@
+//
+// Created by Stepan Usatiuk on 04.01.2025.
+//
+
+
+#include "stuff.hpp"
+
+#include <sstream>
+
+std::vector<std::string> split(const std::string& s, char delim) {
+    std::vector<std::string> elems;
+    std::string              token;
+
+    std::istringstream tokenStream(s);
+    while (std::getline(tokenStream, token, delim))
+        elems.push_back(token);
+
+    return elems;
+}

utils/tests/CMakeLists.txt

@@ -0,0 +1,13 @@
+include(gTest)
+
+add_executable(
+        SerializableHelperTest
+        src/SerializableHelperTest.cpp
+)
+
+target_link_libraries(
+        SerializableHelperTest PRIVATE
+        GTest::gtest_main utils
+)
+
+gtest_discover_tests(SerializableHelperTest DISCOVERY_TIMEOUT 600)

utils/tests/src/SerializableHelperTest.cpp

@@ -0,0 +1,44 @@
+//
+// Created by Stepan Usatiuk on 08.12.2024.
+//
+
+#include <gtest/gtest.h>
+
+#include "Serialize.hpp"
+#include "SerializableStruct.hpp"
+
+#define TEST_STRUCT(FIELD)                                                                                             \
+    FIELD(long, _test2)                                                                                                \
+    FIELD(std::string, _test_str)                                                                                      \
+    FIELD(std::vector<uint8_t>, _test_vec)
+
+DECLARE_SERIALIZABLE(TestStruct, TEST_STRUCT)
+DECLARE_SERIALIZABLE_END
+
+TEST(SerializableHelperTestStruct, Works) {
+    TestStruct test(123, "hello", {});
+
+    std::vector<uint8_t> out;
+    test.serialize(out);
+
+    TestStruct deserialized = Serialize::deserialize<TestStruct>(out);
+
+    ASSERT_EQ(test, deserialized);
+}
+
+TEST(SerializableHelperTestStruct, Variant) {
+    using VarT = std::variant<int, std::string>;
+
+    static_assert(Serialize::is_template_of<VarT, std::variant>::value);
+
+    VarT intV{1};
+    VarT strV{"hello"};
+
+    auto serializedInt   = Serialize::serialize(intV);
+    auto deserializedInt = Serialize::deserialize<VarT>(serializedInt);
+    ASSERT_EQ(intV, deserializedInt);
+
+    auto serializedString   = Serialize::serialize(strV);
+    auto deserializedString = Serialize::deserialize<VarT>(serializedString);
+    ASSERT_EQ(strV, deserializedString);
+}