proper cmake

src/repo/srcs/FileRepository.cpp

@@ -0,0 +1,246 @@
+//
+// Created by Stepan Usatiuk on 14.04.2023.
+//
+
+#include "../includes/FileRepository.h"
+
+#include <exception>
+#include <iterator>
+#include <mutex>
+
+#include "../../filters/includes/CheckFilter.h"
+#include "../../filters/includes/FilterFactory.h"
+#include "../includes/Object.h"
+#include "../includes/Serialize.h"
+
+FileRepository::FileRepository(Config config) : Repository(std::move(config)), root(std::filesystem::path(this->config.getStr("repo"))), writeCacheMax(config.getInt("repo-target") * 1024 * 1024) {}
+
+bool FileRepository::exists() {
+    return std::filesystem::is_directory(root) && std::filesystem::exists(root / "info");
+}
+
+bool FileRepository::flush() {
+    flushWriteCache(std::unique_lock(writeCacheLock));
+    return true;
+}
+
+bool FileRepository::open() {
+    if (!exists()) throw Exception("Repository doesn't exist!");
+
+    auto readConf = Serialize::deserialize<Config>(CheckFilter::filterReadStatic(readFile(root / "info")));
+    std::swap(config, readConf);
+    config.merge(readConf);
+
+    if (config.getStr("compression") != "none") filters.addFilter(FilterFactory::makeFilter(config.getStr("compression"), config));
+    if (config.getStr("encryption") != "none") filters.addFilter(FilterFactory::makeFilter(config.getStr("encryption"), config));
+    filters.addFilter(FilterFactory::makeFilter("crc", config));
+
+    ready = true;
+    try {
+        std::tie(maxFileId, offsetIndex) = Serialize::deserialize<std::pair<decltype(maxFileId), decltype(offsetIndex)>>(filters.filterRead(readFile(root / "offsets")));
+        std::tie(keyIndex, largestUnusedId) = Serialize::deserialize<std::pair<decltype(keyIndex), decltype(largestUnusedId)>>(filters.filterRead(readFile(root / "index")));
+    } catch (const std::exception &e) {
+        ready = false;
+        throw;
+    }
+
+    return true;
+}
+
+bool FileRepository::init() {
+    if (ready) throw Exception("Trying to initialize already initialized repository!");
+    if (exists()) throw Exception("Trying to initialize already existing repository!");
+
+    if (!std::filesystem::is_directory(root) && !std::filesystem::create_directories(root))
+        throw Exception("Can't create directory " + root.u8string());
+
+    writeFile(root / "info", CheckFilter::filterWriteStatic(Serialize::serialize(config)));
+
+    if (config.getStr("compression") != "none") filters.addFilter(FilterFactory::makeFilter(config.getStr("compression"), config));
+    if (config.getStr("encryption") != "none") filters.addFilter(FilterFactory::makeFilter(config.getStr("encryption"), config));
+    filters.addFilter(FilterFactory::makeFilter("crc", config));
+
+    ready = true;
+    return true;
+}
+
+FileRepository::~FileRepository() {
+    if (ready) {
+        ready = false;
+        flushWriteCache(std::unique_lock(writeCacheLock));
+
+        writeFile(root / "offsets", filters.filterWrite(Serialize::serialize(std::make_pair(maxFileId, offsetIndex))));
+        writeFile(root / "index", filters.filterWrite(Serialize::serialize(std::make_pair(keyIndex, largestUnusedId))));
+    }
+}
+
+std::vector<char> FileRepository::getObject(Object::idType id) const {
+    if (!ready) throw Exception("Tried working with uninitialized repo!");
+
+    std::unique_lock lock(repoLock);
+    if (offsetIndex.count(id) == 0)
+        throw Exception("Object with id " + std::to_string(id) + " doesn't exist!");
+    auto entry = offsetIndex.at(id);
+    lock.unlock();
+
+    return filters.filterRead(readFile(root / std::to_string(entry.fileId), entry.offset, entry.length));
+}
+
+bool FileRepository::writeObject(const Object &obj) {
+    if (!ready) throw Exception("Tried working with uninitialized repo!");
+    auto tmp = filters.filterWrite(Serialize::serialize(obj));
+    {
+        std::unique_lock lockW(writeCacheLock);
+        writeCacheSize += tmp.size();
+        writeCache[obj.id] = std::move(tmp);
+
+        // If we have reached the target file size, flush the cache
+        if (writeCacheSize >= writeCacheMax) {
+            flushWriteCache(std::move(lockW));
+        }
+    }
+    return true;
+}
+
+void FileRepository::flushWriteCache(std::unique_lock<std::mutex> &&lockW) {
+    if (writeCache.empty()) {
+        lockW.unlock();
+        return;
+    }
+
+    // Swap the cache for a new one and unlock the mutex so other threads can continue working
+    decltype(writeCache) objs;
+    std::swap(writeCache, objs);
+    writeCacheSize = 0;
+
+    decltype(maxFileId) currentFileId;
+    {
+        std::lock_guard lockI(repoLock);
+        currentFileId = maxFileId;
+        maxFileId++;
+    }
+    lockW.unlock();
+
+    unsigned long long offset = 0;
+    std::ofstream ofstream(root / std::to_string(currentFileId), std::ios::binary | std::ios::trunc | std::ios::out);
+
+    for (auto &i: objs) {
+        {
+            std::lock_guard lockI(repoLock);
+            offsetIndex.emplace(i.first, OffsetEntry(currentFileId, offset, i.second.size()));
+        }
+        offset += i.second.size();
+        ofstream.rdbuf()->sputn(i.second.data(), i.second.size());
+    }
+}
+
+bool FileRepository::putObject(const Object &obj) {
+    // Put the object into index, and then write it to the storage
+    {
+        std::lock_guard lock(repoLock);
+        keyIndex[obj.type][obj.getKey()] = obj.id;
+    }
+    writeObject(obj);
+    return true;
+}
+
+bool FileRepository::deleteObject(const Object &obj) {
+    if (!ready) throw Exception("Tried working with uninitialized repo!");
+    throw Exception("Deletion not implemented!");
+}
+
+std::vector<char> FileRepository::readFile(const std::filesystem::path &file, unsigned long long offset, unsigned long long size) const {
+    if (size > absoluteMaxFileLimit) throw Exception("Tried to read " + std::to_string(size) +
+                                                     " bytes from " + file.u8string() +
+                                                     " which is more than absoluteMaxFileLimit");
+
+    std::ifstream ifstream(file, std::ios::binary | std::ios::in);
+    if (!ifstream.is_open()) throw Exception("Can't open file " + file.u8string() + " for reading!");
+
+    std::vector<char> buf(size);
+
+    if (ifstream.rdbuf()->pubseekpos(offset) == std::streampos(std::streamoff(-1))) throw Exception("Unexpected end of file " + file.u8string());
+    if (ifstream.rdbuf()->sgetn(buf.data(), size) != size) throw Exception("Unexpected end of file " + file.u8string());
+
+    return buf;
+}
+
+std::vector<char> FileRepository::readFile(const std::filesystem::path &file) const {
+    if (!std::filesystem::is_regular_file(file)) throw Exception("File " + file.u8string() + " is not a regular file!");
+    auto fileSize = std::filesystem::file_size(file);
+    if (fileSize == 0) return {};
+    return readFile(file, 0, fileSize);
+}
+
+bool FileRepository::writeFile(const std::filesystem::path &file, const std::vector<char> &data) {
+    std::ofstream ofstream(file, std::ios::binary | std::ios::trunc | std::ios::out);
+    if (!ofstream.is_open()) throw Exception("Can't open file " + file.u8string() + " for writing!");
+
+    if (ofstream.rdbuf()->sputn(data.data(), data.size()) != data.size())
+        throw Exception("Couldn't write all the data for " + file.u8string());
+    return true;
+}
+
+std::vector<char> FileRepository::getObject(Object::ObjectType type, const std::string &key) const {
+    return getObject(getObjectId(type, key));
+}
+
+Object::idType FileRepository::getObjectId(Object::ObjectType type, const std::string &key) const {
+    std::lock_guard lock(repoLock);
+    if (keyIndex.count(type) == 0) throw Exception("No objects of requested type!");
+    return keyIndex.at(type).at(key);
+}
+
+std::vector<std::pair<std::string, Object::idType>> FileRepository::getObjects(Object::ObjectType type) const {
+    std::lock_guard lock(repoLock);
+    std::vector<std::pair<std::string, Object::idType>> out;
+    if (keyIndex.count(type) == 0) return {};
+    for (auto const &i: keyIndex.at(type))
+        out.emplace_back(i);
+    return out;
+}
+
+bool FileRepository::exists(Object::ObjectType type, const std::string &key) const {
+    std::lock_guard lock(repoLock);
+    if (keyIndex.count(type) == 0) return false;
+    return keyIndex.at(type).count(key) > 0;
+}
+
+Object::idType FileRepository::getId() {
+    std::lock_guard lock(repoLock);
+    return largestUnusedId++;
+}
+
+FileRepository::OffsetEntry::OffsetEntry(std::vector<char, std::allocator<char>>::const_iterator &in, const std::vector<char, std::allocator<char>>::const_iterator &end)
+    : fileId(Serialize::deserialize<decltype(fileId)>(in, end)),
+      offset(Serialize::deserialize<decltype(offset)>(in, end)),
+      length(Serialize::deserialize<decltype(length)>(in, end)) {
+}
+
+void FileRepository::OffsetEntry::serialize(std::vector<char> &out) const {
+    Serialize::serialize(fileId, out);
+    Serialize::serialize(offset, out);
+    Serialize::serialize(length, out);
+}
+
+FileRepository::OffsetEntry::OffsetEntry(unsigned long long int fileId, unsigned long long int offset, unsigned long long int length)
+    : fileId(fileId), offset(offset), length(length) {}
+
+bool FileRepository::clearCache(Object::ObjectType type) {
+    keyIndex[type] = {};
+    return true;
+}
+
+bool FileRepository::addToCache(const Object &obj) {
+    {
+        std::unique_lock lock(repoLock);
+        if (offsetIndex.count(obj.id) == 0)
+            throw Exception("Object with id " + std::to_string(obj.id) + " doesn't exist!");
+    }
+    {
+        std::lock_guard lock(repoLock);
+        keyIndex[obj.type][obj.getKey()] = obj.id;
+    }
+
+    return true;
+}

src/repo/srcs/Object.cpp

@@ -0,0 +1,21 @@
+//
+// Created by Stepan Usatiuk on 14.04.2023.
+//
+
+#include "../includes/Object.h"
+
+#include "../includes/Serialize.h"
+
+Object::Object(idType id, ObjectType type) : id(id), type(type) {}
+
+Object::Object(std::vector<char>::const_iterator &in, const std::vector<char>::const_iterator &end)
+    : id(Serialize::deserialize<idType>(in, end)),
+      type(Serialize::deserialize<ObjectType>(in, end)) {
+}
+
+void Object::serialize(std::vector<char> &out) const {
+    Serialize::serialize(id, out);
+    Serialize::serialize(type, out);
+}
+
+Object::~Object() = default;

src/repo/srcs/Repository.cpp

@@ -0,0 +1,12 @@
+//
+// Created by Stepan Usatiuk on 14.04.2023.
+//
+
+#include "../includes/Repository.h"
+Repository::~Repository() = default;
+
+Repository::Repository(Config config) : config(std::move(config)) {}
+
+const Config &Repository::getConfig() const {
+    return config;
+}

src/repo/srcs/objects/Archive.cpp

@@ -0,0 +1,35 @@
+//
+// Created by Stepan Usatiuk on 14.04.2023.
+//
+
+#include "../../includes/objects/Archive.h"
+
+#include "../../../utils/includes/Exception.h"
+#include "../../includes/Serialize.h"
+
+Archive::Archive(Object::idType id, std::string name, unsigned long long mtime, std::vector<idType> files, bool full)
+    : Object(id, ObjectType::Archive), name(name), mtime(mtime), files(files), isFull(full) {}
+
+Archive::Archive(std::vector<char>::const_iterator &in, const std::vector<char>::const_iterator &end)
+    : Object(in, end),
+      name(Serialize::deserialize<std::string>(in, end)),
+      mtime(Serialize::deserialize<unsigned long long>(in, end)),
+      files(Serialize::deserialize<std::remove_const<decltype(files)>::type>(in, end)),
+      isFull(Serialize::deserialize<bool>(in, end)) {
+    if (type != ObjectType::Archive) throw Exception("Type mismatch for Archive!");
+    auto filesN = Serialize::deserialize<decltype(files.size())>(in, end);
+    if (files.size() != filesN) throw Exception("Number of files recorded doesn't match the number of files read!");
+}
+
+void Archive::serialize(std::vector<char> &out) const {
+    Object::serialize(out);
+    Serialize::serialize(name, out);
+    Serialize::serialize(mtime, out);
+    Serialize::serialize(files, out);
+    Serialize::serialize(isFull, out);
+    Serialize::serialize(files.size(), out);
+}
+
+std::string Archive::getKey() const {
+    return name;
+}

src/repo/srcs/objects/Chunk.cpp

@@ -0,0 +1,30 @@
+//
+// Created by Stepan Usatiuk on 14.04.2023.
+//
+
+#include "../../includes/objects/Chunk.h"
+
+#include "../../../utils/includes/Exception.h"
+#include "../../includes/Serialize.h"
+
+Chunk::Chunk(idType id, std::string md5, std::vector<char> data) : Object(id, ObjectType::Chunk), data(std::move(data)), md5(std::move(md5)), length(this->data.size()) {}
+
+Chunk::Chunk(std::vector<char>::const_iterator &in, const std::vector<char>::const_iterator &end)
+    : Object(in, end),
+      md5(Serialize::deserialize<std::remove_const<decltype(md5)>::type>(in, end)),
+      data(Serialize::deserialize<std::remove_const<decltype(data)>::type>(in, end)),
+      length(Serialize::deserialize<std::remove_const<decltype(length)>::type>(in, end)) {
+    if (type != ObjectType::Chunk) throw Exception("Type mismatch for Chunk!");
+    if (length != data.size()) throw Exception("Recorded length and actual length don't match for Chunk!");
+}
+
+void Chunk::serialize(std::vector<char> &out) const {
+    Object::serialize(out);
+    Serialize::serialize(md5, out);
+    Serialize::serialize(data, out);
+    Serialize::serialize(length, out);
+}
+
+std::string Chunk::getKey() const {
+    return md5;
+}

src/repo/srcs/objects/File.cpp

@@ -0,0 +1,84 @@
+//
+// Created by Stepan Usatiuk on 14.04.2023.
+//
+
+#include "../../includes/objects/File.h"
+
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "../../../utils/includes/Exception.h"
+#include "../../includes/Serialize.h"
+
+File::File(Object::idType id, std::string name, unsigned long long bytes, unsigned long long mtime, std::string md5, std::vector<idType> chunks, Type fileType)
+    : Object(id, ObjectType::File), name(name), bytes(bytes), mtime(mtime), md5(md5), fileType(fileType), chunks(chunks) {}
+
+File::File(std::vector<char>::const_iterator &in, const std::vector<char>::const_iterator &end)
+    : Object(in, end),
+      name(Serialize::deserialize<std::string>(in, end)),
+      bytes(Serialize::deserialize<unsigned long long>(in, end)),
+      mtime(Serialize::deserialize<unsigned long long>(in, end)),
+      md5(Serialize::deserialize<std::remove_const<decltype(md5)>::type>(in, end)),
+      fileType(Serialize::deserialize<std::remove_const<decltype(fileType)>::type>(in, end)),
+      chunks(Serialize::deserialize<std::remove_const<decltype(chunks)>::type>(in, end)) {
+    if (type != ObjectType::File) throw Exception("Type mismatch for File!");
+}
+
+void File::serialize(std::vector<char> &out) const {
+    Object::serialize(out);
+    Serialize::serialize(name, out);
+    Serialize::serialize(bytes, out);
+    Serialize::serialize(mtime, out);
+    Serialize::serialize(md5, out);
+    Serialize::serialize(fileType, out);
+    Serialize::serialize(chunks, out);
+}
+
+std::string File::getKey() const {
+    return name;
+}
+
+File::Type File::getFileType(const std::filesystem::path &p) {
+    if (std::filesystem::is_symlink(p)) return Type::Symlink;
+    if (std::filesystem::is_directory(p)) return Type::Directory;
+    if (std::filesystem::is_regular_file(p)) return Type::Normal;
+    throw Exception("Unsupported file type! " + p.u8string());
+}
+
+std::vector<char> File::getFileContents(const std::filesystem::path &p) {
+    auto type = getFileType(p);
+    if (type == Type::Normal) throw Exception(p.u8string() + " is a normal file!");
+    if (type == Type::Directory) {
+        return {};
+    }
+    if (type == Type::Symlink) {
+        auto target = std::filesystem::read_symlink(p).u8string();
+        return {target.begin(), target.end()};
+    }
+    throw Exception("Error with file " + p.u8string());
+}
+
+unsigned long long File::getFileMtime(const std::filesystem::path &p) {
+    auto type = getFileType(p);
+    if (type == Type::Normal || type == Type::Directory)
+        return static_cast<const unsigned long long int>(std::chrono::duration_cast<std::chrono::seconds>(std::filesystem::last_write_time(p).time_since_epoch()).count());
+    else if (type == Type::Symlink) {
+        auto path = p.u8string();
+        struct stat sb;
+        if (lstat(path.c_str(), &sb) != 0) throw Exception("Error reading mtime for " + p.u8string());
+#ifdef __APPLE__
+        return sb.st_mtimespec.tv_sec;
+#else
+        return sb.st_mtime;
+#endif
+    }
+    throw Exception("Error with file " + p.u8string());
+}
+
+unsigned long long File::getFileSize(const std::filesystem::path &p) {
+    auto type = getFileType(p);
+    if (type == Type::Normal) return std::filesystem::file_size(p);
+    else
+        return getFileContents(p).size();
+}

src/repo/srcs/objects/FileBuffer.cpp

@@ -0,0 +1,51 @@
+//
+// Created by Stepan Usatiuk on 23.04.2023.
+//
+
+#include "../../includes/objects/FileBuffer.h"
+
+#include "../../includes/Serialize.h"
+
+FileBuffer::FileBuffer(const Repository *repo, Object::idType fileId) : repo(repo), file(Serialize::deserialize<File>(repo->getObject(fileId))), chunksQueue() {
+    for (auto const &id: file.chunks) chunksQueue.emplace(id);
+};
+
+int FileBuffer::sync() {
+    return 0;
+}
+
+std::streamsize FileBuffer::xsgetn(char *s, std::streamsize countr) {
+    if (underflow() == std::char_traits<char>::eof()) return 0;
+    for (int i = 0; i < countr; i++) {
+        auto c = uflow();
+        if (c != traits_type::eof()) {
+            s[i] = traits_type::to_char_type(c);
+        } else
+            return i;
+    }
+    return countr;
+}
+
+int FileBuffer::uflow() {
+    auto out = underflow();
+    if (out != traits_type::eof())
+        curGetBufPos++;
+    return out;
+}
+
+int FileBuffer::underflow() {
+    if (getBuf.empty() || curGetBufPos == getBuf.size()) {
+        if (chunksQueue.empty()) return traits_type::eof();
+        else {
+            auto chunk = Serialize::deserialize<Chunk>(repo->getObject(chunksQueue.front()));
+            getBuf = chunk.data;
+            chunksQueue.pop();
+            curGetBufPos = 0;
+        }
+    }
+
+    if (!getBuf.empty())
+        return traits_type::to_int_type(getBuf[curGetBufPos]);
+    else
+        return traits_type::eof();
+}