diff --git a/src/commands/srcs/CommandRun.cpp b/src/commands/srcs/CommandRun.cpp index 31d499c..25ae23b 100644 --- a/src/commands/srcs/CommandRun.cpp +++ b/src/commands/srcs/CommandRun.cpp @@ -34,34 +34,6 @@ void CommandRun::run(Context ctx) { RunnerStats runnerStats;///< Backup target metrics std::filesystem::path from = ctx.repo->getConfig().getStr("from");///< Directory to back up from - bool fullBackup = ctx.repo->getConfig().getStr("type") == "full"; - if (fullBackup) { - ctx.logger->write("Backup is full because of the config\n", 1); - } - /// For progtest task compliance - if (!fullBackup) { - /// If it's time for full backup as per config, force it - auto per = ctx.repo->getConfig().getInt("full-period"); - auto list = ctx.repo->getObjects(Object::ObjectType::Archive); - std::sort(list.begin(), list.end(), [](const auto &l, const auto &r) { return l.second > r.second; }); - int lastInc = 0; - for (auto const &a: list) { - auto archiveO = Serialize::deserialize(ctx.repo->getObject(a.second)); - if (!archiveO.isFull) { - lastInc++; - continue; - } else - break; - } - if (lastInc >= per) { - fullBackup = true; - ctx.logger->write("Backup is full because of the interval\n", 1); - } - if (list.size() == 0) { - fullBackup = true; - ctx.logger->write("Backup is full because there are no backups\n", 1); - } - } /// Worker callback, bound to the local workerStats variable workerStatsFunction workerCallback = [&](unsigned long long bytesWritten, unsigned long long bytesSkipped, unsigned long long filesWritten) { @@ -73,14 +45,6 @@ void CommandRun::run(Context ctx) { /// Function to safely add new file ids to `files` std::function addFile = [&](Object::idType id) {std::lock_guard lock(filesLock); files.emplace_back(id); }; - /// Technically the progtest task says that only the files from the last backup should be compared against... - std::map prevArchiveFiles; - { - auto prevArchiveFilesList = ctx.repo->getObjects(Object::ObjectType::File); - prevArchiveFiles = {prevArchiveFilesList.begin(), prevArchiveFilesList.end()}; - } - ctx.repo->clearCache(Object::ObjectType::File); - { /// Calculate the average speed of backup RunningDiffAverage avg( @@ -123,32 +87,24 @@ void CommandRun::run(Context ctx) { }; /// Task to process an individual file in the backup - std::function processFile; - /// If it's a full backup, just save the file, otherwise re-chunk it only if it's changed - if (fullBackup) - processFile = - [&, this](const std::filesystem::path &p) { - saveFile(p, p.lexically_relative(from).u8string()); - }; - else - processFile = - [&, this](const std::filesystem::path &p) { - auto relPath = p.lexically_relative(from).u8string(); + std::function processFile = + [&, this](const std::filesystem::path &p) { + auto relPath = p.lexically_relative(from).u8string(); - if (prevArchiveFiles.count(relPath) != 0) { - File repoFile = Serialize::deserialize(ctx.repo->getObject(prevArchiveFiles.at(relPath))); - if (!changeDetector.check({repoFile, ctx.repo}, {p, from})) { - addFile(repoFile.id); - ctx.repo->addToCache(repoFile); - progress.print("Skipped: " + relPath, 1); - runnerStats.filesSkipped++; - return; - } + if (ctx.repo->exists(Object::ObjectType::File, relPath) != 0) { + File repoFile = Serialize::deserialize(ctx.repo->getObject(Object::ObjectType::File, relPath)); + if (!changeDetector.check({repoFile, ctx.repo}, {p, from})) { + addFile(repoFile.id); + ctx.repo->addToCache(repoFile); + progress.print("Skipped: " + relPath, 1); + runnerStats.filesSkipped++; + return; } + } - saveFile(p, relPath); - return; - }; + saveFile(p, relPath); + return; + }; /// Start the backup with the root directory and empty ignore list threadPool.push([&]() { @@ -178,7 +134,7 @@ void CommandRun::run(Context ctx) { s << std::put_time(ltime, "%d-%m-%Y %H-%M-%S"); /// Avoid archive name collisions while (ctx.repo->exists(Object::ObjectType::Archive, s.str())) s << "N"; - Archive a(ctx.repo->getId(), s.str(), time, files, fullBackup); + Archive a(ctx.repo->getId(), s.str(), time, files); ctx.repo->putObject(a); } @@ -210,7 +166,7 @@ Object::idType CommandRun::backupChunkFile(const std::filesystem::path &orig, co if (Signals::shouldQuit) break; Object::idType chunkId; - if (ctx.repo->getConfig().getStr("dedup") == "on" && ctx.repo->exists(Object::ObjectType::Chunk, chunkp.first)) { + if (ctx.repo->exists(Object::ObjectType::Chunk, chunkp.first)) { /// If the chunk already exists, reuse it chunkId = ctx.repo->getObjectId(Object::ObjectType::Chunk, chunkp.first); callback(0, chunkp.second.size(), 0); diff --git a/src/repo/includes/objects/Archive.h b/src/repo/includes/objects/Archive.h index d0186cf..cde078c 100644 --- a/src/repo/includes/objects/Archive.h +++ b/src/repo/includes/objects/Archive.h @@ -12,7 +12,7 @@ /// Object representing a backup class Archive : public Object { public: - Archive(Object::idType id, std::string name, unsigned long long mtime, std::vector files, bool full = false); + Archive(Object::idType id, std::string name, unsigned long long mtime, std::vector files); /// \copydoc Object::serialize Archive(std::vector::const_iterator &in, const std::vector::const_iterator &end); @@ -25,7 +25,6 @@ public: const std::string name; ///< Archive name const unsigned long long mtime; ///< Time of creation const std::vector files;///< List of ids of File objects in the Archive - const bool isFull = false; ///< Whether this was a full archive }; diff --git a/src/repo/srcs/objects/Archive.cpp b/src/repo/srcs/objects/Archive.cpp index e7bc7c7..97f50db 100644 --- a/src/repo/srcs/objects/Archive.cpp +++ b/src/repo/srcs/objects/Archive.cpp @@ -7,15 +7,14 @@ #include "../../../utils/includes/Exception.h" #include "../../includes/Serialize.h" -Archive::Archive(Object::idType id, std::string name, unsigned long long mtime, std::vector files, bool full) - : Object(id, ObjectType::Archive), name(name), mtime(mtime), files(files), isFull(full) {} +Archive::Archive(Object::idType id, std::string name, unsigned long long mtime, std::vector files) + : Object(id, ObjectType::Archive), name(name), mtime(mtime), files(files) {} Archive::Archive(std::vector::const_iterator &in, const std::vector::const_iterator &end) : Object(in, end), name(Serialize::deserialize(in, end)), mtime(Serialize::deserialize(in, end)), - files(Serialize::deserialize::type>(in, end)), - isFull(Serialize::deserialize(in, end)) { + files(Serialize::deserialize::type>(in, end)) { if (type != ObjectType::Archive) throw Exception("Type mismatch for Archive!"); auto filesN = Serialize::deserialize(in, end); if (files.size() != filesN) throw Exception("Number of files recorded doesn't match the number of files read!"); @@ -26,7 +25,6 @@ void Archive::serialize(std::vector &out) const { Serialize::serialize(name, out); Serialize::serialize(mtime, out); Serialize::serialize(files, out); - Serialize::serialize(isFull, out); Serialize::serialize(files.size(), out); } diff --git a/src/utils/includes/Config.h b/src/utils/includes/Config.h index 8fe99e3..f9dec66 100644 --- a/src/utils/includes/Config.h +++ b/src/utils/includes/Config.h @@ -89,7 +89,6 @@ public: {"repo", {std::nullopt, KeyType::STRING, false, "Repository root"}}, {"to", {std::nullopt, KeyType::STRING, false, "Destination of restore"}}, {"from", {std::nullopt, KeyType::STRING, true, "Backed up folder"}}, - {"type", {"normal", KeyType::STRING, false, "Type of archive"}}, {"aid", {std::nullopt, KeyType::INT, false, "ID of archive to restore/compare to"}}, {"aid2", {std::nullopt, KeyType::INT, false, "ID of archive to compare with"}}, {"threads", {std::nullopt, KeyType::INT, false, "Number of threads to use"}}, @@ -101,10 +100,8 @@ public: {"chunker-max", {"4096", KeyType::INT, true, "Max chunk size in KB"}}, {"chunker-mask", {"20", KeyType::INT, true, "Chunker hash bit mask (mask of n bits results in average chunk size of 2^n bytes)"}}, {"repo-target", {"128", KeyType::INT, true, "Target size of files for FileRepository"}}, - {"full-period", {"2", KeyType::INT, true, "Interval between forced full backups"}}, {"progress", {"pretty", KeyType::STRING, false, "How to print progress (simple, pretty, none)"}}, {"verbose", {"1", KeyType::INT, false, "Message verbosity (0 - error, 1 - info, -1 - quiet)"}}, - {"dedup", {"on", KeyType::STRING, true, "Turns deduplication on/off"}}, {"change-detectors", {"type,size,etime", KeyType::LIST, true, "Change detectors to use (in order)"}}, {"diff-mode", {"normal", KeyType::STRING, false, "Diff mode (file or normal)"}}, }; diff --git a/tests/clitests/scripts/backup.sh b/tests/clitests/scripts/backup.sh index 2ffee4a..f7d173e 100755 --- a/tests/clitests/scripts/backup.sh +++ b/tests/clitests/scripts/backup.sh @@ -49,7 +49,7 @@ cat testdata/7/b >> testdata/8/b echo "Data created" -if ! $CMD init --repo testdir/to1 --compression zlib --compression-level 4 --encryption aes --password asdff --salt e --full-period 999; then +if ! $CMD init --repo testdir/to1 --compression zlib --compression-level 4 --encryption aes --password asdff --salt e; then echo "Error creating repo!" exit 1 fi @@ -107,7 +107,7 @@ echo "Backup 6 ok" OUT=$($CMD run --from testdata/7 --repo testdir/to1 --password asdff --progress simple --verbose 1) echo "$OUT" -if ! ( ( echo "$OUT" | grep -q 'Copied: a' ) && ( echo "$OUT" | grep -q 'Skipped: aa' ) && ( echo "$OUT" | grep -q 'Skipped: b' ) && ( echo "$OUT" | grep -q 'Skipped: c' ) ); then +if ! ( ( echo "$OUT" | grep -q 'Skipped: a' ) && ( echo "$OUT" | grep -q 'Skipped: aa' ) && ( echo "$OUT" | grep -q 'Skipped: b' ) && ( echo "$OUT" | grep -q 'Skipped: c' ) ); then echo "Error backing up 7 dir!" exit 1 fi @@ -212,7 +212,7 @@ echo "Repo created" OUT=$($CMD run --from testdata/1 --repo testdir/to2 --password asdff --progress simple --verbose 1) echo "$OUT" -if ! ( (echo "$OUT" | grep -q 'Backup is full because there are no backups')&&( echo "$OUT" | grep -q 'Copied: a' ) && ( echo "$OUT" | grep -q 'Copied: aa' ) && ( echo "$OUT" | grep -q 'Copied: b' ) && ( echo "$OUT" | grep -q 'Copied: c' ) ); then +if ! ( ( echo "$OUT" | grep -q 'Copied: a' ) && ( echo "$OUT" | grep -q 'Copied: aa' ) && ( echo "$OUT" | grep -q 'Copied: b' ) && ( echo "$OUT" | grep -q 'Copied: c' ) ); then echo "Error backing up 1 dir!" exit 1 fi @@ -220,7 +220,7 @@ echo "Backup 1 ok" OUT=$($CMD run --from testdata/1 --repo testdir/to2 --password asdff --progress simple --verbose 1) echo "$OUT" -if ! ( ! ( echo "$OUT" | grep -q 'Backup is full' ) && ( echo "$OUT" | grep -q 'Skipped: a' ) && ( echo "$OUT" | grep -q 'Skipped: aa' ) && ( echo "$OUT" | grep -q 'Skipped: b' ) && ( echo "$OUT" | grep -q 'Skipped: c' )); then +if ! ( ( echo "$OUT" | grep -q 'Skipped: a' ) && ( echo "$OUT" | grep -q 'Skipped: aa' ) && ( echo "$OUT" | grep -q 'Skipped: b' ) && ( echo "$OUT" | grep -q 'Skipped: c' )); then echo "Error backing up 2 dir!" exit 1 fi @@ -228,7 +228,7 @@ echo "Backup 2 ok" OUT=$($CMD run --from testdata/1 --repo testdir/to2 --password asdff --progress simple --verbose 1) echo "$OUT" -if ! ( ! ( echo "$OUT" | grep -q 'Backup is full' )&& ( echo "$OUT" | grep -q 'Skipped: a' ) && ( echo "$OUT" | grep -q 'Skipped: aa' ) && ( echo "$OUT" | grep -q 'Skipped: b' ) && ( echo "$OUT" | grep -q 'Skipped: c' ) ); then +if ! ( ( echo "$OUT" | grep -q 'Skipped: a' ) && ( echo "$OUT" | grep -q 'Skipped: aa' ) && ( echo "$OUT" | grep -q 'Skipped: b' ) && ( echo "$OUT" | grep -q 'Skipped: c' ) ); then echo "Error backing up 3 dir!" exit 1 fi @@ -236,7 +236,7 @@ echo "Backup 3 ok" OUT=$($CMD run --from testdata/1 --repo testdir/to2 --password asdff --progress simple --verbose 1) echo "$OUT" -if ! ( ( echo "$OUT" | grep -q 'Backup is full because of the interval') &&( echo "$OUT" | grep -q 'Copied: a' ) && ( echo "$OUT" | grep -q 'Copied: aa' ) && ( echo "$OUT" | grep -q 'Copied: b' ) && ( echo "$OUT" | grep -q 'Copied: c' )); then +if ! ( ( echo "$OUT" | grep -q 'Skipped: a' ) && ( echo "$OUT" | grep -q 'Skipped: aa' ) && ( echo "$OUT" | grep -q 'Skipped: b' ) && ( echo "$OUT" | grep -q 'Skipped: c' )); then echo "Error backing up 4 dir!" exit 1 fi @@ -244,7 +244,7 @@ echo "Backup 4 ok" OUT=$($CMD run --from testdata/1 --repo testdir/to2 --password asdff --progress simple --verbose 1) echo "$OUT" -if ! ( ! ( echo "$OUT" | grep -q 'Backup is full' ) && ( echo "$OUT" | grep -q 'Skipped: a' ) && ( echo "$OUT" | grep -q 'Skipped: aa' ) && ( echo "$OUT" | grep -q 'Skipped: b' ) && ( echo "$OUT" | grep -q 'Skipped: c' ) ); then +if ! ( ( echo "$OUT" | grep -q 'Skipped: a' ) && ( echo "$OUT" | grep -q 'Skipped: aa' ) && ( echo "$OUT" | grep -q 'Skipped: b' ) && ( echo "$OUT" | grep -q 'Skipped: c' ) ); then echo "Error backing up 5 dir!" exit 1 fi @@ -252,7 +252,7 @@ echo "Backup 5 ok" OUT=$($CMD run --from testdata/1 --repo testdir/to2 --password asdff --progress simple --verbose 1) echo "$OUT" -if ! ( ! ( echo "$OUT" | grep -q 'Backup is full' ) && ( echo "$OUT" | grep -q 'Skipped: a' ) && ( echo "$OUT" | grep -q 'Skipped: aa' ) && ( echo "$OUT" | grep -q 'Skipped: b' ) && ( echo "$OUT" | grep -q 'Skipped: c' ) ); then +if ! ( ( echo "$OUT" | grep -q 'Skipped: a' ) && ( echo "$OUT" | grep -q 'Skipped: aa' ) && ( echo "$OUT" | grep -q 'Skipped: b' ) && ( echo "$OUT" | grep -q 'Skipped: c' ) ); then echo "Error backing up 6 dir!" exit 1 fi @@ -260,15 +260,15 @@ echo "Backup 6 ok" OUT=$($CMD run --from testdata/1 --repo testdir/to2 --password asdff --progress simple --verbose 1) echo "$OUT" -if ! ( ( echo "$OUT" | grep -q 'Backup is full because of the interval' ) &&( echo "$OUT" | grep -q 'Copied: a' ) && ( echo "$OUT" | grep -q 'Copied: aa' ) && ( echo "$OUT" | grep -q 'Copied: b' ) && ( echo "$OUT" | grep -q 'Copied: c' ) ); then +if ! ( ( echo "$OUT" | grep -q 'Skipped: a' ) && ( echo "$OUT" | grep -q 'Skipped: aa' ) && ( echo "$OUT" | grep -q 'Skipped: b' ) && ( echo "$OUT" | grep -q 'Skipped: c' ) ); then echo "Error backing up 7 dir!" exit 1 fi echo "Backup 7 ok" -OUT=$($CMD run --from testdata/1 --repo testdir/to2 --password asdff --progress simple --verbose 1 --type full) +OUT=$($CMD run --from testdata/1 --repo testdir/to2 --password asdff --progress simple --verbose 1) echo "$OUT" -if ! ( ( echo "$OUT" | grep -q 'Backup is full because of the config') &&( echo "$OUT" | grep -q 'Copied: a' ) && ( echo "$OUT" | grep -q 'Copied: aa' ) && ( echo "$OUT" | grep -q 'Copied: b' ) && ( echo "$OUT" | grep -q 'Copied: c' )); then +if ! ( ( echo "$OUT" | grep -q 'Skipped: a' ) && ( echo "$OUT" | grep -q 'Skipped: aa' ) && ( echo "$OUT" | grep -q 'Skipped: b' ) && ( echo "$OUT" | grep -q 'Skipped: c' )); then echo "Error backing up 8 dir!" exit 1 fi @@ -276,7 +276,7 @@ echo "Backup 8 ok" OUT=$($CMD run --from testdata/1 --repo testdir/to2 --password asdff --progress simple --verbose 1) echo "$OUT" -if ! ( ! ( echo "$OUT" | grep -q 'Backup is full' ) && ( echo "$OUT" | grep -q 'Skipped: a' ) && ( echo "$OUT" | grep -q 'Skipped: aa' ) && ( echo "$OUT" | grep -q 'Skipped: b' ) && ( echo "$OUT" | grep -q 'Skipped: c' ) ); then +if ! ( ( echo "$OUT" | grep -q 'Skipped: a' ) && ( echo "$OUT" | grep -q 'Skipped: aa' ) && ( echo "$OUT" | grep -q 'Skipped: b' ) && ( echo "$OUT" | grep -q 'Skipped: c' ) ); then echo "Error backing up 9 dir!" exit 1 fi @@ -284,7 +284,7 @@ echo "Backup 9 ok" OUT=$($CMD run --from testdata/1 --repo testdir/to2 --password asdff --progress simple --verbose 1) echo "$OUT" -if ! ( ! ( echo "$OUT" | grep -q 'Backup is full' ) && ( echo "$OUT" | grep -q 'Skipped: a' ) && ( echo "$OUT" | grep -q 'Skipped: aa' ) && ( echo "$OUT" | grep -q 'Skipped: b' ) && ( echo "$OUT" | grep -q 'Skipped: c' ) ); then +if ! ( ( echo "$OUT" | grep -q 'Skipped: a' ) && ( echo "$OUT" | grep -q 'Skipped: aa' ) && ( echo "$OUT" | grep -q 'Skipped: b' ) && ( echo "$OUT" | grep -q 'Skipped: c' ) ); then echo "Error backing up 10 dir!" exit 1 fi @@ -292,7 +292,7 @@ echo "Backup 10 ok" OUT=$($CMD run --from testdata/1 --repo testdir/to2 --password asdff --progress simple --verbose 1) echo "$OUT" -if ! ( ( echo "$OUT" | grep -q 'Backup is full because of the interval' ) &&( echo "$OUT" | grep -q 'Copied: a' ) && ( echo "$OUT" | grep -q 'Copied: aa' ) && ( echo "$OUT" | grep -q 'Copied: b' ) && ( echo "$OUT" | grep -q 'Copied: c' ) ); then +if ! ( ( echo "$OUT" | grep -q 'Skipped: a' ) && ( echo "$OUT" | grep -q 'Skipped: aa' ) && ( echo "$OUT" | grep -q 'Skipped: b' ) && ( echo "$OUT" | grep -q 'Skipped: c' ) ); then echo "Error backing up 11 dir!" exit 1 fi diff --git a/tests/clitests/scripts/diff.sh b/tests/clitests/scripts/diff.sh index ba9051e..3e6601f 100755 --- a/tests/clitests/scripts/diff.sh +++ b/tests/clitests/scripts/diff.sh @@ -60,7 +60,7 @@ echo "testtestasdf9uuu" > testdata/4/filexd echo "Data created" -if ! $CMD init --repo testdir/to1 --compression zlib --compression-level 4 --encryption aes --password asdff --salt e --full-period 999; then +if ! $CMD init --repo testdir/to1 --compression zlib --compression-level 4 --encryption aes --password asdff --salt e; then echo "Error creating repo!" exit 1 fi diff --git a/tests/clitests/scripts/ignore.sh b/tests/clitests/scripts/ignore.sh index 5c86a9e..ade1aae 100755 --- a/tests/clitests/scripts/ignore.sh +++ b/tests/clitests/scripts/ignore.sh @@ -32,7 +32,7 @@ touch testdata/1/emptyi/.nobackup echo "Data created" -if ! $CMD init --repo testdir/to1 --compression zlib --compression-level 4 --encryption aes --password asdff --salt e --full-period 999; then +if ! $CMD init --repo testdir/to1 --compression zlib --compression-level 4 --encryption aes --password asdff --salt e; then echo "Error creating repo!" exit 1 fi