mirror of
https://github.com/usatiuk/backup.git
synced 2025-10-26 09:27:48 +01:00
init
This commit is contained in:
67
.clang-format
Normal file
67
.clang-format
Normal file
@@ -0,0 +1,67 @@
|
|||||||
|
# Generated from CLion C/C++ Code Style settings
|
||||||
|
BasedOnStyle: LLVM
|
||||||
|
AccessModifierOffset: -4
|
||||||
|
AlignAfterOpenBracket: Align
|
||||||
|
AlignConsecutiveAssignments: None
|
||||||
|
AlignOperands: Align
|
||||||
|
AllowAllArgumentsOnNextLine: false
|
||||||
|
AllowAllConstructorInitializersOnNextLine: false
|
||||||
|
AllowAllParametersOfDeclarationOnNextLine: false
|
||||||
|
AllowShortBlocksOnASingleLine: Always
|
||||||
|
AllowShortCaseLabelsOnASingleLine: false
|
||||||
|
AllowShortFunctionsOnASingleLine: All
|
||||||
|
AllowShortIfStatementsOnASingleLine: Always
|
||||||
|
AllowShortLambdasOnASingleLine: All
|
||||||
|
AllowShortLoopsOnASingleLine: true
|
||||||
|
AlwaysBreakAfterReturnType: None
|
||||||
|
AlwaysBreakTemplateDeclarations: Yes
|
||||||
|
SeparateDefinitionBlocks: Always
|
||||||
|
BreakBeforeBraces: Custom
|
||||||
|
BraceWrapping:
|
||||||
|
AfterCaseLabel: false
|
||||||
|
AfterClass: false
|
||||||
|
AfterControlStatement: Never
|
||||||
|
AfterEnum: false
|
||||||
|
AfterFunction: false
|
||||||
|
AfterNamespace: false
|
||||||
|
AfterUnion: false
|
||||||
|
BeforeCatch: false
|
||||||
|
BeforeElse: false
|
||||||
|
IndentBraces: false
|
||||||
|
SplitEmptyFunction: false
|
||||||
|
SplitEmptyRecord: false
|
||||||
|
BreakBeforeBinaryOperators: None
|
||||||
|
BreakBeforeTernaryOperators: true
|
||||||
|
BreakConstructorInitializers: BeforeColon
|
||||||
|
BreakInheritanceList: BeforeColon
|
||||||
|
ColumnLimit: 0
|
||||||
|
CompactNamespaces: true
|
||||||
|
ContinuationIndentWidth: 8
|
||||||
|
IndentCaseLabels: true
|
||||||
|
IndentPPDirectives: None
|
||||||
|
IndentWidth: 4
|
||||||
|
KeepEmptyLinesAtTheStartOfBlocks: false
|
||||||
|
MaxEmptyLinesToKeep: 2
|
||||||
|
NamespaceIndentation: All
|
||||||
|
ObjCSpaceAfterProperty: false
|
||||||
|
ObjCSpaceBeforeProtocolList: true
|
||||||
|
PointerAlignment: Right
|
||||||
|
ReflowComments: false
|
||||||
|
SpaceAfterCStyleCast: true
|
||||||
|
SpaceAfterLogicalNot: false
|
||||||
|
SpaceAfterTemplateKeyword: false
|
||||||
|
SpaceBeforeAssignmentOperators: true
|
||||||
|
SpaceBeforeCpp11BracedList: false
|
||||||
|
SpaceBeforeCtorInitializerColon: true
|
||||||
|
SpaceBeforeInheritanceColon: true
|
||||||
|
SpaceBeforeParens: ControlStatements
|
||||||
|
SpaceBeforeRangeBasedForLoopColon: false
|
||||||
|
SpaceInEmptyParentheses: false
|
||||||
|
SpacesBeforeTrailingComments: 0
|
||||||
|
SpacesInAngles: false
|
||||||
|
SpacesInCStyleCastParentheses: false
|
||||||
|
SpacesInContainerLiterals: false
|
||||||
|
SpacesInParentheses: false
|
||||||
|
SpacesInSquareBrackets: false
|
||||||
|
TabWidth: 4
|
||||||
|
UseTab: Never
|
||||||
147
.clang-tidy
Normal file
147
.clang-tidy
Normal file
@@ -0,0 +1,147 @@
|
|||||||
|
# Generated from CLion Inspection settings
|
||||||
|
---
|
||||||
|
Checks: '-*,
|
||||||
|
bugprone-argument-comment,
|
||||||
|
bugprone-assert-side-effect,
|
||||||
|
bugprone-bad-signal-to-kill-thread,
|
||||||
|
bugprone-branch-clone,
|
||||||
|
bugprone-copy-constructor-init,
|
||||||
|
bugprone-dangling-handle,
|
||||||
|
bugprone-dynamic-static-initializers,
|
||||||
|
bugprone-fold-init-type,
|
||||||
|
bugprone-forward-declaration-namespace,
|
||||||
|
bugprone-forwarding-reference-overload,
|
||||||
|
bugprone-inaccurate-erase,
|
||||||
|
bugprone-incorrect-roundings,
|
||||||
|
bugprone-integer-division,
|
||||||
|
bugprone-lambda-function-name,
|
||||||
|
bugprone-macro-parentheses,
|
||||||
|
bugprone-macro-repeated-side-effects,
|
||||||
|
bugprone-misplaced-operator-in-strlen-in-alloc,
|
||||||
|
bugprone-misplaced-pointer-arithmetic-in-alloc,
|
||||||
|
bugprone-misplaced-widening-cast,
|
||||||
|
bugprone-move-forwarding-reference,
|
||||||
|
bugprone-multiple-statement-macro,
|
||||||
|
bugprone-no-escape,
|
||||||
|
bugprone-not-null-terminated-result,
|
||||||
|
bugprone-parent-virtual-call,
|
||||||
|
bugprone-posix-return,
|
||||||
|
bugprone-reserved-identifier,
|
||||||
|
bugprone-sizeof-container,
|
||||||
|
bugprone-sizeof-expression,
|
||||||
|
bugprone-spuriously-wake-up-functions,
|
||||||
|
bugprone-string-constructor,
|
||||||
|
bugprone-string-integer-assignment,
|
||||||
|
bugprone-string-literal-with-embedded-nul,
|
||||||
|
bugprone-suspicious-enum-usage,
|
||||||
|
bugprone-suspicious-include,
|
||||||
|
bugprone-suspicious-memory-comparison,
|
||||||
|
bugprone-suspicious-memset-usage,
|
||||||
|
bugprone-suspicious-missing-comma,
|
||||||
|
bugprone-suspicious-semicolon,
|
||||||
|
bugprone-suspicious-string-compare,
|
||||||
|
bugprone-swapped-arguments,
|
||||||
|
bugprone-terminating-continue,
|
||||||
|
bugprone-throw-keyword-missing,
|
||||||
|
bugprone-too-small-loop-variable,
|
||||||
|
bugprone-undefined-memory-manipulation,
|
||||||
|
bugprone-undelegated-constructor,
|
||||||
|
bugprone-unhandled-self-assignment,
|
||||||
|
bugprone-unused-raii,
|
||||||
|
bugprone-unused-return-value,
|
||||||
|
bugprone-use-after-move,
|
||||||
|
bugprone-virtual-near-miss,
|
||||||
|
cert-dcl21-cpp,
|
||||||
|
cert-dcl58-cpp,
|
||||||
|
cert-err34-c,
|
||||||
|
cert-err52-cpp,
|
||||||
|
cert-err60-cpp,
|
||||||
|
cert-flp30-c,
|
||||||
|
cert-msc50-cpp,
|
||||||
|
cert-msc51-cpp,
|
||||||
|
cert-str34-c,
|
||||||
|
cppcoreguidelines-interfaces-global-init,
|
||||||
|
cppcoreguidelines-narrowing-conversions,
|
||||||
|
cppcoreguidelines-pro-type-member-init,
|
||||||
|
cppcoreguidelines-pro-type-static-cast-downcast,
|
||||||
|
cppcoreguidelines-slicing,
|
||||||
|
google-default-arguments,
|
||||||
|
google-explicit-constructor,
|
||||||
|
google-runtime-operator,
|
||||||
|
hicpp-exception-baseclass,
|
||||||
|
hicpp-multiway-paths-covered,
|
||||||
|
misc-misplaced-const,
|
||||||
|
misc-new-delete-overloads,
|
||||||
|
misc-no-recursion,
|
||||||
|
misc-non-copyable-objects,
|
||||||
|
misc-throw-by-value-catch-by-reference,
|
||||||
|
misc-unconventional-assign-operator,
|
||||||
|
misc-uniqueptr-reset-release,
|
||||||
|
modernize-avoid-bind,
|
||||||
|
modernize-concat-nested-namespaces,
|
||||||
|
modernize-deprecated-headers,
|
||||||
|
modernize-deprecated-ios-base-aliases,
|
||||||
|
modernize-loop-convert,
|
||||||
|
modernize-make-shared,
|
||||||
|
modernize-make-unique,
|
||||||
|
modernize-pass-by-value,
|
||||||
|
modernize-raw-string-literal,
|
||||||
|
modernize-redundant-void-arg,
|
||||||
|
modernize-replace-auto-ptr,
|
||||||
|
modernize-replace-disallow-copy-and-assign-macro,
|
||||||
|
modernize-replace-random-shuffle,
|
||||||
|
modernize-return-braced-init-list,
|
||||||
|
modernize-shrink-to-fit,
|
||||||
|
modernize-unary-static-assert,
|
||||||
|
modernize-use-auto,
|
||||||
|
modernize-use-bool-literals,
|
||||||
|
modernize-use-emplace,
|
||||||
|
modernize-use-equals-default,
|
||||||
|
modernize-use-equals-delete,
|
||||||
|
modernize-use-nodiscard,
|
||||||
|
modernize-use-noexcept,
|
||||||
|
modernize-use-nullptr,
|
||||||
|
modernize-use-override,
|
||||||
|
modernize-use-transparent-functors,
|
||||||
|
modernize-use-uncaught-exceptions,
|
||||||
|
mpi-buffer-deref,
|
||||||
|
mpi-type-mismatch,
|
||||||
|
openmp-use-default-none,
|
||||||
|
performance-faster-string-find,
|
||||||
|
performance-for-range-copy,
|
||||||
|
performance-implicit-conversion-in-loop,
|
||||||
|
performance-inefficient-algorithm,
|
||||||
|
performance-inefficient-string-concatenation,
|
||||||
|
performance-inefficient-vector-operation,
|
||||||
|
performance-move-const-arg,
|
||||||
|
performance-move-constructor-init,
|
||||||
|
performance-no-automatic-move,
|
||||||
|
performance-noexcept-move-constructor,
|
||||||
|
performance-trivially-destructible,
|
||||||
|
performance-type-promotion-in-math-fn,
|
||||||
|
performance-unnecessary-copy-initialization,
|
||||||
|
performance-unnecessary-value-param,
|
||||||
|
portability-simd-intrinsics,
|
||||||
|
readability-avoid-const-params-in-decls,
|
||||||
|
readability-const-return-type,
|
||||||
|
readability-container-size-empty,
|
||||||
|
readability-convert-member-functions-to-static,
|
||||||
|
readability-delete-null-pointer,
|
||||||
|
readability-deleted-default,
|
||||||
|
readability-inconsistent-declaration-parameter-name,
|
||||||
|
readability-make-member-function-const,
|
||||||
|
readability-misleading-indentation,
|
||||||
|
readability-misplaced-array-index,
|
||||||
|
readability-non-const-parameter,
|
||||||
|
readability-redundant-control-flow,
|
||||||
|
readability-redundant-declaration,
|
||||||
|
readability-redundant-function-ptr-dereference,
|
||||||
|
readability-redundant-smartptr-get,
|
||||||
|
readability-redundant-string-cstr,
|
||||||
|
readability-redundant-string-init,
|
||||||
|
readability-simplify-subscript-expr,
|
||||||
|
readability-static-accessed-through-instance,
|
||||||
|
readability-static-definition-in-anonymous-namespace,
|
||||||
|
readability-string-compare,
|
||||||
|
readability-uniqueptr-delete-release,
|
||||||
|
readability-use-anyofallof'
|
||||||
586
.gitignore
vendored
Normal file
586
.gitignore
vendored
Normal file
@@ -0,0 +1,586 @@
|
|||||||
|
doc
|
||||||
|
html
|
||||||
|
latex
|
||||||
|
build
|
||||||
|
testBuild
|
||||||
|
test*.txt
|
||||||
|
|
||||||
|
# Taken from:
|
||||||
|
# https://github.com/github/gitignore
|
||||||
|
|
||||||
|
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
|
||||||
|
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
|
||||||
|
|
||||||
|
# User-specific stuff
|
||||||
|
.idea/**/workspace.xml
|
||||||
|
.idea/**/tasks.xml
|
||||||
|
.idea/**/usage.statistics.xml
|
||||||
|
.idea/**/dictionaries
|
||||||
|
.idea/**/shelf
|
||||||
|
|
||||||
|
# AWS User-specific
|
||||||
|
.idea/**/aws.xml
|
||||||
|
|
||||||
|
# Generated files
|
||||||
|
.idea/**/contentModel.xml
|
||||||
|
|
||||||
|
# Sensitive or high-churn files
|
||||||
|
.idea/**/dataSources/
|
||||||
|
.idea/**/dataSources.ids
|
||||||
|
.idea/**/dataSources.local.xml
|
||||||
|
.idea/**/sqlDataSources.xml
|
||||||
|
.idea/**/dynamic.xml
|
||||||
|
.idea/**/uiDesigner.xml
|
||||||
|
.idea/**/dbnavigator.xml
|
||||||
|
|
||||||
|
# Gradle
|
||||||
|
.idea/**/gradle.xml
|
||||||
|
.idea/**/libraries
|
||||||
|
|
||||||
|
# Gradle and Maven with auto-import
|
||||||
|
# When using Gradle or Maven with auto-import, you should exclude module files,
|
||||||
|
# since they will be recreated, and may cause churn. Uncomment if using
|
||||||
|
# auto-import.
|
||||||
|
# .idea/artifacts
|
||||||
|
# .idea/compiler.xml
|
||||||
|
# .idea/jarRepositories.xml
|
||||||
|
# .idea/modules.xml
|
||||||
|
# .idea/*.iml
|
||||||
|
# .idea/modules
|
||||||
|
# *.iml
|
||||||
|
# *.ipr
|
||||||
|
|
||||||
|
# CMake
|
||||||
|
cmake-build-*/
|
||||||
|
|
||||||
|
# Mongo Explorer plugin
|
||||||
|
.idea/**/mongoSettings.xml
|
||||||
|
|
||||||
|
# File-based project format
|
||||||
|
*.iws
|
||||||
|
|
||||||
|
# IntelliJ
|
||||||
|
out/
|
||||||
|
|
||||||
|
# mpeltonen/sbt-idea plugin
|
||||||
|
.idea_modules/
|
||||||
|
|
||||||
|
# JIRA plugin
|
||||||
|
atlassian-ide-plugin.xml
|
||||||
|
|
||||||
|
# Cursive Clojure plugin
|
||||||
|
.idea/replstate.xml
|
||||||
|
|
||||||
|
# SonarLint plugin
|
||||||
|
.idea/sonarlint/
|
||||||
|
|
||||||
|
# Crashlytics plugin (for Android Studio and IntelliJ)
|
||||||
|
com_crashlytics_export_strings.xml
|
||||||
|
crashlytics.properties
|
||||||
|
crashlytics-build.properties
|
||||||
|
fabric.properties
|
||||||
|
|
||||||
|
# Editor-based Rest Client
|
||||||
|
.idea/httpRequests
|
||||||
|
|
||||||
|
# Android studio 3.1+ serialized cache file
|
||||||
|
.idea/caches/build_file_checksums.ser
|
||||||
|
|
||||||
|
## Ignore Visual Studio temporary files, build results, and
|
||||||
|
## files generated by popular Visual Studio add-ons.
|
||||||
|
##
|
||||||
|
## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore
|
||||||
|
|
||||||
|
# User-specific files
|
||||||
|
*.rsuser
|
||||||
|
*.suo
|
||||||
|
*.user
|
||||||
|
*.userosscache
|
||||||
|
*.sln.docstates
|
||||||
|
|
||||||
|
# User-specific files (MonoDevelop/Xamarin Studio)
|
||||||
|
*.userprefs
|
||||||
|
|
||||||
|
# Mono auto generated files
|
||||||
|
mono_crash.*
|
||||||
|
|
||||||
|
# Build results
|
||||||
|
[Dd]ebug/
|
||||||
|
[Dd]ebugPublic/
|
||||||
|
[Rr]elease/
|
||||||
|
[Rr]eleases/
|
||||||
|
x64/
|
||||||
|
x86/
|
||||||
|
[Ww][Ii][Nn]32/
|
||||||
|
[Aa][Rr][Mm]/
|
||||||
|
[Aa][Rr][Mm]64/
|
||||||
|
bld/
|
||||||
|
[Bb]in/
|
||||||
|
[Oo]bj/
|
||||||
|
[Ll]og/
|
||||||
|
[Ll]ogs/
|
||||||
|
|
||||||
|
# Visual Studio 2015/2017 cache/options directory
|
||||||
|
.vs/
|
||||||
|
# Uncomment if you have tasks that create the project's static files in wwwroot
|
||||||
|
#wwwroot/
|
||||||
|
|
||||||
|
# Visual Studio 2017 auto generated files
|
||||||
|
Generated\ Files/
|
||||||
|
|
||||||
|
# MSTest test Results
|
||||||
|
[Tt]est[Rr]esult*/
|
||||||
|
[Bb]uild[Ll]og.*
|
||||||
|
|
||||||
|
# NUnit
|
||||||
|
*.VisualState.xml
|
||||||
|
TestResult.xml
|
||||||
|
nunit-*.xml
|
||||||
|
|
||||||
|
# Build Results of an ATL Project
|
||||||
|
[Dd]ebugPS/
|
||||||
|
[Rr]eleasePS/
|
||||||
|
dlldata.c
|
||||||
|
|
||||||
|
# Benchmark Results
|
||||||
|
BenchmarkDotNet.Artifacts/
|
||||||
|
|
||||||
|
# .NET Core
|
||||||
|
project.lock.json
|
||||||
|
project.fragment.lock.json
|
||||||
|
artifacts/
|
||||||
|
|
||||||
|
# ASP.NET Scaffolding
|
||||||
|
ScaffoldingReadMe.txt
|
||||||
|
|
||||||
|
# StyleCop
|
||||||
|
StyleCopReport.xml
|
||||||
|
|
||||||
|
# Files built by Visual Studio
|
||||||
|
*_i.c
|
||||||
|
*_p.c
|
||||||
|
*_h.h
|
||||||
|
*.ilk
|
||||||
|
*.meta
|
||||||
|
*.obj
|
||||||
|
*.iobj
|
||||||
|
*.pch
|
||||||
|
*.pdb
|
||||||
|
*.ipdb
|
||||||
|
*.pgc
|
||||||
|
*.pgd
|
||||||
|
*.rsp
|
||||||
|
*.sbr
|
||||||
|
*.tlb
|
||||||
|
*.tli
|
||||||
|
*.tlh
|
||||||
|
*.tmp
|
||||||
|
*.tmp_proj
|
||||||
|
*_wpftmp.csproj
|
||||||
|
*.log
|
||||||
|
*.tlog
|
||||||
|
*.vspscc
|
||||||
|
*.vssscc
|
||||||
|
.builds
|
||||||
|
*.pidb
|
||||||
|
*.svclog
|
||||||
|
*.scc
|
||||||
|
|
||||||
|
# Chutzpah Test files
|
||||||
|
_Chutzpah*
|
||||||
|
|
||||||
|
# Visual C++ cache files
|
||||||
|
ipch/
|
||||||
|
*.aps
|
||||||
|
*.ncb
|
||||||
|
*.opendb
|
||||||
|
*.opensdf
|
||||||
|
*.sdf
|
||||||
|
*.cachefile
|
||||||
|
*.VC.db
|
||||||
|
*.VC.VC.opendb
|
||||||
|
|
||||||
|
# Visual Studio profiler
|
||||||
|
*.psess
|
||||||
|
*.vsp
|
||||||
|
*.vspx
|
||||||
|
*.sap
|
||||||
|
|
||||||
|
# Visual Studio Trace Files
|
||||||
|
*.e2e
|
||||||
|
|
||||||
|
# TFS 2012 Local Workspace
|
||||||
|
$tf/
|
||||||
|
|
||||||
|
# Guidance Automation Toolkit
|
||||||
|
*.gpState
|
||||||
|
|
||||||
|
# ReSharper is a .NET coding add-in
|
||||||
|
_ReSharper*/
|
||||||
|
*.[Rr]e[Ss]harper
|
||||||
|
*.DotSettings.user
|
||||||
|
|
||||||
|
# TeamCity is a build add-in
|
||||||
|
_TeamCity*
|
||||||
|
|
||||||
|
# DotCover is a Code Coverage Tool
|
||||||
|
*.dotCover
|
||||||
|
|
||||||
|
# AxoCover is a Code Coverage Tool
|
||||||
|
.axoCover/*
|
||||||
|
!.axoCover/settings.json
|
||||||
|
|
||||||
|
# Coverlet is a free, cross platform Code Coverage Tool
|
||||||
|
coverage*.json
|
||||||
|
coverage*.xml
|
||||||
|
coverage*.info
|
||||||
|
|
||||||
|
# Visual Studio code coverage results
|
||||||
|
*.coverage
|
||||||
|
*.coveragexml
|
||||||
|
|
||||||
|
# NCrunch
|
||||||
|
_NCrunch_*
|
||||||
|
.*crunch*.local.xml
|
||||||
|
nCrunchTemp_*
|
||||||
|
|
||||||
|
# MightyMoose
|
||||||
|
*.mm.*
|
||||||
|
AutoTest.Net/
|
||||||
|
|
||||||
|
# Web workbench (sass)
|
||||||
|
.sass-cache/
|
||||||
|
|
||||||
|
# Installshield output folder
|
||||||
|
[Ee]xpress/
|
||||||
|
|
||||||
|
# DocProject is a documentation generator add-in
|
||||||
|
DocProject/buildhelp/
|
||||||
|
DocProject/Help/*.HxT
|
||||||
|
DocProject/Help/*.HxC
|
||||||
|
DocProject/Help/*.hhc
|
||||||
|
DocProject/Help/*.hhk
|
||||||
|
DocProject/Help/*.hhp
|
||||||
|
DocProject/Help/Html2
|
||||||
|
DocProject/Help/html
|
||||||
|
|
||||||
|
# Click-Once directory
|
||||||
|
publish/
|
||||||
|
|
||||||
|
# Publish Web Output
|
||||||
|
*.[Pp]ublish.xml
|
||||||
|
*.azurePubxml
|
||||||
|
# Note: Comment the next line if you want to checkin your web deploy settings,
|
||||||
|
# but database connection strings (with potential passwords) will be unencrypted
|
||||||
|
*.pubxml
|
||||||
|
*.publishproj
|
||||||
|
|
||||||
|
# Microsoft Azure Web App publish settings. Comment the next line if you want to
|
||||||
|
# checkin your Azure Web App publish settings, but sensitive information contained
|
||||||
|
# in these scripts will be unencrypted
|
||||||
|
PublishScripts/
|
||||||
|
|
||||||
|
# NuGet Packages
|
||||||
|
*.nupkg
|
||||||
|
# NuGet Symbol Packages
|
||||||
|
*.snupkg
|
||||||
|
# The packages folder can be ignored because of Package Restore
|
||||||
|
**/[Pp]ackages/*
|
||||||
|
# except build/, which is used as an MSBuild target.
|
||||||
|
!**/[Pp]ackages/build/
|
||||||
|
# Uncomment if necessary however generally it will be regenerated when needed
|
||||||
|
#!**/[Pp]ackages/repositories.config
|
||||||
|
# NuGet v3's project.json files produces more ignorable files
|
||||||
|
*.nuget.props
|
||||||
|
*.nuget.targets
|
||||||
|
|
||||||
|
# Microsoft Azure Build Output
|
||||||
|
csx/
|
||||||
|
*.build.csdef
|
||||||
|
|
||||||
|
# Microsoft Azure Emulator
|
||||||
|
ecf/
|
||||||
|
rcf/
|
||||||
|
|
||||||
|
# Windows Store app package directories and files
|
||||||
|
AppPackages/
|
||||||
|
BundleArtifacts/
|
||||||
|
Package.StoreAssociation.xml
|
||||||
|
_pkginfo.txt
|
||||||
|
*.appx
|
||||||
|
*.appxbundle
|
||||||
|
*.appxupload
|
||||||
|
|
||||||
|
# Visual Studio cache files
|
||||||
|
# files ending in .cache can be ignored
|
||||||
|
*.[Cc]ache
|
||||||
|
# but keep track of directories ending in .cache
|
||||||
|
!?*.[Cc]ache/
|
||||||
|
|
||||||
|
# Others
|
||||||
|
ClientBin/
|
||||||
|
~$*
|
||||||
|
*~
|
||||||
|
*.dbmdl
|
||||||
|
*.dbproj.schemaview
|
||||||
|
*.jfm
|
||||||
|
*.pfx
|
||||||
|
*.publishsettings
|
||||||
|
orleans.codegen.cs
|
||||||
|
|
||||||
|
# Including strong name files can present a security risk
|
||||||
|
# (https://github.com/github/gitignore/pull/2483#issue-259490424)
|
||||||
|
#*.snk
|
||||||
|
|
||||||
|
# Since there are multiple workflows, uncomment next line to ignore bower_components
|
||||||
|
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
|
||||||
|
#bower_components/
|
||||||
|
|
||||||
|
# RIA/Silverlight projects
|
||||||
|
Generated_Code/
|
||||||
|
|
||||||
|
# Backup & report files from converting an old project file
|
||||||
|
# to a newer Visual Studio version. Backup files are not needed,
|
||||||
|
# because we have git ;-)
|
||||||
|
_UpgradeReport_Files/
|
||||||
|
Backup*/
|
||||||
|
UpgradeLog*.XML
|
||||||
|
UpgradeLog*.htm
|
||||||
|
ServiceFabricBackup/
|
||||||
|
*.rptproj.bak
|
||||||
|
|
||||||
|
# SQL Server files
|
||||||
|
*.mdf
|
||||||
|
*.ldf
|
||||||
|
*.ndf
|
||||||
|
|
||||||
|
# Business Intelligence projects
|
||||||
|
*.rdl.data
|
||||||
|
*.bim.layout
|
||||||
|
*.bim_*.settings
|
||||||
|
*.rptproj.rsuser
|
||||||
|
*- [Bb]ackup.rdl
|
||||||
|
*- [Bb]ackup ([0-9]).rdl
|
||||||
|
*- [Bb]ackup ([0-9][0-9]).rdl
|
||||||
|
|
||||||
|
# Microsoft Fakes
|
||||||
|
FakesAssemblies/
|
||||||
|
|
||||||
|
# GhostDoc plugin setting file
|
||||||
|
*.GhostDoc.xml
|
||||||
|
|
||||||
|
# Node.js Tools for Visual Studio
|
||||||
|
.ntvs_analysis.dat
|
||||||
|
node_modules/
|
||||||
|
|
||||||
|
# Visual Studio 6 build log
|
||||||
|
*.plg
|
||||||
|
|
||||||
|
# Visual Studio 6 workspace options file
|
||||||
|
*.opt
|
||||||
|
|
||||||
|
# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
|
||||||
|
*.vbw
|
||||||
|
|
||||||
|
# Visual Studio 6 auto-generated project file (contains which files were open etc.)
|
||||||
|
*.vbp
|
||||||
|
|
||||||
|
# Visual Studio 6 workspace and project file (working project files containing files to include in project)
|
||||||
|
*.dsw
|
||||||
|
*.dsp
|
||||||
|
|
||||||
|
# Visual Studio 6 technical files
|
||||||
|
*.ncb
|
||||||
|
*.aps
|
||||||
|
|
||||||
|
# Visual Studio LightSwitch build output
|
||||||
|
**/*.HTMLClient/GeneratedArtifacts
|
||||||
|
**/*.DesktopClient/GeneratedArtifacts
|
||||||
|
**/*.DesktopClient/ModelManifest.xml
|
||||||
|
**/*.Server/GeneratedArtifacts
|
||||||
|
**/*.Server/ModelManifest.xml
|
||||||
|
_Pvt_Extensions
|
||||||
|
|
||||||
|
# Paket dependency manager
|
||||||
|
.paket/paket.exe
|
||||||
|
paket-files/
|
||||||
|
|
||||||
|
# FAKE - F# Make
|
||||||
|
.fake/
|
||||||
|
|
||||||
|
# CodeRush personal settings
|
||||||
|
.cr/personal
|
||||||
|
|
||||||
|
# Python Tools for Visual Studio (PTVS)
|
||||||
|
__pycache__/
|
||||||
|
*.pyc
|
||||||
|
|
||||||
|
# Cake - Uncomment if you are using it
|
||||||
|
# tools/**
|
||||||
|
# !tools/packages.config
|
||||||
|
|
||||||
|
# Tabs Studio
|
||||||
|
*.tss
|
||||||
|
|
||||||
|
# Telerik's JustMock configuration file
|
||||||
|
*.jmconfig
|
||||||
|
|
||||||
|
# BizTalk build output
|
||||||
|
*.btp.cs
|
||||||
|
*.btm.cs
|
||||||
|
*.odx.cs
|
||||||
|
*.xsd.cs
|
||||||
|
|
||||||
|
# OpenCover UI analysis results
|
||||||
|
OpenCover/
|
||||||
|
|
||||||
|
# Azure Stream Analytics local run output
|
||||||
|
ASALocalRun/
|
||||||
|
|
||||||
|
# MSBuild Binary and Structured Log
|
||||||
|
*.binlog
|
||||||
|
|
||||||
|
# NVidia Nsight GPU debugger configuration file
|
||||||
|
*.nvuser
|
||||||
|
|
||||||
|
# MFractors (Xamarin productivity tool) working folder
|
||||||
|
.mfractor/
|
||||||
|
|
||||||
|
# Local History for Visual Studio
|
||||||
|
.localhistory/
|
||||||
|
|
||||||
|
# Visual Studio History (VSHistory) files
|
||||||
|
.vshistory/
|
||||||
|
|
||||||
|
# BeatPulse healthcheck temp database
|
||||||
|
healthchecksdb
|
||||||
|
|
||||||
|
# Backup folder for Package Reference Convert tool in Visual Studio 2017
|
||||||
|
MigrationBackup/
|
||||||
|
|
||||||
|
# Ionide (cross platform F# VS Code tools) working folder
|
||||||
|
.ionide/
|
||||||
|
|
||||||
|
# Fody - auto-generated XML schema
|
||||||
|
FodyWeavers.xsd
|
||||||
|
|
||||||
|
# VS Code files for those working on multiple tools
|
||||||
|
.vscode/*
|
||||||
|
!.vscode/settings.json
|
||||||
|
!.vscode/tasks.json
|
||||||
|
!.vscode/launch.json
|
||||||
|
!.vscode/extensions.json
|
||||||
|
*.code-workspace
|
||||||
|
|
||||||
|
# Local History for Visual Studio Code
|
||||||
|
.history/
|
||||||
|
|
||||||
|
# Windows Installer files from build outputs
|
||||||
|
*.cab
|
||||||
|
*.msi
|
||||||
|
*.msix
|
||||||
|
*.msm
|
||||||
|
*.msp
|
||||||
|
|
||||||
|
# JetBrains Rider
|
||||||
|
*.sln.iml
|
||||||
|
|
||||||
|
# Prerequisites
|
||||||
|
*.d
|
||||||
|
|
||||||
|
# Compiled Object files
|
||||||
|
*.slo
|
||||||
|
*.lo
|
||||||
|
*.o
|
||||||
|
*.obj
|
||||||
|
|
||||||
|
# Precompiled Headers
|
||||||
|
*.gch
|
||||||
|
*.pch
|
||||||
|
|
||||||
|
# Compiled Dynamic libraries
|
||||||
|
*.so
|
||||||
|
*.dylib
|
||||||
|
*.dll
|
||||||
|
|
||||||
|
# Fortran module files
|
||||||
|
*.mod
|
||||||
|
*.smod
|
||||||
|
|
||||||
|
# Compiled Static libraries
|
||||||
|
*.lai
|
||||||
|
*.la
|
||||||
|
*.a
|
||||||
|
*.lib
|
||||||
|
|
||||||
|
# Executables
|
||||||
|
*.exe
|
||||||
|
*.out
|
||||||
|
*.app
|
||||||
|
|
||||||
|
# General
|
||||||
|
.DS_Store
|
||||||
|
.AppleDouble
|
||||||
|
.LSOverride
|
||||||
|
|
||||||
|
# Icon must end with two \r
|
||||||
|
Icon
|
||||||
|
|
||||||
|
|
||||||
|
# Thumbnails
|
||||||
|
._*
|
||||||
|
|
||||||
|
# Files that might appear in the root of a volume
|
||||||
|
.DocumentRevisions-V100
|
||||||
|
.fseventsd
|
||||||
|
.Spotlight-V100
|
||||||
|
.TemporaryItems
|
||||||
|
.Trashes
|
||||||
|
.VolumeIcon.icns
|
||||||
|
.com.apple.timemachine.donotpresent
|
||||||
|
|
||||||
|
# Directories potentially created on remote AFP share
|
||||||
|
.AppleDB
|
||||||
|
.AppleDesktop
|
||||||
|
Network Trash Folder
|
||||||
|
Temporary Items
|
||||||
|
.apdisk
|
||||||
|
|
||||||
|
|
||||||
|
# Windows thumbnail cache files
|
||||||
|
Thumbs.db
|
||||||
|
Thumbs.db:encryptable
|
||||||
|
ehthumbs.db
|
||||||
|
ehthumbs_vista.db
|
||||||
|
|
||||||
|
# Dump file
|
||||||
|
*.stackdump
|
||||||
|
|
||||||
|
# Folder config file
|
||||||
|
[Dd]esktop.ini
|
||||||
|
|
||||||
|
# Recycle Bin used on file shares
|
||||||
|
$RECYCLE.BIN/
|
||||||
|
|
||||||
|
# Windows Installer files
|
||||||
|
*.cab
|
||||||
|
*.msi
|
||||||
|
*.msix
|
||||||
|
*.msm
|
||||||
|
*.msp
|
||||||
|
|
||||||
|
# Windows shortcuts
|
||||||
|
*.lnk
|
||||||
|
|
||||||
|
.vscode/*
|
||||||
|
!.vscode/settings.json
|
||||||
|
!.vscode/tasks.json
|
||||||
|
!.vscode/launch.json
|
||||||
|
!.vscode/extensions.json
|
||||||
|
!.vscode/*.code-snippets
|
||||||
|
|
||||||
|
# Local History for Visual Studio Code
|
||||||
|
.history/
|
||||||
|
|
||||||
|
# Built Visual Studio Code Extensions
|
||||||
|
*.vsix
|
||||||
8
.idea/.gitignore
generated
vendored
Normal file
8
.idea/.gitignore
generated
vendored
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
# Default ignored files
|
||||||
|
/shelf/
|
||||||
|
/workspace.xml
|
||||||
|
# Editor-based HTTP Client requests
|
||||||
|
/httpRequests/
|
||||||
|
# Datasource local storage ignored files
|
||||||
|
/dataSources/
|
||||||
|
/dataSources.local.xml
|
||||||
1
.idea/.name
generated
Normal file
1
.idea/.name
generated
Normal file
@@ -0,0 +1 @@
|
|||||||
|
sembackup
|
||||||
2
.idea/backup.iml
generated
Normal file
2
.idea/backup.iml
generated
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<module classpath="CMake" type="CPP_MODULE" version="4" />
|
||||||
7
.idea/codeStyles/Project.xml
generated
Normal file
7
.idea/codeStyles/Project.xml
generated
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
<component name="ProjectCodeStyleConfiguration">
|
||||||
|
<code_scheme name="Project" version="173">
|
||||||
|
<clangFormatSettings>
|
||||||
|
<option name="ENABLED" value="true" />
|
||||||
|
</clangFormatSettings>
|
||||||
|
</code_scheme>
|
||||||
|
</component>
|
||||||
5
.idea/codeStyles/codeStyleConfig.xml
generated
Normal file
5
.idea/codeStyles/codeStyleConfig.xml
generated
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
<component name="ProjectCodeStyleConfiguration">
|
||||||
|
<state>
|
||||||
|
<option name="USE_PER_PROJECT_SETTINGS" value="true" />
|
||||||
|
</state>
|
||||||
|
</component>
|
||||||
4
.idea/misc.xml
generated
Normal file
4
.idea/misc.xml
generated
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project version="4">
|
||||||
|
<component name="CMakeWorkspace" PROJECT_DIR="$PROJECT_DIR$" />
|
||||||
|
</project>
|
||||||
8
.idea/modules.xml
generated
Normal file
8
.idea/modules.xml
generated
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project version="4">
|
||||||
|
<component name="ProjectModuleManager">
|
||||||
|
<modules>
|
||||||
|
<module fileurl="file://$PROJECT_DIR$/.idea/backup.iml" filepath="$PROJECT_DIR$/.idea/backup.iml" />
|
||||||
|
</modules>
|
||||||
|
</component>
|
||||||
|
</project>
|
||||||
28
CMakeLists.txt
Normal file
28
CMakeLists.txt
Normal file
@@ -0,0 +1,28 @@
|
|||||||
|
cmake_minimum_required(VERSION 3.22)
|
||||||
|
|
||||||
|
add_compile_options(-Ofast)
|
||||||
|
add_link_options(-Ofast)
|
||||||
|
|
||||||
|
# add_compile_options(-Ofast -flto)
|
||||||
|
# add_link_options(-Ofast -flto)
|
||||||
|
|
||||||
|
#add_compile_options(-Wall -O0 -Wextra -pedantic -Wshadow -Wformat=2 -Wfloat-equal -D_GLIBCXX_DEBUG -Wconversion -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -g -rdynamic)
|
||||||
|
#add_compile_options(-fsanitize=address -fsanitize=undefined -fno-sanitize-recover)
|
||||||
|
#add_link_options(-fsanitize=address -fsanitize=undefined -fno-sanitize-recover)
|
||||||
|
#add_link_options(-rdynamic)
|
||||||
|
|
||||||
|
project(sembackup)
|
||||||
|
|
||||||
|
set(CMAKE_CXX_STANDARD 17)
|
||||||
|
|
||||||
|
find_package(OpenSSL REQUIRED)
|
||||||
|
find_package(ZLIB REQUIRED)
|
||||||
|
|
||||||
|
add_executable(sembackup src/Config.cpp src/main.cpp src/repo/Object.cpp src/repo/Object.h src/repo/Repository.cpp src/repo/Repository.h src/repo/FileRepository.cpp src/repo/FileRepository.h src/repo/objects/Archive.cpp src/repo/objects/Archive.h src/repo/objects/File.cpp src/repo/objects/File.h src/repo/objects/Chunk.cpp src/repo/objects/Chunk.h src/repo/Serialize.h src/chunkers/Chunker.cpp src/chunkers/Chunker.h src/chunkers/ConstChunker.cpp src/chunkers/ConstChunker.h src/crypto/MD5.cpp src/crypto/MD5.h src/change_detectors/ChangeDetector.cpp src/change_detectors/ChangeDetector.h src/change_detectors/SizeChangeDetector.cpp src/change_detectors/SizeChangeDetector.h src/change_detectors/EditTimeChangeDetector.cpp src/change_detectors/EditTimeChangeDetector.h src/change_detectors/ChangeDetectorFactory.cpp src/change_detectors/ChangeDetectorFactory.h src/Signals.h src/Signals.cpp src/filters/Filter.cpp src/filters/Filter.h src/filters/FilterShift.cpp src/filters/FilterShift.h src/filters/FilterShiftSecret.cpp src/filters/FilterShiftSecret.h src/repo/objects/FileBuffer.cpp src/repo/objects/FileBuffer.h src/filters/FilterZlib.cpp src/filters/FilterZlib.h src/filters/FilterAES.cpp src/filters/FilterAES.h src/chunkers/BuzhashChunker.cpp src/chunkers/BuzhashChunker.h src/chunkers/Buzhash.cpp src/chunkers/Buzhash.h src/crypto/AES.cpp src/crypto/AES.h src/chunkers/ChunkerFactory.cpp src/chunkers/ChunkerFactory.h src/Exception.cpp src/Exception.h src/filters/FilterFactory.h src/filters/FilterContainer.h src/filters/FilterFactory.cpp src/filters/FilterContainer.cpp src/change_detectors/ChangeDetectorContainer.cpp src/change_detectors/ChangeDetectorContainer.h src/Progress.cpp src/Progress.h src/change_detectors/ContentsChangeDetector.cpp src/change_detectors/ContentsChangeDetector.h src/change_detectors/ComparableFile.cpp src/change_detectors/ComparableFile.h src/RunningAverage.cpp src/RunningAverage.h src/Diff.cpp src/Diff.h src/ThreadPool.cpp src/filters/CheckFilter.cpp src/filters/CheckFilter.h src/crypto/CRC32.cpp src/crypto/CRC32.h src/change_detectors/TypeChangeDetector.cpp src/change_detectors/TypeChangeDetector.h src/RunningDiffAverage.cpp src/RunningDiffAverage.h src/BytesFormatter.cpp src/BytesFormatter.h src/Logger.cpp src/Logger.h src/Context.h src/commands/Command.cpp src/commands/Command.h src/commands/CommandRun.cpp src/commands/CommandRun.h src/commands/CommandsCommon.cpp src/commands/CommandsCommon.h src/commands/CommandRestore.cpp src/commands/CommandRestore.h src/commands/CommandDiff.cpp src/commands/CommandDiff.h src/commands/CommandList.cpp src/commands/CommandList.h src/commands/CommandListFiles.cpp src/commands/CommandListFiles.h)
|
||||||
|
add_executable(test src/Config.cpp src/crypto/MD5.cpp src/change_detectors/EditTimeChangeDetector.cpp tests/runTests.cpp tests/utils/Runnable.cpp tests/utils/HelpfulAssertTest.cpp tests/utils/HelpfulAssertTest.h tests/utils/TestGroup.cpp tests/utils/TestGroup.h tests/utils/Test.cpp tests/utils/Test.h tests/utils/TestGroupGenerator.h src/repo/Object.cpp src/repo/Object.h src/repo/Repository.cpp src/repo/Repository.h src/repo/FileRepository.cpp src/repo/FileRepository.h src/repo/objects/Archive.cpp src/repo/objects/Archive.h src/repo/objects/File.cpp src/repo/objects/File.h src/repo/objects/Chunk.cpp src/repo/objects/Chunk.h tests/repo/ChunkTest.cpp tests/repo/ChunkTest.h tests/repo/FileRepositoryTest.cpp tests/repo/FileRepositoryTest.h tests/utils/Cleaner.cpp tests/utils/Cleaner.h src/repo/Serialize.h src/chunkers/Chunker.cpp src/chunkers/Chunker.h src/chunkers/ConstChunker.cpp src/chunkers/ConstChunker.h tests/crypto/MD5Test.cpp tests/crypto/MD5Test.h tests/fulltests/FullTest.cpp tests/fulltests/FullTest.h src/change_detectors/ChangeDetector.cpp src/change_detectors/ChangeDetector.h src/change_detectors/SizeChangeDetector.cpp src/change_detectors/SizeChangeDetector.h src/change_detectors/ChangeDetectorFactory.cpp src/change_detectors/ChangeDetectorFactory.h src/filters/Filter.cpp src/filters/Filter.h src/filters/FilterShift.cpp src/filters/FilterShift.h src/filters/FilterShiftSecret.cpp src/filters/FilterShiftSecret.h src/repo/objects/FileBuffer.cpp src/repo/objects/FileBuffer.h src/filters/FilterZlib.cpp src/filters/FilterZlib.h src/filters/FilterAES.cpp src/filters/FilterAES.h src/chunkers/BuzhashChunker.cpp src/chunkers/BuzhashChunker.h src/chunkers/Buzhash.cpp src/chunkers/Buzhash.h tests/BuzhashTest.cpp tests/BuzhashTest.h src/crypto/AES.cpp src/crypto/AES.h tests/crypto/AESTest.cpp tests/crypto/AESTest.h src/chunkers/ChunkerFactory.cpp src/chunkers/ChunkerFactory.h src/Exception.cpp src/Exception.h src/filters/FilterFactory.h src/filters/FilterContainer.h src/filters/FilterFactory.cpp src/filters/FilterContainer.cpp src/change_detectors/ChangeDetectorContainer.cpp src/change_detectors/ChangeDetectorContainer.h src/Progress.cpp src/Progress.h src/change_detectors/ContentsChangeDetector.cpp src/change_detectors/ContentsChangeDetector.h src/change_detectors/ComparableFile.cpp src/change_detectors/ComparableFile.h src/RunningAverage.cpp src/RunningAverage.h src/Diff.cpp src/Diff.h src/ThreadPool.cpp tests/CLITestWrapper.cpp tests/CLITestWrapper.h src/filters/CheckFilter.cpp src/filters/CheckFilter.h src/crypto/CRC32.cpp src/crypto/CRC32.h src/change_detectors/TypeChangeDetector.cpp src/change_detectors/TypeChangeDetector.h src/RunningDiffAverage.cpp src/RunningDiffAverage.h src/BytesFormatter.cpp src/BytesFormatter.h src/Logger.cpp src/Logger.h src/Context.h src/commands/Command.cpp src/commands/Command.h src/commands/CommandRun.cpp src/commands/CommandRun.h src/commands/CommandsCommon.cpp src/commands/CommandsCommon.h src/commands/CommandRestore.cpp src/commands/CommandRestore.h src/commands/CommandList.cpp src/commands/CommandList.h src/commands/CommandListFiles.cpp src/commands/CommandListFiles.h)
|
||||||
|
|
||||||
|
add_dependencies(test sembackup)
|
||||||
|
|
||||||
|
target_link_libraries(sembackup OpenSSL::SSL ZLIB::ZLIB)
|
||||||
|
target_link_libraries(test OpenSSL::SSL ZLIB::ZLIB)
|
||||||
|
target_compile_definitions(test PUBLIC TEST)
|
||||||
102
README.md
Normal file
102
README.md
Normal file
@@ -0,0 +1,102 @@
|
|||||||
|
# Backup app
|
||||||
|
|
||||||
|
## Key features
|
||||||
|
|
||||||
|
- Deduplicated backups with additional compression and encryption
|
||||||
|
|
||||||
|
- Files are split into chunks and stored in a repository as a list of
|
||||||
|
pointers to these chunks
|
||||||
|
|
||||||
|
- These chunks are reused for all files in the repository, based on
|
||||||
|
their hash matching.
|
||||||
|
|
||||||
|
- If when creating an archive, a file hasn’t changed it is reused from
|
||||||
|
a previous archive
|
||||||
|
|
||||||
|
- Content defined chunking algorithm based on a rolling hash
|
||||||
|
|
||||||
|
- Unlike when splitting file in fixed size chunks, still works
|
||||||
|
when there is an insertion/deletion of bytes in middle/beginning
|
||||||
|
of the file
|
||||||
|
|
||||||
|
- These Files, Chunks and Archives are stored in object storage
|
||||||
|
Repository
|
||||||
|
|
||||||
|
- Default implementation - FileRepository, stores these objects in
|
||||||
|
the file system
|
||||||
|
|
||||||
|
- Possible other implementations - database, cloud object storage…
|
||||||
|
|
||||||
|
- Multithreading
|
||||||
|
|
||||||
|
- Directories are ignored if they have a `.nobackup` file
|
||||||
|
|
||||||
|
- Files are ignored in a directory recursively based on a `.ignore`
|
||||||
|
file - one line contains one regex rule, which, if a directory entry
|
||||||
|
matches, it is ignored
|
||||||
|
|
||||||
|
# Quick start
|
||||||
|
|
||||||
|
Initialize a repository with compression and encryption (you can also
|
||||||
|
specify other options like –from here, they will be written to the
|
||||||
|
repository if it’s possible, so far changing options of an already
|
||||||
|
existing repo is not implemented)
|
||||||
|
|
||||||
|
sembackup init --repo <target dir> --compression zlib --compression-level 4 --encryption aes --password <password> --salt <random salt>
|
||||||
|
|
||||||
|
Run a backup
|
||||||
|
|
||||||
|
sembackup run --from <source dir> --repo <target dir> --password <password>
|
||||||
|
|
||||||
|
List avaiable archives
|
||||||
|
|
||||||
|
sembackup list --repo <repo dir> --password <password>
|
||||||
|
|
||||||
|
List files in an archive
|
||||||
|
|
||||||
|
sembackup list-files --repo <repo dir> --password <password> --aid <archive id>
|
||||||
|
|
||||||
|
Restore a backup
|
||||||
|
|
||||||
|
sembackup restore --repo <repo dir> --password <password> --aid <archive id> --to <destination>
|
||||||
|
|
||||||
|
Compare source dir with latest archive
|
||||||
|
|
||||||
|
sembackup diff --repo <repo dir> --password <password> --from <source dir>
|
||||||
|
|
||||||
|
Compare source dir with an archive
|
||||||
|
|
||||||
|
sembackup diff --repo <repo dir> --password <password> --from <source dir> --aid <archive id>
|
||||||
|
|
||||||
|
Compare two archives
|
||||||
|
|
||||||
|
sembackup diff --repo <repo dir> --password <password> --from <source dir> --aid <archive id> --aid2 <archive id>
|
||||||
|
|
||||||
|
Compare subdirectory in the source dir with latest archive
|
||||||
|
|
||||||
|
sembackup diff --repo <repo dir> --password <password> --from <source dir> --prefix <subdir>
|
||||||
|
|
||||||
|
# Data format
|
||||||
|
|
||||||
|
All data is represented as objects, stored in a repository.
|
||||||
|
|
||||||
|
An `Archive` object represents a snapshot of the file system in the
|
||||||
|
moment of its creation, and consists of a list of pointers (Object ids)
|
||||||
|
to `File` objectsю
|
||||||
|
|
||||||
|
`File` object consists of its basic metadata, and a list of chunks,
|
||||||
|
identified by their ids, which can be shared between multiple files (and
|
||||||
|
within the same file) if their MD5 hashes match.
|
||||||
|
|
||||||
|
`Chunk` object is a binary blob, identified by its MD5 hash.
|
||||||
|
|
||||||
|
These objects are children of `Object`, providing a `getKey()` method,
|
||||||
|
(name for `Archive`, path for `File`, and MD5 hash for `Chunk`) which is
|
||||||
|
used by `Repository` to make them easily accessible.
|
||||||
|
|
||||||
|
In default (and so far the only) repository implementation
|
||||||
|
`FileRepository` these objects are grouped together into files of size
|
||||||
|
approixmately `repo-target` MB, (by default 128), and there exists a
|
||||||
|
key-value index of indexed objects written into `index`, and an
|
||||||
|
`offsets` file recording the location and offset of each object in the
|
||||||
|
file system.
|
||||||
37
src/BytesFormatter.cpp
Normal file
37
src/BytesFormatter.cpp
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 13.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "BytesFormatter.h"
|
||||||
|
|
||||||
|
#include <iomanip>
|
||||||
|
#include <sstream>
|
||||||
|
|
||||||
|
BytesFormatter::BytesFormat BytesFormatter::format(unsigned long long int bytes) {
|
||||||
|
std::stringstream outNum;
|
||||||
|
outNum << std::fixed << std::setprecision(2);
|
||||||
|
|
||||||
|
if (bytes > 1024UL * 1024 * 1024 * 1024) {
|
||||||
|
outNum << (double) bytes / (1024.0 * 1024.0 * 1024.0 * 1024.0);
|
||||||
|
return {outNum.str(), "TiB"};
|
||||||
|
}
|
||||||
|
if (bytes > 1024UL * 1024 * 1024) {
|
||||||
|
outNum << (double) bytes / (1024.0 * 1024.0 * 1024.0);
|
||||||
|
return {outNum.str(), "GiB"};
|
||||||
|
}
|
||||||
|
if (bytes > 1024UL * 1024) {
|
||||||
|
outNum << (double) bytes / (1024.0 * 1024.0);
|
||||||
|
return {outNum.str(), "MiB"};
|
||||||
|
}
|
||||||
|
if (bytes > 1024UL) {
|
||||||
|
outNum << (double) bytes / (1024.0);
|
||||||
|
return {outNum.str(), "KiB"};
|
||||||
|
}
|
||||||
|
outNum << bytes;
|
||||||
|
return {outNum.str(), "Bytes"};
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string BytesFormatter::formatStr(unsigned long long int bytes) {
|
||||||
|
auto fmt = format(bytes);
|
||||||
|
return fmt.number + " " + fmt.prefix;
|
||||||
|
}
|
||||||
31
src/BytesFormatter.h
Normal file
31
src/BytesFormatter.h
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 13.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_BYTESFORMATTER_H
|
||||||
|
#define SEMBACKUP_BYTESFORMATTER_H
|
||||||
|
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
/// Utility class to format byte values according to their magnitude
|
||||||
|
class BytesFormatter {
|
||||||
|
public:
|
||||||
|
/// Structure for returning the processed byte value
|
||||||
|
struct BytesFormat {
|
||||||
|
std::string number;///< Number part of the value
|
||||||
|
std::string prefix;///< Unit of measure
|
||||||
|
};
|
||||||
|
|
||||||
|
/// Formats the bytes in BytesFormat format
|
||||||
|
/// \param bytes Number of bytes
|
||||||
|
/// \return BytesFormat value
|
||||||
|
static BytesFormat format(unsigned long long bytes);
|
||||||
|
|
||||||
|
/// Formats the bytes into a string
|
||||||
|
/// \param bytes Number of bytes
|
||||||
|
/// \return String, consisting of the scaled number and the unit of measure separated by a space
|
||||||
|
static std::string formatStr(unsigned long long bytes);
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_BYTESFORMATTER_H
|
||||||
81
src/Config.cpp
Normal file
81
src/Config.cpp
Normal file
@@ -0,0 +1,81 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 01.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "Config.h"
|
||||||
|
#include "Exception.h"
|
||||||
|
#include "repo/Serialize.h"
|
||||||
|
#include <sstream>
|
||||||
|
|
||||||
|
Config &Config::add(const std::string &k, const std::string &v) {
|
||||||
|
if (keys.count(k) == 0) throw Exception("Unknown key " + k);
|
||||||
|
if (data.count(k) > 0)
|
||||||
|
if (data.at(k) != v) throw Exception("Trying to rewrite config!");
|
||||||
|
else if (data.at(k) == v)
|
||||||
|
return *this;
|
||||||
|
|
||||||
|
switch (keys.at(k).type) {
|
||||||
|
case KeyType::STRING:
|
||||||
|
break;
|
||||||
|
case KeyType::INT:
|
||||||
|
try {
|
||||||
|
std::stoi(v);
|
||||||
|
} catch (...) {
|
||||||
|
throw Exception("Can't convert " + k + " to integer!");
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case KeyType::LIST:
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
data.emplace(k, v);
|
||||||
|
return *this;
|
||||||
|
}
|
||||||
|
|
||||||
|
int Config::getInt(const std::string &k) const {
|
||||||
|
return std::stoi(getStr(k));
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<std::string> Config::getList(const std::string &k) const {
|
||||||
|
std::vector<std::string> out;
|
||||||
|
std::string next;
|
||||||
|
std::stringstream inss(getStr(k));
|
||||||
|
while (std::getline(inss, next, ',')) {
|
||||||
|
if (next != "")
|
||||||
|
out.emplace_back(next);
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string Config::getStr(const std::string &k) const {
|
||||||
|
if (data.count(k) > 0) return data.at(k);
|
||||||
|
else if (keys.at(k).defaultval.has_value())
|
||||||
|
return keys.at(k).defaultval.value();
|
||||||
|
throw Exception("Option " + k + " not specified and no default value exists!");
|
||||||
|
}
|
||||||
|
|
||||||
|
bool Config::exists(const std::string &k) const {
|
||||||
|
return (data.count(k) > 0) || (keys.at(k).defaultval.has_value());
|
||||||
|
}
|
||||||
|
|
||||||
|
Config::Config() = default;
|
||||||
|
|
||||||
|
Config::Config(std::vector<char, std::allocator<char>>::const_iterator &in, const std::vector<char, std::allocator<char>>::const_iterator &end) {
|
||||||
|
data = Serialize::deserialize<decltype(data)>(in, end);
|
||||||
|
}
|
||||||
|
|
||||||
|
void Config::serialize(std::vector<char> &out) const {
|
||||||
|
std::vector<decltype(data)::value_type> temp;
|
||||||
|
for (const auto &d: data) {
|
||||||
|
if (keys.at(d.first).remember) {
|
||||||
|
temp.emplace_back(d);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Serialize::serialize(temp, out);
|
||||||
|
}
|
||||||
|
|
||||||
|
void Config::merge(const Config &config) {
|
||||||
|
for (const auto &d: config.data) {
|
||||||
|
add(d.first, d.second);
|
||||||
|
}
|
||||||
|
}
|
||||||
117
src/Config.h
Normal file
117
src/Config.h
Normal file
@@ -0,0 +1,117 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 01.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_CONFIG_H
|
||||||
|
#define SEMBACKUP_CONFIG_H
|
||||||
|
|
||||||
|
#include <optional>
|
||||||
|
#include <set>
|
||||||
|
#include <string>
|
||||||
|
#include <unordered_map>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
/// Utility class to manage configuration
|
||||||
|
/**
|
||||||
|
* Also provides keys map for information about config keys
|
||||||
|
* Serializable, remembers only the keys with remember option set in keys
|
||||||
|
*/
|
||||||
|
class Config {
|
||||||
|
public:
|
||||||
|
/// Constructs an empty Config instance
|
||||||
|
Config();
|
||||||
|
/// Deserialization constructor
|
||||||
|
Config(std::vector<char>::const_iterator &in, const std::vector<char>::const_iterator &end);
|
||||||
|
|
||||||
|
/// Adds a key \p k with value \p v to the config
|
||||||
|
/// \param k Const reference to the config key
|
||||||
|
/// \param v Config value
|
||||||
|
/// \return Reference to itself
|
||||||
|
/// \throws Exception if key is invalid or is already set with different value
|
||||||
|
Config &add(const std::string &k, const std::string &v);
|
||||||
|
|
||||||
|
/// Merges \p config to itself
|
||||||
|
/// Adds every config pair from \p config to itself, throws on conflict
|
||||||
|
/// \param config Constant reference to the source Config
|
||||||
|
/// \throws Exception on merge conflict
|
||||||
|
void merge(const Config &config);
|
||||||
|
|
||||||
|
/// Returns an int from config key \p k
|
||||||
|
/// \param k Constant reference to the key string
|
||||||
|
/// \return Config int
|
||||||
|
/// \throws Exception if key is invalid or value isn't an int
|
||||||
|
int getInt(const std::string &k) const;
|
||||||
|
|
||||||
|
/// Returns a string from config key \p k
|
||||||
|
/// \param k Constant reference to the key string
|
||||||
|
/// \return Config value for key
|
||||||
|
/// \throws Exception if key is invalid
|
||||||
|
std::string getStr(const std::string &k) const;
|
||||||
|
|
||||||
|
/// Returns a list of strings delimited by commas from config key \p k
|
||||||
|
/// \param k Constant reference to the key string
|
||||||
|
/// \return Vector of strings
|
||||||
|
/// \throws Exception if key is invalid
|
||||||
|
std::vector<std::string> getList(const std::string &k) const;
|
||||||
|
|
||||||
|
/// Checks if key \p k exists in the config
|
||||||
|
/// \param k Constant reference to the key string
|
||||||
|
/// \return True if key exists or its default value exists
|
||||||
|
bool exists(const std::string &k) const;
|
||||||
|
|
||||||
|
/// Serialization function
|
||||||
|
void serialize(std::vector<char> &out) const;
|
||||||
|
|
||||||
|
using serializable = std::true_type;
|
||||||
|
|
||||||
|
enum class KeyType {
|
||||||
|
STRING,
|
||||||
|
INT,
|
||||||
|
LIST
|
||||||
|
};
|
||||||
|
|
||||||
|
/// Struct to record key options
|
||||||
|
struct keyopts {
|
||||||
|
std::optional<std::string> defaultval;///< Key's default value
|
||||||
|
KeyType type; ///< Key's type
|
||||||
|
bool remember; ///< Whether the key should be serialized
|
||||||
|
std::string info; ///< Printed in help
|
||||||
|
};
|
||||||
|
|
||||||
|
/// Used for printing help
|
||||||
|
const static inline std::unordered_map<KeyType, std::string> KeyTypeToStr{{KeyType::STRING, "string"}, {KeyType::INT, "number"}, {KeyType::LIST, "comma-separated list"}};
|
||||||
|
|
||||||
|
/// Default values and their metadata
|
||||||
|
const static inline std::unordered_map<std::string, keyopts> keys{
|
||||||
|
{"compression", {"none", KeyType::STRING, true, "Compression algorighm to use (zlib or none)"}},
|
||||||
|
{"encryption", {"none", KeyType::STRING, true, "Encryption algorighm to use (aes or none)"}},
|
||||||
|
{"compression-level", {"-1", KeyType::INT, true, "Compression level to use (0 to 9)"}},
|
||||||
|
{"repo", {std::nullopt, KeyType::STRING, false, "Repository root"}},
|
||||||
|
{"to", {std::nullopt, KeyType::STRING, false, "Destination of restore"}},
|
||||||
|
{"from", {std::nullopt, KeyType::STRING, true, "Backed up folder"}},
|
||||||
|
{"type", {"normal", KeyType::STRING, false, "Type of archive"}},
|
||||||
|
{"aid", {std::nullopt, KeyType::INT, false, "ID of archive to restore/compare to"}},
|
||||||
|
{"aid2", {std::nullopt, KeyType::INT, false, "ID of archive to compare with"}},
|
||||||
|
{"threads", {std::nullopt, KeyType::INT, false, "Number of threads to use"}},
|
||||||
|
{"prefix", {"", KeyType::STRING, false, "Prefix of files to compare"}},
|
||||||
|
{"password", {std::nullopt, KeyType::STRING, false, "Encryption password"}},
|
||||||
|
{"salt", {std::nullopt, KeyType::STRING, true, "Encryption salt"}},
|
||||||
|
{"chunker", {"buzhash", KeyType::STRING, true, "Chunker to use (const, buzhash)"}},
|
||||||
|
{"chunker-min", {"256", KeyType::INT, true, "Min chunk size in KB"}},
|
||||||
|
{"chunker-max", {"4096", KeyType::INT, true, "Max chunk size in KB"}},
|
||||||
|
{"chunker-mask", {"20", KeyType::INT, true, "Chunker hash bit mask (mask of n bits results in average chunk size of 2^n bytes)"}},
|
||||||
|
{"repo-target", {"128", KeyType::INT, true, "Target size of files for FileRepository"}},
|
||||||
|
{"full-period", {"2", KeyType::INT, true, "Interval between forced full backups"}},
|
||||||
|
{"progress", {"pretty", KeyType::STRING, false, "How to print progress (simple, pretty, none)"}},
|
||||||
|
{"verbose", {"1", KeyType::INT, false, "Message verbosity (0 - error, 1 - info, -1 - quiet)"}},
|
||||||
|
{"dedup", {"on", KeyType::STRING, true, "Turns deduplication on/off"}},
|
||||||
|
{"change-detectors", {"type,size,etime", KeyType::LIST, true, "Change detectors to use (in order)"}},
|
||||||
|
{"diff-mode", {"normal", KeyType::STRING, false, "Diff mode (file or normal)"}},
|
||||||
|
};
|
||||||
|
|
||||||
|
private:
|
||||||
|
std::unordered_map<std::string, std::string> data;
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_CONFIG_H
|
||||||
18
src/Context.h
Normal file
18
src/Context.h
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 23.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_CONTEXT_H
|
||||||
|
#define SEMBACKUP_CONTEXT_H
|
||||||
|
|
||||||
|
#include "Config.h"
|
||||||
|
#include "Logger.h"
|
||||||
|
#include "repo/Repository.h"
|
||||||
|
|
||||||
|
struct Context {
|
||||||
|
Logger *logger;
|
||||||
|
Repository *repo;
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_CONTEXT_H
|
||||||
103
src/Diff.cpp
Normal file
103
src/Diff.cpp
Normal file
@@ -0,0 +1,103 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 06.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "Diff.h"
|
||||||
|
|
||||||
|
#include "BytesFormatter.h"
|
||||||
|
#include "Exception.h"
|
||||||
|
#include "Signals.h"
|
||||||
|
#include "chunkers/BuzhashChunker.h"
|
||||||
|
|
||||||
|
bool Diff::isBinary(const ComparableFile &c) {
|
||||||
|
auto b = c.contents();
|
||||||
|
for (unsigned int i = 0; i < std::min(c.bytes, 2048ULL); i++) {
|
||||||
|
auto e = b->sbumpc();
|
||||||
|
if (std::streambuf::traits_type::to_char_type(e) == '\0') return true;
|
||||||
|
if (e == std::streambuf::traits_type::eof()) return false;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string Diff::diff(const ComparableFile &c1, const ComparableFile &c2) {
|
||||||
|
if (isBinary(c1) || isBinary(c2)) {
|
||||||
|
if (!(isBinary(c1) && isBinary(c2))) return "One of the files is binary, the other is not";
|
||||||
|
return diffPercent(c1, c2);
|
||||||
|
}
|
||||||
|
|
||||||
|
std::stringstream out;
|
||||||
|
auto b1 = c1.contents();
|
||||||
|
auto b2 = c2.contents();
|
||||||
|
std::multimap<std::string, unsigned long> f1lines;
|
||||||
|
std::multimap<std::string, unsigned long> f2diff;
|
||||||
|
std::string line;
|
||||||
|
std::istream is1(b1.get());
|
||||||
|
std::istream is2(b2.get());
|
||||||
|
|
||||||
|
int i = 0;
|
||||||
|
while (std::getline(is1, line)) {
|
||||||
|
/// Exit when asked to
|
||||||
|
if (Signals::shouldQuit) throw Exception("Quitting");
|
||||||
|
f1lines.emplace(line, ++i);
|
||||||
|
}
|
||||||
|
|
||||||
|
i = 0;
|
||||||
|
while (std::getline(is2, line)) {
|
||||||
|
/// Exit when asked to
|
||||||
|
if (Signals::shouldQuit) throw Exception("Quitting");
|
||||||
|
if (f1lines.count(line) > 0) f1lines.erase(f1lines.find(line));
|
||||||
|
else
|
||||||
|
f2diff.emplace(line, ++i);
|
||||||
|
}
|
||||||
|
|
||||||
|
out << "\nLines only in first file: " << std::endl;
|
||||||
|
for (const auto &s: f1lines) {
|
||||||
|
out << s.second << "<" << s.first << std::endl;
|
||||||
|
}
|
||||||
|
out << "Lines only in second file: " << std::endl;
|
||||||
|
for (const auto &s: f2diff) {
|
||||||
|
out << s.second << ">" << s.first << std::endl;
|
||||||
|
}
|
||||||
|
out << "^^^\n";
|
||||||
|
return out.str();
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string Diff::diffPercent(const ComparableFile &c1, const ComparableFile &c2) {
|
||||||
|
auto b1 = c1.contents();
|
||||||
|
auto b2 = c2.contents();
|
||||||
|
BuzhashChunker ch1(b1.get(), 512 * 1024, 1024 * 1024, 19, 31);
|
||||||
|
BuzhashChunker ch2(b2.get(), 512 * 1024, 1024 * 1024, 19, 31);
|
||||||
|
std::multiset<std::string> ch1hashes;
|
||||||
|
std::multiset<std::string> ch2diff;
|
||||||
|
std::unordered_map<std::string, unsigned long long> hashsize;
|
||||||
|
for (auto chunkp: ch1) {
|
||||||
|
/// Exit when asked to
|
||||||
|
if (Signals::shouldQuit) throw Exception("Quitting");
|
||||||
|
if (chunkp.second.empty()) continue;
|
||||||
|
std::string md5(chunkp.first.begin(), chunkp.first.end());
|
||||||
|
ch1hashes.emplace(md5);
|
||||||
|
hashsize[md5] = chunkp.second.size();
|
||||||
|
}
|
||||||
|
|
||||||
|
for (auto chunkp: ch2) {
|
||||||
|
/// Exit when asked to
|
||||||
|
if (Signals::shouldQuit) throw Exception("Quitting");
|
||||||
|
if (chunkp.second.empty()) continue;
|
||||||
|
std::string md5(chunkp.first.begin(), chunkp.first.end());
|
||||||
|
hashsize[md5] = chunkp.second.size();
|
||||||
|
if (ch1hashes.count(md5) > 0) ch1hashes.erase(md5);
|
||||||
|
else if (ch1hashes.count(md5) == 0)
|
||||||
|
ch2diff.emplace(md5);
|
||||||
|
}
|
||||||
|
|
||||||
|
unsigned long long diff = 0;
|
||||||
|
|
||||||
|
for (const auto &c: ch1hashes) {
|
||||||
|
diff += hashsize[c];
|
||||||
|
}
|
||||||
|
for (const auto &c: ch2diff) {
|
||||||
|
diff += hashsize[c];
|
||||||
|
}
|
||||||
|
|
||||||
|
return "at most " + BytesFormatter::formatStr(diff);
|
||||||
|
}
|
||||||
38
src/Diff.h
Normal file
38
src/Diff.h
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 06.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_DIFF_H
|
||||||
|
#define SEMBACKUP_DIFF_H
|
||||||
|
|
||||||
|
#include <sstream>
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
#include "change_detectors/ComparableFile.h"
|
||||||
|
|
||||||
|
/// Utility class to compute difference between two ComparableFile%s
|
||||||
|
class Diff {
|
||||||
|
public:
|
||||||
|
/// Compute the difference between two ComparableFile%s
|
||||||
|
/// If the file is binary, calls diffPercent, which outputs the difference between files in bytes
|
||||||
|
/// Otherwise prints linewise difference
|
||||||
|
/// \param c1 Constant reference to the first ComparableFile
|
||||||
|
/// \param c2 Constant reference to the second ComparableFile
|
||||||
|
/// \returns Difference message
|
||||||
|
static std::string diff(const ComparableFile &c1, const ComparableFile &c2);
|
||||||
|
|
||||||
|
/// Calculates the difference between \p c1 amd \p c2 in bytes
|
||||||
|
/// \param c1 Constant reference to the first ComparableFile
|
||||||
|
/// \param c2 Constant reference to the second ComparableFile
|
||||||
|
/// \returns Difference message
|
||||||
|
static std::string diffPercent(const ComparableFile &c1, const ComparableFile &c2);
|
||||||
|
|
||||||
|
/// Checks if a file is binary
|
||||||
|
/// A file is considered binary if its first 2048 bytes contain a null byte
|
||||||
|
/// \param c1 Constant reference to the checked ComparableFile
|
||||||
|
/// \return True if the file is considered binary, false otherwise
|
||||||
|
static bool isBinary(const ComparableFile &c1);
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_DIFF_H
|
||||||
32
src/Exception.cpp
Normal file
32
src/Exception.cpp
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 01.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "Exception.h"
|
||||||
|
#include <execinfo.h>
|
||||||
|
#include <sstream>
|
||||||
|
|
||||||
|
Exception::Exception(const std::string &text) : runtime_error(text + "\n" + getStacktrace()) {}
|
||||||
|
|
||||||
|
Exception::Exception(const char *text) : runtime_error(std::string(text) + "\n" + getStacktrace()) {}
|
||||||
|
|
||||||
|
// Based on: https://www.gnu.org/software/libc/manual/html_node/Backtraces.html
|
||||||
|
std::string Exception::getStacktrace() {
|
||||||
|
std::vector<void *> functions(50);
|
||||||
|
char **strings;
|
||||||
|
int n;
|
||||||
|
|
||||||
|
n = backtrace(functions.data(), 50);
|
||||||
|
strings = backtrace_symbols(functions.data(), n);
|
||||||
|
|
||||||
|
std::stringstream out;
|
||||||
|
|
||||||
|
if (strings != nullptr) {
|
||||||
|
out << "Stacktrace:" << std::endl;
|
||||||
|
for (int i = 0; i < n; i++)
|
||||||
|
out << strings[i] << std::endl;
|
||||||
|
}
|
||||||
|
|
||||||
|
free(strings);
|
||||||
|
return out.str();
|
||||||
|
}
|
||||||
24
src/Exception.h
Normal file
24
src/Exception.h
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 01.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_EXCEPTION_H
|
||||||
|
#define SEMBACKUP_EXCEPTION_H
|
||||||
|
|
||||||
|
#include <stdexcept>
|
||||||
|
#include <string>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
/// Custom exception class that uses execinfo to append a stacktrace to the exception message
|
||||||
|
class Exception : public std::runtime_error {
|
||||||
|
public:
|
||||||
|
Exception(const std::string &text);
|
||||||
|
Exception(const char *text);
|
||||||
|
|
||||||
|
private:
|
||||||
|
/// Static function to get the current stacktrace
|
||||||
|
static std::string getStacktrace();
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_EXCEPTION_H
|
||||||
19
src/Logger.cpp
Normal file
19
src/Logger.cpp
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 23.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "Logger.h"
|
||||||
|
|
||||||
|
Logger::Logger(int level, std::ostream &out) : loglevel(level), out(out) {
|
||||||
|
}
|
||||||
|
|
||||||
|
void Logger::write(const std::string &what, int whatlevel) {
|
||||||
|
if (whatlevel <= loglevel) {
|
||||||
|
std::lock_guard outLock(outM);
|
||||||
|
out.get() << what << std::flush;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void Logger::setLevel(int level) {
|
||||||
|
loglevel = level;
|
||||||
|
}
|
||||||
25
src/Logger.h
Normal file
25
src/Logger.h
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 23.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_LOGGER_H
|
||||||
|
#define SEMBACKUP_LOGGER_H
|
||||||
|
|
||||||
|
#include <functional>
|
||||||
|
#include <iostream>
|
||||||
|
#include <mutex>
|
||||||
|
|
||||||
|
class Logger {
|
||||||
|
public:
|
||||||
|
Logger(int level = 3, std::ostream &out = {std::cout});
|
||||||
|
void write(const std::string &what, int whatlevel);
|
||||||
|
void setLevel(int level);
|
||||||
|
|
||||||
|
private:
|
||||||
|
int loglevel;
|
||||||
|
std::mutex outM;
|
||||||
|
std::reference_wrapper<std::ostream> out;
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_LOGGER_H
|
||||||
57
src/Progress.cpp
Normal file
57
src/Progress.cpp
Normal file
@@ -0,0 +1,57 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 05.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "Progress.h"
|
||||||
|
|
||||||
|
#include <sstream>
|
||||||
|
#include <utility>
|
||||||
|
|
||||||
|
Progress::Progress(std::function<void(std::string, int)> out, std::vector<std::variant<std::function<std::string()>, std::string>> format, const Config &conf, int level) : format(std::move(format)), out(std::move(out)), type(conf.getStr("progress")), progresslevel(level) {
|
||||||
|
if (type != "none") {
|
||||||
|
this->out("\n\n", level);
|
||||||
|
thread = std::thread(&Progress::showProgress, this);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Progress::~Progress() {
|
||||||
|
stop = true;
|
||||||
|
if (thread.joinable())
|
||||||
|
thread.join();
|
||||||
|
}
|
||||||
|
|
||||||
|
void Progress::showProgress() {
|
||||||
|
while (!stop) {
|
||||||
|
std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
||||||
|
{
|
||||||
|
update(std::unique_lock(refreshM));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void Progress::print(const std::string &s, int level) {
|
||||||
|
std::unique_lock refreshL(refreshM);
|
||||||
|
out((type == "pretty" ? "\r\33[2K " : "") + s + "\n", level);
|
||||||
|
update(std::move(refreshL));
|
||||||
|
}
|
||||||
|
|
||||||
|
void Progress::update(std::unique_lock<std::mutex> &&lock) {
|
||||||
|
std::stringstream outs;
|
||||||
|
|
||||||
|
if (type == "pretty")
|
||||||
|
outs << "\r\33[2K ";
|
||||||
|
|
||||||
|
for (auto const &l: format) {
|
||||||
|
if (std::holds_alternative<std::string>(l)) outs << std::get<std::string>(l);
|
||||||
|
else
|
||||||
|
outs << std::get<std::function<std::string()>>(l)();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (type == "pretty")
|
||||||
|
outs << "\r";
|
||||||
|
else
|
||||||
|
outs << "\n";
|
||||||
|
|
||||||
|
out(outs.str(), progresslevel);
|
||||||
|
lock.unlock();
|
||||||
|
}
|
||||||
55
src/Progress.h
Normal file
55
src/Progress.h
Normal file
@@ -0,0 +1,55 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 05.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_PROGRESS_H
|
||||||
|
#define SEMBACKUP_PROGRESS_H
|
||||||
|
|
||||||
|
#include <atomic>
|
||||||
|
#include <condition_variable>
|
||||||
|
#include <functional>
|
||||||
|
#include <mutex>
|
||||||
|
#include <string>
|
||||||
|
#include <thread>
|
||||||
|
#include <variant>
|
||||||
|
|
||||||
|
#include "Config.h"
|
||||||
|
|
||||||
|
/// Class to handle writing progress to the screen
|
||||||
|
class Progress {
|
||||||
|
public:
|
||||||
|
/// Constructs the Progress instance
|
||||||
|
/// \param out Function to call for output
|
||||||
|
/// \param format Format of the progress string, vector of strings or functions that return strings
|
||||||
|
/// \param conf Config, used to specify format (`pretty` for line rewriting, `simple` for normal line printing, or `none`)
|
||||||
|
Progress(std::function<void(std::string, int)> out, std::vector<std::variant<std::function<std::string()>, std::string>> format, const Config &conf, int level = 1);
|
||||||
|
|
||||||
|
Progress &operator=(Progress rhs) = delete;
|
||||||
|
Progress(const Progress &orig) = delete;
|
||||||
|
|
||||||
|
/// Write a string to the terminal without disturbing the progress bar
|
||||||
|
void print(const std::string &s, int level);
|
||||||
|
|
||||||
|
/// Destructor, instructs the worker thread to stop
|
||||||
|
~Progress();
|
||||||
|
|
||||||
|
private:
|
||||||
|
int progresslevel;
|
||||||
|
std::vector<std::variant<std::function<std::string()>, std::string>> format;///< Format of the progressbar
|
||||||
|
std::function<void(std::string, int)> out; ///< Output function
|
||||||
|
|
||||||
|
/// Thread loop function
|
||||||
|
void showProgress();
|
||||||
|
std::atomic<bool> stop = false;///< Stop flag
|
||||||
|
|
||||||
|
std::mutex refreshM;///< Used to prevent mangling the output between print and progressbar update
|
||||||
|
|
||||||
|
/// Prints the progressbar on screen, then unlocks the mutex
|
||||||
|
void update(std::unique_lock<std::mutex> &&lock);
|
||||||
|
const std::string type;///< Progressbar type (Taken from Config)
|
||||||
|
|
||||||
|
std::thread thread;///< Worker thread
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_PROGRESS_H
|
||||||
31
src/RunningAverage.cpp
Normal file
31
src/RunningAverage.cpp
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 05.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "RunningAverage.h"
|
||||||
|
|
||||||
|
RunningAverage::RunningAverage(std::function<unsigned long long int()> getFunc, int max, int ms)
|
||||||
|
: getFunc(std::move(getFunc)), max(max), ms(ms), thread(&RunningAverage::loop, this) {
|
||||||
|
}
|
||||||
|
|
||||||
|
void RunningAverage::loop() {
|
||||||
|
while (!stop) {
|
||||||
|
{
|
||||||
|
std::lock_guard lock(dataLock);
|
||||||
|
data.emplace_front(getFunc());
|
||||||
|
if (data.size() > max) data.pop_back();
|
||||||
|
}
|
||||||
|
std::this_thread::sleep_for(std::chrono::duration(std::chrono::milliseconds(ms)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
RunningAverage::~RunningAverage() {
|
||||||
|
stop = true;
|
||||||
|
thread.join();
|
||||||
|
}
|
||||||
|
|
||||||
|
unsigned long long RunningAverage::get() {
|
||||||
|
std::lock_guard lock(dataLock);
|
||||||
|
if (data.empty()) return 0;
|
||||||
|
return std::accumulate(data.begin(), data.end(), 0UL) / data.size();
|
||||||
|
}
|
||||||
44
src/RunningAverage.h
Normal file
44
src/RunningAverage.h
Normal file
@@ -0,0 +1,44 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 05.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_RUNNINGAVERAGE_H
|
||||||
|
#define SEMBACKUP_RUNNINGAVERAGE_H
|
||||||
|
|
||||||
|
#include <atomic>
|
||||||
|
#include <deque>
|
||||||
|
#include <functional>
|
||||||
|
#include <mutex>
|
||||||
|
#include <numeric>
|
||||||
|
#include <thread>
|
||||||
|
|
||||||
|
/// Class to compute running average of some value
|
||||||
|
class RunningAverage {
|
||||||
|
public:
|
||||||
|
///
|
||||||
|
/// \param getFunc Function that samples the value
|
||||||
|
/// \param max Max number of samples to average
|
||||||
|
/// \param ms Sampling period
|
||||||
|
RunningAverage(std::function<unsigned long long()> getFunc, int max, int ms);
|
||||||
|
|
||||||
|
/// Destructor, instructs the thread to exit
|
||||||
|
~RunningAverage();
|
||||||
|
|
||||||
|
/// Returns the average
|
||||||
|
unsigned long long get();
|
||||||
|
|
||||||
|
private:
|
||||||
|
std::atomic<bool> stop = false; ///< Stop signal
|
||||||
|
std::function<unsigned long long()> getFunc;///< Sampling function
|
||||||
|
std::deque<unsigned long long> data; ///< Data collected
|
||||||
|
int max; ///< Max number of samples
|
||||||
|
int ms; ///< Sampling period
|
||||||
|
std::mutex dataLock; ///< Deque lock
|
||||||
|
std::thread thread; ///< Worker thread
|
||||||
|
|
||||||
|
/// Worker thread loop
|
||||||
|
void loop();
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_RUNNINGAVERAGE_H
|
||||||
20
src/RunningDiffAverage.cpp
Normal file
20
src/RunningDiffAverage.cpp
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 12.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "RunningDiffAverage.h"
|
||||||
|
|
||||||
|
RunningDiffAverage::RunningDiffAverage(std::function<unsigned long long int()> getFunc, int max, int ms)
|
||||||
|
: runningAverage(
|
||||||
|
[this, get = std::move(getFunc)] {
|
||||||
|
auto cur = get();
|
||||||
|
auto calc = cur - prev;
|
||||||
|
prev = cur;
|
||||||
|
return calc;
|
||||||
|
},
|
||||||
|
max, ms) {
|
||||||
|
}
|
||||||
|
|
||||||
|
unsigned long long RunningDiffAverage::get() {
|
||||||
|
return runningAverage.get();
|
||||||
|
}
|
||||||
30
src/RunningDiffAverage.h
Normal file
30
src/RunningDiffAverage.h
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 12.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_RUNNINGDIFFAVERAGE_H
|
||||||
|
#define SEMBACKUP_RUNNINGDIFFAVERAGE_H
|
||||||
|
|
||||||
|
#include <functional>
|
||||||
|
|
||||||
|
#include "RunningAverage.h"
|
||||||
|
|
||||||
|
/// Computes the rolling average of differences between last sampled and currently sampled numbers
|
||||||
|
class RunningDiffAverage {
|
||||||
|
public:
|
||||||
|
///
|
||||||
|
/// \param getFunc Function that samples the value
|
||||||
|
/// \param max Max number of samples to average
|
||||||
|
/// \param ms Sampling period
|
||||||
|
RunningDiffAverage(std::function<unsigned long long()> getFunc, int max, int ms);
|
||||||
|
|
||||||
|
/// Returns the average
|
||||||
|
unsigned long long get();
|
||||||
|
|
||||||
|
private:
|
||||||
|
unsigned long long prev = 0; ///< Previously sampled value
|
||||||
|
RunningAverage runningAverage;///< Backing RunningAverage
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_RUNNINGDIFFAVERAGE_H
|
||||||
12
src/Signals.cpp
Normal file
12
src/Signals.cpp
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 16.04.2023.
|
||||||
|
//
|
||||||
|
#include "Signals.h"
|
||||||
|
|
||||||
|
void Signals::setup() {
|
||||||
|
signal(SIGINT, handle);
|
||||||
|
}
|
||||||
|
|
||||||
|
void Signals::handle(int signum) {
|
||||||
|
shouldQuit = true;
|
||||||
|
}
|
||||||
24
src/Signals.h
Normal file
24
src/Signals.h
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 16.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_SIGNALS_H
|
||||||
|
#define SEMBACKUP_SIGNALS_H
|
||||||
|
|
||||||
|
#include <csignal>
|
||||||
|
|
||||||
|
/// Class to handle signals sent to the process
|
||||||
|
class Signals {
|
||||||
|
public:
|
||||||
|
/// Setup the signal handlers
|
||||||
|
static void setup();
|
||||||
|
|
||||||
|
volatile static inline std::sig_atomic_t shouldQuit = false;///< Indicates whether the program was requested to exit
|
||||||
|
|
||||||
|
private:
|
||||||
|
/// Handle the signals
|
||||||
|
static void handle(int signum);
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_SIGNALS_H
|
||||||
67
src/ThreadPool.cpp
Normal file
67
src/ThreadPool.cpp
Normal file
@@ -0,0 +1,67 @@
|
|||||||
|
#include <string>
|
||||||
|
|
||||||
|
#include "Signals.h"
|
||||||
|
#include "ThreadPool.h"
|
||||||
|
|
||||||
|
ThreadPool::ThreadPool(std::function<void(std::string)> onError, std::size_t workersNum) : onError(std::move(onError)) {
|
||||||
|
for (int i = 0; i < workersNum; i++) threads.emplace_back(&ThreadPool::loop, this);
|
||||||
|
}
|
||||||
|
|
||||||
|
ThreadPool::~ThreadPool() {
|
||||||
|
stop = true;
|
||||||
|
somethingNew.notify_all();
|
||||||
|
for (auto &t: threads) {
|
||||||
|
t.join();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void ThreadPool::push(std::function<void()> &&func) {
|
||||||
|
{
|
||||||
|
std::lock_guard lock(queueLock);
|
||||||
|
queue.push(std::move(func));
|
||||||
|
}
|
||||||
|
somethingNew.notify_one();
|
||||||
|
}
|
||||||
|
|
||||||
|
void ThreadPool::loop() {
|
||||||
|
while (true) {
|
||||||
|
std::unique_lock qLock(queueLock);
|
||||||
|
|
||||||
|
while (queue.empty() && !stop && !Signals::shouldQuit) {
|
||||||
|
// Check for any of the stop signals every second
|
||||||
|
somethingNew.wait_for(qLock, std::chrono::seconds(1));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (stop || Signals::shouldQuit) {
|
||||||
|
// Drop all tasks if requested to exit
|
||||||
|
queue = {};
|
||||||
|
if (queue.empty() && running == 0) { finished.notify_all(); }
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
auto task = std::move(queue.front());
|
||||||
|
|
||||||
|
running++;
|
||||||
|
queue.pop();
|
||||||
|
|
||||||
|
qLock.unlock();
|
||||||
|
|
||||||
|
try {
|
||||||
|
task();
|
||||||
|
} catch (std::exception &e) {
|
||||||
|
onError(std::string(e.what()));
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
std::lock_guard qLock(queueLock);
|
||||||
|
running--;
|
||||||
|
if (queue.empty() && running == 0) { finished.notify_all(); }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool ThreadPool::empty() {
|
||||||
|
std::lock_guard qLock(queueLock);
|
||||||
|
if (queue.empty() && running == 0) return true;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
54
src/ThreadPool.h
Normal file
54
src/ThreadPool.h
Normal file
@@ -0,0 +1,54 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 17.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_THREADPOOL_H
|
||||||
|
#define SEMBACKUP_THREADPOOL_H
|
||||||
|
|
||||||
|
#include <atomic>
|
||||||
|
#include <condition_variable>
|
||||||
|
#include <functional>
|
||||||
|
#include <mutex>
|
||||||
|
#include <queue>
|
||||||
|
#include <thread>
|
||||||
|
#include <utility>
|
||||||
|
|
||||||
|
/// Thread pool
|
||||||
|
/**
|
||||||
|
* Handles ctrl-c via Signals, but it is expected of tasks to also do so
|
||||||
|
* Forwards exception messages to the provided handler
|
||||||
|
*/
|
||||||
|
class ThreadPool {
|
||||||
|
public:
|
||||||
|
/// Constructs a thread pool
|
||||||
|
/// \param onError Callback function that is called when an exception happens when executing a task
|
||||||
|
/// \param workersNum Amount of worker threads (default = number of cpu threads)
|
||||||
|
ThreadPool(std::function<void(std::string)> onError, std::size_t workersNum = std::thread::hardware_concurrency());
|
||||||
|
|
||||||
|
/// Destructor, instructs the threads to stop and joins them
|
||||||
|
~ThreadPool();
|
||||||
|
|
||||||
|
/// Pushes a new task to the queue
|
||||||
|
/// \param func Rvalue to the task functon
|
||||||
|
void push(std::function<void()> &&func);
|
||||||
|
|
||||||
|
/// Returns True if the queue is empty and there are no tasks running
|
||||||
|
bool empty();
|
||||||
|
|
||||||
|
std::mutex finishedLock; ///< Lock to use when waiting on the finished variable
|
||||||
|
std::condition_variable finished;///< Condition variable to wait for all tasks to finish
|
||||||
|
|
||||||
|
private:
|
||||||
|
/// Thread loop
|
||||||
|
void loop();
|
||||||
|
|
||||||
|
std::queue<std::function<void()>> queue; ///< Task queue
|
||||||
|
std::mutex queueLock; ///< Task queue lock
|
||||||
|
std::condition_variable somethingNew; ///< Condition variable to wait for new tasks
|
||||||
|
std::vector<std::thread> threads; ///< Vector of worker threads
|
||||||
|
std::atomic<bool> stop = false; ///< Stop signal for threads
|
||||||
|
std::atomic<int> running = 0; ///< Number of currently running tasks
|
||||||
|
std::function<void(std::string)> onError;///< Function to call on exception in task
|
||||||
|
};
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_THREADPOOL_H
|
||||||
7
src/change_detectors/ChangeDetector.cpp
Normal file
7
src/change_detectors/ChangeDetector.cpp
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 16.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "ChangeDetector.h"
|
||||||
|
|
||||||
|
ChangeDetector::~ChangeDetector() = default;
|
||||||
24
src/change_detectors/ChangeDetector.h
Normal file
24
src/change_detectors/ChangeDetector.h
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 16.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_CHANGEDETECTOR_H
|
||||||
|
#define SEMBACKUP_CHANGEDETECTOR_H
|
||||||
|
|
||||||
|
#include "ComparableFile.h"
|
||||||
|
|
||||||
|
/// An interface for a class comparing any two given ComparableFile%s
|
||||||
|
class ChangeDetector {
|
||||||
|
public:
|
||||||
|
/// Abstract method for comparing two ComparableFile%s
|
||||||
|
/// \param f1 Constant reference to the first ComparableFile
|
||||||
|
/// \param f2 Constant reference to the second ComparableFile
|
||||||
|
/// \return True if these objects are considered *different*, False otherwise
|
||||||
|
virtual bool check(const ComparableFile &f1, const ComparableFile &f2) const = 0;
|
||||||
|
|
||||||
|
/// Default virtual destructor
|
||||||
|
virtual ~ChangeDetector();
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_CHANGEDETECTOR_H
|
||||||
16
src/change_detectors/ChangeDetectorContainer.cpp
Normal file
16
src/change_detectors/ChangeDetectorContainer.cpp
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 04.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "ChangeDetectorContainer.h"
|
||||||
|
|
||||||
|
#include <functional>
|
||||||
|
|
||||||
|
bool ChangeDetectorContainer::check(const ComparableFile &f1, const ComparableFile &f2) const {
|
||||||
|
return std::any_of(changeDetectors.begin(), changeDetectors.end(),
|
||||||
|
[&](const auto &changeDetector) {
|
||||||
|
return changeDetector->check(f1, f2);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
ChangeDetectorContainer::ChangeDetectorContainer(std::vector<std::unique_ptr<ChangeDetector>> &&changeDetectors) : changeDetectors(std::move(changeDetectors)) {}
|
||||||
33
src/change_detectors/ChangeDetectorContainer.h
Normal file
33
src/change_detectors/ChangeDetectorContainer.h
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 04.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_CHANGEDETECTORCONTAINER_H
|
||||||
|
#define SEMBACKUP_CHANGEDETECTORCONTAINER_H
|
||||||
|
|
||||||
|
#include <memory>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
#include "ChangeDetector.h"
|
||||||
|
#include "ComparableFile.h"
|
||||||
|
|
||||||
|
/// Wrapper for multiple ChangeDetector%s
|
||||||
|
/** A ChangeDetector implementation that serves as a convenience wrapper for
|
||||||
|
* multiple ChangeDetector%s, its check returns true if any of the wrapped ChangeDetector%s return true
|
||||||
|
*/
|
||||||
|
class ChangeDetectorContainer : public ChangeDetector {
|
||||||
|
public:
|
||||||
|
/// Constructs a ChangeDetectorContainer using a vector of existing ChangeDetector%s
|
||||||
|
/// \param changeDetectors An rvalue reference to a vector of unique pointers of ChangeDetector
|
||||||
|
ChangeDetectorContainer(std::vector<std::unique_ptr<ChangeDetector>> &&changeDetectors);
|
||||||
|
|
||||||
|
/// \copydoc ChangeDetector::check
|
||||||
|
/// \return ComparableFile%s are considered different if any of the wrapped ChangeDetector%s return true
|
||||||
|
bool check(const ComparableFile &f1, const ComparableFile &f2) const override;
|
||||||
|
|
||||||
|
private:
|
||||||
|
std::vector<std::unique_ptr<ChangeDetector>> changeDetectors;
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_CHANGEDETECTORCONTAINER_H
|
||||||
35
src/change_detectors/ChangeDetectorFactory.cpp
Normal file
35
src/change_detectors/ChangeDetectorFactory.cpp
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 16.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "ChangeDetectorFactory.h"
|
||||||
|
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
#include "../Exception.h"
|
||||||
|
#include "ContentsChangeDetector.h"
|
||||||
|
#include "EditTimeChangeDetector.h"
|
||||||
|
#include "SizeChangeDetector.h"
|
||||||
|
#include "TypeChangeDetector.h"
|
||||||
|
|
||||||
|
std::unique_ptr<ChangeDetector> ChangeDetectorFactory::getChangeDetector(const std::string &type) {
|
||||||
|
if (type == "etime") {
|
||||||
|
return std::make_unique<EditTimeChangeDetector>();
|
||||||
|
} else if (type == "size") {
|
||||||
|
return std::make_unique<SizeChangeDetector>();
|
||||||
|
} else if (type == "type") {
|
||||||
|
return std::make_unique<TypeChangeDetector>();
|
||||||
|
} else if (type == "contents") {
|
||||||
|
return std::make_unique<ContentsChangeDetector>();
|
||||||
|
} else
|
||||||
|
throw Exception("Unknown ChangeDetector type " + type);
|
||||||
|
}
|
||||||
|
|
||||||
|
ChangeDetectorContainer ChangeDetectorFactory::getChangeDetectors(const Config &config) {
|
||||||
|
std::vector<std::unique_ptr<ChangeDetector>> changeDetectors;
|
||||||
|
for (auto const &i: config.getList("change-detectors")) {
|
||||||
|
changeDetectors.emplace_back(ChangeDetectorFactory::getChangeDetector(i));
|
||||||
|
}
|
||||||
|
|
||||||
|
return ChangeDetectorContainer(std::move(changeDetectors));
|
||||||
|
}
|
||||||
33
src/change_detectors/ChangeDetectorFactory.h
Normal file
33
src/change_detectors/ChangeDetectorFactory.h
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 16.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_CHANGEDETECTORFACTORY_H
|
||||||
|
#define SEMBACKUP_CHANGEDETECTORFACTORY_H
|
||||||
|
|
||||||
|
#include <memory>
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
#include "../Config.h"
|
||||||
|
#include "ChangeDetector.h"
|
||||||
|
#include "ChangeDetectorContainer.h"
|
||||||
|
|
||||||
|
/// Factory class for ChangeDetector
|
||||||
|
/** Can create either a vector of ChangeDetector%s according to Config,
|
||||||
|
* or an individual ChangeDetector from a type string
|
||||||
|
*/
|
||||||
|
class ChangeDetectorFactory {
|
||||||
|
public:
|
||||||
|
/// Creates a ChangeDetector of given type and returns an unique pointer to it
|
||||||
|
/// \param type Constant reference to a string containing type of the ChangeDetector to create
|
||||||
|
/// \return Unique pointer to constructed ChangeDetector
|
||||||
|
static std::unique_ptr<ChangeDetector> getChangeDetector(const std::string &type);
|
||||||
|
|
||||||
|
/// Constructs a vector of unique pointers to ChangeDetector%s according to the given \p config
|
||||||
|
/// \param config Config with comma-separated "change-detectors" option set, for each entry a ChangeDetector will be created
|
||||||
|
/// \return A vector of unique pointers to ChangeDetector%s constructed according to \p config
|
||||||
|
static ChangeDetectorContainer getChangeDetectors(const Config &config);
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_CHANGEDETECTORFACTORY_H
|
||||||
42
src/change_detectors/ComparableFile.cpp
Normal file
42
src/change_detectors/ComparableFile.cpp
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 05.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
|
||||||
|
#include "ComparableFile.h"
|
||||||
|
|
||||||
|
#include <fstream>
|
||||||
|
#include <sstream>
|
||||||
|
|
||||||
|
#include "../Exception.h"
|
||||||
|
#include "../repo/objects/FileBuffer.h"
|
||||||
|
|
||||||
|
ComparableFile::ComparableFile(const File &file, const Repository *repo)
|
||||||
|
: path(file.name),
|
||||||
|
type(file.fileType),
|
||||||
|
bytes(file.bytes),
|
||||||
|
mtime(file.mtime),
|
||||||
|
contents(
|
||||||
|
[file, repo]() {
|
||||||
|
return std::make_unique<FileBuffer>(repo, file.id);
|
||||||
|
}) {}
|
||||||
|
|
||||||
|
ComparableFile::ComparableFile(const std::filesystem::path &p, const std::filesystem::path &base)
|
||||||
|
: path(p.lexically_relative(base).u8string()),
|
||||||
|
type(File::getFileType(p)),
|
||||||
|
bytes(File::getFileSize(p)),
|
||||||
|
mtime(File::getFileMtime(p)),
|
||||||
|
contents(
|
||||||
|
[p, path = this->path, type = this->type]() -> std::unique_ptr<std::streambuf> {
|
||||||
|
if (type == File::Type::Normal) {
|
||||||
|
auto fb = std::make_unique<std::filebuf>();
|
||||||
|
fb->open(p, std::ios::in | std::ios::binary);
|
||||||
|
if (!fb->is_open()) throw Exception("Can't open " + p.u8string() + " for reading!");
|
||||||
|
return fb;
|
||||||
|
}
|
||||||
|
|
||||||
|
auto contentsVector = File::getFileContents(p);
|
||||||
|
std::string contents = {contentsVector.begin(), contentsVector.end()};
|
||||||
|
|
||||||
|
return std::make_unique<std::stringbuf>(contents, std::ios::in | std::ios::binary);
|
||||||
|
}) {}
|
||||||
43
src/change_detectors/ComparableFile.h
Normal file
43
src/change_detectors/ComparableFile.h
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 05.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_COMPARABLEFILE_H
|
||||||
|
#define SEMBACKUP_COMPARABLEFILE_H
|
||||||
|
|
||||||
|
#include <filesystem>
|
||||||
|
#include <functional>
|
||||||
|
#include <streambuf>
|
||||||
|
|
||||||
|
#include "../repo/Repository.h"
|
||||||
|
#include "../repo/objects/File.h"
|
||||||
|
|
||||||
|
/// Helper class to allow comparing files from different sources
|
||||||
|
/**
|
||||||
|
* As we are required to allow comparisons between a File in a repository and a file in filesystem,
|
||||||
|
* comparisons between two files that are already in a Repository,
|
||||||
|
* and between File%s that are in a repository cache and between files in the filesystem (when making backups),
|
||||||
|
* this helper class exists to provide a uniform interface to be used when calling ChangeDetector%s.
|
||||||
|
*/
|
||||||
|
struct ComparableFile {
|
||||||
|
/// Constructs a ComparableFile based on a File in a Repository
|
||||||
|
/// The resulting ComparableFile will have a #contents function that returns an instance of FileBuffer for given \p file
|
||||||
|
/// \param file Constant reference to a File object
|
||||||
|
/// \param repo Constant pointer to Repository from which the File object was taken, must be valid during the lifetime of created ComparableFile
|
||||||
|
ComparableFile(const File &file, const Repository *repo);
|
||||||
|
|
||||||
|
/// Constructs a ComparableFile based on a file in the filesystem
|
||||||
|
/// The resulting ComparableFile will have a #contents function that returns an instance of std::filebuf for file at given path
|
||||||
|
/// \param p Constant reference to an absolute path to the file
|
||||||
|
/// \param base Constant reference to a base path against which #path will be set
|
||||||
|
ComparableFile(const std::filesystem::path &p, const std::filesystem::path &base);
|
||||||
|
|
||||||
|
const std::string path; ///< Relative path to the file
|
||||||
|
const File::Type type; ///< File type
|
||||||
|
const unsigned long long bytes; ///< Number of bytes in the file
|
||||||
|
const unsigned long long mtime; ///< Timestamp of last file modification
|
||||||
|
const std::function<std::unique_ptr<std::streambuf>()> contents;///< Function that returns a unique pointer to a std::streambuf instance linked to the contents of the file
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_COMPARABLEFILE_H
|
||||||
19
src/change_detectors/ContentsChangeDetector.cpp
Normal file
19
src/change_detectors/ContentsChangeDetector.cpp
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 05.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "ContentsChangeDetector.h"
|
||||||
|
|
||||||
|
#include <iterator>
|
||||||
|
|
||||||
|
bool ContentsChangeDetector::check(const ComparableFile &f1, const ComparableFile &f2) const {
|
||||||
|
if (f1.type != f2.type) return true;
|
||||||
|
|
||||||
|
auto b1 = f1.contents();
|
||||||
|
auto b2 = f2.contents();
|
||||||
|
|
||||||
|
return !std::equal(std::istreambuf_iterator<char>(b1.get()),
|
||||||
|
std::istreambuf_iterator<char>(),
|
||||||
|
std::istreambuf_iterator<char>(b2.get()),
|
||||||
|
std::istreambuf_iterator<char>());
|
||||||
|
}
|
||||||
19
src/change_detectors/ContentsChangeDetector.h
Normal file
19
src/change_detectors/ContentsChangeDetector.h
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 05.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_CONTENTSCHANGEDETECTOR_H
|
||||||
|
#define SEMBACKUP_CONTENTSCHANGEDETECTOR_H
|
||||||
|
|
||||||
|
#include "ChangeDetector.h"
|
||||||
|
|
||||||
|
/// A ChangeDetector implementation that compares two files by their contents
|
||||||
|
class ContentsChangeDetector : public ChangeDetector {
|
||||||
|
public:
|
||||||
|
/// \copydoc ChangeDetector::check
|
||||||
|
/// \return ComparableFile%s are considered different if their contents are different
|
||||||
|
bool check(const ComparableFile &f1, const ComparableFile &f2) const override;
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_CONTENTSCHANGEDETECTOR_H
|
||||||
9
src/change_detectors/EditTimeChangeDetector.cpp
Normal file
9
src/change_detectors/EditTimeChangeDetector.cpp
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 16.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "EditTimeChangeDetector.h"
|
||||||
|
|
||||||
|
bool EditTimeChangeDetector::check(const ComparableFile &f1, const ComparableFile &f2) const {
|
||||||
|
return f1.mtime != f2.mtime;
|
||||||
|
}
|
||||||
20
src/change_detectors/EditTimeChangeDetector.h
Normal file
20
src/change_detectors/EditTimeChangeDetector.h
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 16.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_EDITTIMECHANGEDETECTOR_H
|
||||||
|
#define SEMBACKUP_EDITTIMECHANGEDETECTOR_H
|
||||||
|
|
||||||
|
|
||||||
|
#include "ChangeDetector.h"
|
||||||
|
|
||||||
|
/// A ChangeDetector implementation that compares two files by their modification time
|
||||||
|
class EditTimeChangeDetector : public ChangeDetector {
|
||||||
|
public:
|
||||||
|
/// \copydoc ChangeDetector::check
|
||||||
|
/// \return ComparableFile%s are considered different if their modification times are different
|
||||||
|
bool check(const ComparableFile &f1, const ComparableFile &f2) const override;
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_EDITTIMECHANGEDETECTOR_H
|
||||||
9
src/change_detectors/SizeChangeDetector.cpp
Normal file
9
src/change_detectors/SizeChangeDetector.cpp
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 16.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "SizeChangeDetector.h"
|
||||||
|
|
||||||
|
bool SizeChangeDetector::check(const ComparableFile &f1, const ComparableFile &f2) const {
|
||||||
|
return f1.bytes != f2.bytes;
|
||||||
|
}
|
||||||
19
src/change_detectors/SizeChangeDetector.h
Normal file
19
src/change_detectors/SizeChangeDetector.h
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 16.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_SIZECHANGEDETECTOR_H
|
||||||
|
#define SEMBACKUP_SIZECHANGEDETECTOR_H
|
||||||
|
|
||||||
|
#include "ChangeDetector.h"
|
||||||
|
|
||||||
|
/// A ChangeDetector implementation that compares two files by their size
|
||||||
|
class SizeChangeDetector : public ChangeDetector {
|
||||||
|
public:
|
||||||
|
/// \copydoc ChangeDetector::check
|
||||||
|
/// \return ComparableFile%s are considered different if their sizes are different
|
||||||
|
bool check(const ComparableFile &f1, const ComparableFile &f2) const override;
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_SIZECHANGEDETECTOR_H
|
||||||
9
src/change_detectors/TypeChangeDetector.cpp
Normal file
9
src/change_detectors/TypeChangeDetector.cpp
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 12.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "TypeChangeDetector.h"
|
||||||
|
|
||||||
|
bool TypeChangeDetector::check(const ComparableFile &f1, const ComparableFile &f2) const {
|
||||||
|
return f1.type != f2.type;
|
||||||
|
}
|
||||||
19
src/change_detectors/TypeChangeDetector.h
Normal file
19
src/change_detectors/TypeChangeDetector.h
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 12.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_TYPECHANGEDETECTOR_H
|
||||||
|
#define SEMBACKUP_TYPECHANGEDETECTOR_H
|
||||||
|
|
||||||
|
#include "ChangeDetector.h"
|
||||||
|
|
||||||
|
/// A ChangeDetector implementation that compares two files by their type
|
||||||
|
class TypeChangeDetector : public ChangeDetector {
|
||||||
|
public:
|
||||||
|
/// \copydoc ChangeDetector::check
|
||||||
|
/// \return ComparableFile%s are considered different if their types are different
|
||||||
|
bool check(const ComparableFile &f1, const ComparableFile &f2) const override;
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_TYPECHANGEDETECTOR_H
|
||||||
34
src/chunkers/Buzhash.cpp
Normal file
34
src/chunkers/Buzhash.cpp
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 26.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "Buzhash.h"
|
||||||
|
|
||||||
|
Buzhash::Buzhash(uint32_t blockSize) : blockSize(blockSize), history() {}
|
||||||
|
|
||||||
|
uint32_t Buzhash::get() const {
|
||||||
|
return cur;
|
||||||
|
}
|
||||||
|
|
||||||
|
uint32_t Buzhash::feed(uint8_t in) {
|
||||||
|
cur = rotr32(cur, 1);
|
||||||
|
|
||||||
|
if (history.size() >= blockSize) {
|
||||||
|
auto oldest = history.back();
|
||||||
|
history.pop_back();
|
||||||
|
cur ^= rotr32(randomNumbers[oldest], blockSize);
|
||||||
|
}
|
||||||
|
|
||||||
|
history.emplace_front(in);
|
||||||
|
|
||||||
|
cur ^= randomNumbers[in];
|
||||||
|
|
||||||
|
return cur;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Circular shift taken from: https://en.wikipedia.org/wiki/Circular_shift
|
||||||
|
uint32_t Buzhash::rotr32(uint32_t value, unsigned int count) {
|
||||||
|
const unsigned int mask = CHAR_BIT * sizeof(value) - 1;
|
||||||
|
count &= mask;
|
||||||
|
return (value >> count) | (value << (-count & mask));
|
||||||
|
}
|
||||||
85
src/chunkers/Buzhash.h
Normal file
85
src/chunkers/Buzhash.h
Normal file
@@ -0,0 +1,85 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 26.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_BUZHASH_H
|
||||||
|
#define SEMBACKUP_BUZHASH_H
|
||||||
|
|
||||||
|
#include <array>
|
||||||
|
#include <climits>
|
||||||
|
#include <cstdint>
|
||||||
|
#include <deque>
|
||||||
|
|
||||||
|
/// Cyclic polynomial rolling hash
|
||||||
|
/** Based on: http://www.serve.net/buz/hash.adt/java.002.html
|
||||||
|
* https://github.com/silvasur/buzhash/blob/master/hash.go
|
||||||
|
* https://en.wikipedia.org/wiki/Rolling_hash#Cyclic_polynomial
|
||||||
|
*/
|
||||||
|
class Buzhash {
|
||||||
|
public:
|
||||||
|
/// Constructs a new Buzhash instance
|
||||||
|
/// \param blockSize Rolling hash window
|
||||||
|
Buzhash(uint32_t blockSize);
|
||||||
|
|
||||||
|
/// Returns current hash value
|
||||||
|
uint32_t get() const;
|
||||||
|
|
||||||
|
/// Adds \p in to the hash
|
||||||
|
/// \param in Byte to add
|
||||||
|
/// \return New hash value
|
||||||
|
uint32_t feed(uint8_t in);
|
||||||
|
|
||||||
|
private:
|
||||||
|
uint32_t cur = 0; ///< Current hash value
|
||||||
|
const uint32_t blockSize; ///< Hashing window size
|
||||||
|
std::deque<uint32_t> history;///< Bytes used to calculate current hash, used to compute the hash in a rolling fashion (to remove the oldest byte from the hash when blockSize is reached)
|
||||||
|
|
||||||
|
// Circular shift taken from: https://en.wikipedia.org/wiki/Circular_shift
|
||||||
|
/// Shift \p value \p count bits to the right circularly
|
||||||
|
/// \param value Value to shift
|
||||||
|
/// \param count By how many bytes
|
||||||
|
/// \return Shifted value
|
||||||
|
static uint32_t rotr32(uint32_t value, unsigned int count);
|
||||||
|
|
||||||
|
/// 256 32-bit random numbers used for hashing
|
||||||
|
/// Ideally, should have an equal distribution of 0s and 1s, but I didn't bother checking it
|
||||||
|
// clang-format off
|
||||||
|
static constexpr std::array<uint32_t, 256> randomNumbers{
|
||||||
|
0x827f934c, 0xebcd9924, 0x667fdea2, 0x8a8b0997, 0x42af49e8, 0x556cb313, 0x505da41b, 0xb23be60f,
|
||||||
|
0xc3901be4, 0xee1d8d4d, 0x4d59795c, 0x8d542ba4, 0x043f073c, 0x2af19a39, 0xb2c4aa36, 0x6e30ff43,
|
||||||
|
0x77ad3ef7, 0xd4c077e5, 0x3a1155aa, 0x866b07d3, 0xc16022b2, 0x6d4dad6e, 0x7a69c6dd, 0xd436dc23,
|
||||||
|
0x32b64948, 0x1f72475f, 0x129be871, 0x05d46f6e, 0x7e405cd5, 0x31fdd272, 0x84a56b1a, 0xeaf43633,
|
||||||
|
0x5f8148d4, 0x6d4bf6d9, 0xc2b4dbd7, 0xaa804cc7, 0xcb3de5ca, 0x6503cdb3, 0xa3c6d727, 0x20e2f098,
|
||||||
|
0xd525bb67, 0x37b1b81e, 0xc1f1fd79, 0x4fe91240, 0x6a4ea716, 0x71245e33, 0xdbaab854, 0xfc24600e,
|
||||||
|
0xd72dc72f, 0x2d7139ae, 0x075fb38d, 0xb18028a5, 0x9970d103, 0x235ec64b, 0x68645255, 0x352945f0,
|
||||||
|
0x7a4b19a1, 0xe17df5f5, 0x676a6644, 0x75aad7aa, 0x63bdfc9a, 0x607586c7, 0x1546400e, 0xfe582141,
|
||||||
|
0xb50a199f, 0xb0769910, 0x5d74ab3b, 0x2404799b, 0xa66a3a78, 0x1b6e24aa, 0x630674cc, 0x3272fea4,
|
||||||
|
0xd4e9e078, 0xe586d12a, 0x579f8b98, 0xfd16bcb5, 0xd1e4faee, 0xe30953c7, 0x3ac73f87, 0xab66983f,
|
||||||
|
0x5fe12f90, 0x10952ef1, 0x5c7ac32a, 0x89ccd941, 0xb82c3fa9, 0xacd374e5, 0x50984746, 0x09f082e8,
|
||||||
|
0x11ee3b91, 0x31764e3a, 0xb59df38a, 0x67e94f2d, 0xcceaca68, 0xc68a89d8, 0x5f2e80ac, 0xd5556741,
|
||||||
|
0x8c815df6, 0xde71c2b5, 0x7b1f5c49, 0xd64682a4, 0x4fb59748, 0x4968707f, 0x909c0c1a, 0x5f1dd608,
|
||||||
|
0x1c601e37, 0x96e01ada, 0xc5582ef8, 0xae6834c1, 0xbe63b0ce, 0xab2aea9f, 0xf13e77c2, 0xe433350b,
|
||||||
|
0x17a24a33, 0xc1f31bb6, 0xa23e9de4, 0x7e28ef69, 0x23e0ef42, 0x0796e53f, 0xf9e3045d, 0x7bbacd31,
|
||||||
|
0xa48bee27, 0x15f3c3b3, 0x4c320cb4, 0x916429d9, 0xa15ccb3c, 0x82a4a23c, 0xb0cc6a4a, 0xcf8d93fa,
|
||||||
|
0x3b18b937, 0xad0488e4, 0xaa568114, 0x80b9b8c7, 0x8f3a9071, 0x818b790d, 0x99c8dbf2, 0x0d23b2a4,
|
||||||
|
0x74c81a28, 0x1aa65d76, 0x7168ee7d, 0xc0d40b6c, 0x77c70a0c, 0xd3752839, 0xc2f7981c, 0x83767124,
|
||||||
|
0xb881618f, 0xb263d8cf, 0xbbb40400, 0xdb9702eb, 0xaccad841, 0x806af5a7, 0x16f096e3, 0x64bf45d9,
|
||||||
|
0x5f7c0a58, 0xdac0c665, 0x1dbebaac, 0xb97027a6, 0xfc934433, 0xfc7b2d06, 0x8871fe4e, 0x0df24135,
|
||||||
|
0x6ddf7cc8, 0x32e0d1cd, 0xe88abedd, 0x214af930, 0x90990f97, 0xc7691171, 0xbf7b6ca3, 0x8af6589c,
|
||||||
|
0x452c8ee0, 0xbc2c5891, 0xcf8d13b4, 0x698d1f1f, 0x802a011a, 0x19820708, 0x25c79d2f, 0xedf91253,
|
||||||
|
0xc93fe5dd, 0xa03a117b, 0x10912ae7, 0xc90d59d0, 0xc3522549, 0x3e4f3e81, 0x494ae40f, 0x2d157b6e,
|
||||||
|
0xd7bf06b2, 0x19c5bb2a, 0xa869261c, 0xa80cfd2c, 0x1ea7c6ec, 0x1b36a51f, 0x8bd227cc, 0xad2d2260,
|
||||||
|
0x181258c3, 0xbd253a58, 0x3273f94b, 0x9c315309, 0xb2d8d3e3, 0x11ec35a8, 0x384e6475, 0x855a9009,
|
||||||
|
0x854cc06a, 0xe7408809, 0xe583ce2a, 0x895fb756, 0x6a8a2072, 0x6598a92b, 0x530f41bb, 0xb1bd57f1,
|
||||||
|
0x62d57fa0, 0xe6505776, 0x42fcfe4d, 0x0fbdf1ee, 0x8e3104c4, 0xf11c8a65, 0x5bc51ad9, 0x5f1f8ce9,
|
||||||
|
0xab179a87, 0xd5448444, 0x7bd4a26b, 0x658f1963, 0x86db95b8, 0xaba6734e, 0x486fddea, 0x859c3e0b,
|
||||||
|
0xebce0106, 0x99c3014e, 0xc151b942, 0x9604aad8, 0xf6ce654b, 0xa1e7982e, 0xf6d8ed14, 0xd4bdf7e2,
|
||||||
|
0x13696254, 0x05ec638c, 0x306dbc29, 0x1676eb60, 0xadbf3ce3, 0x966dde56, 0x6d5bea46, 0x719aa10d,
|
||||||
|
0x0e65093d, 0x0b1a3c43, 0x0321ea8c, 0xe0ef2cbd, 0x43432ee3, 0x3e62046d, 0x425e7b44, 0x892e119c,
|
||||||
|
0xfdec4de5, 0x48c5dd6c, 0x79e6bfcd, 0x8d53372e, 0xe96f6d32, 0x52cddacd, 0x3e99e0eb, 0xa9e5d28f,
|
||||||
|
};
|
||||||
|
// clang-format on
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_BUZHASH_H
|
||||||
42
src/chunkers/BuzhashChunker.cpp
Normal file
42
src/chunkers/BuzhashChunker.cpp
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 26.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "BuzhashChunker.h"
|
||||||
|
|
||||||
|
#include "../Exception.h"
|
||||||
|
#include "../crypto/MD5.h"
|
||||||
|
|
||||||
|
BuzhashChunker::BuzhashChunker(std::streambuf *buf, unsigned long long minBytes, unsigned long long maxBytes, unsigned long long mask, uint32_t window) : Chunker(buf, maxBytes), window(window), minBytes(minBytes), mask(mask), buzhash(window) {}
|
||||||
|
|
||||||
|
std::pair<std::string, std::vector<char>> BuzhashChunker::getNext() {
|
||||||
|
if (eof) throw Exception("Trying to read from a file that is finished!");
|
||||||
|
std::vector<char> rbuf(minBytes);
|
||||||
|
|
||||||
|
auto read = static_cast<unsigned long>(buf->sgetn(rbuf.data(), (long) minBytes));
|
||||||
|
|
||||||
|
if (read != minBytes) {
|
||||||
|
eof = true;
|
||||||
|
rbuf.resize(read);
|
||||||
|
return {MD5::calculate(rbuf), rbuf};
|
||||||
|
}
|
||||||
|
|
||||||
|
for (auto c: rbuf) {
|
||||||
|
buzhash.feed(static_cast<uint8_t>(c));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Continue reading the file until either the last mask bits are zero of we exceed the maxSize
|
||||||
|
while (((buzhash.get() & (~0UL >> (sizeof(unsigned long long) * 8 - mask))) != 0) && rbuf.size() < maxBytes) {
|
||||||
|
auto r = buf->sbumpc();
|
||||||
|
if (r == std::streambuf::traits_type::eof()) {
|
||||||
|
eof = true;
|
||||||
|
break;
|
||||||
|
} else {
|
||||||
|
char c = std::streambuf::traits_type::to_char_type(r);
|
||||||
|
rbuf.emplace_back(c);
|
||||||
|
buzhash.feed(static_cast<uint8_t>(c));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {MD5::calculate(rbuf), rbuf};
|
||||||
|
}
|
||||||
34
src/chunkers/BuzhashChunker.h
Normal file
34
src/chunkers/BuzhashChunker.h
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 26.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_BUZHASHCHUNKER_H
|
||||||
|
#define SEMBACKUP_BUZHASHCHUNKER_H
|
||||||
|
|
||||||
|
#include <streambuf>
|
||||||
|
|
||||||
|
#include "Buzhash.h"
|
||||||
|
#include "Chunker.h"
|
||||||
|
|
||||||
|
/// Chunker implementation using rolling hash
|
||||||
|
class BuzhashChunker : public Chunker {
|
||||||
|
public:
|
||||||
|
/// Constructs a BuzhashChunker
|
||||||
|
/// \copydoc Chunker::Chunker
|
||||||
|
/// \param minBytes Minimum amount of bytes in returned chunks
|
||||||
|
/// \param mask Amount of trailing zeroes in the rolling hash at which the file is cut (results in average chunk size of 2^mask bytes)
|
||||||
|
/// \param window Rolling hash window (how many of chunks last bytes are included in the hash, the default is recommended)
|
||||||
|
BuzhashChunker(std::streambuf *buf, unsigned long long minBytes, unsigned long long maxBytes, unsigned long long mask, uint32_t window = 4095);
|
||||||
|
|
||||||
|
/// \copydoc Chunker::getNext
|
||||||
|
std::pair<std::string, std::vector<char>> getNext() override;
|
||||||
|
|
||||||
|
private:
|
||||||
|
const unsigned long long window; ///< Rolling hash window
|
||||||
|
const unsigned long long minBytes;///< Minimum amount of bytes in returned chunks
|
||||||
|
const unsigned long long mask; ///< Amount of trailing zeroes in the rolling hash at which the file is cut
|
||||||
|
Buzhash buzhash; ///< Hasher instance
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_BUZHASHCHUNKER_H
|
||||||
51
src/chunkers/Chunker.cpp
Normal file
51
src/chunkers/Chunker.cpp
Normal file
@@ -0,0 +1,51 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 15.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "Chunker.h"
|
||||||
|
|
||||||
|
#include "../Exception.h"
|
||||||
|
|
||||||
|
Chunker::Chunker(std::streambuf *buf, unsigned long long maxBytes) : buf(buf), maxBytes(maxBytes) {}
|
||||||
|
|
||||||
|
bool Chunker::getEof() const {
|
||||||
|
return eof;
|
||||||
|
}
|
||||||
|
|
||||||
|
Chunker::~Chunker() = default;
|
||||||
|
|
||||||
|
Chunker::ChunkerIterator Chunker::begin() {
|
||||||
|
return {this};
|
||||||
|
}
|
||||||
|
|
||||||
|
Chunker::ChunkerIterator Chunker::end() {
|
||||||
|
return {nullptr};
|
||||||
|
}
|
||||||
|
|
||||||
|
Chunker::ChunkerIterator &Chunker::ChunkerIterator::operator++() {
|
||||||
|
if (pastEOF) throw Exception("Trying to increment pastEOF ChunkerIterator!");
|
||||||
|
if (source->getEof())
|
||||||
|
pastEOF = true;
|
||||||
|
else
|
||||||
|
buf = source->getNext();
|
||||||
|
return *this;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool Chunker::ChunkerIterator::operator!=(const Chunker::ChunkerIterator &rhs) const {
|
||||||
|
return pastEOF != rhs.pastEOF;
|
||||||
|
}
|
||||||
|
|
||||||
|
Chunker::ChunkerIterator::value_type Chunker::ChunkerIterator::operator*() const {
|
||||||
|
if (pastEOF) throw Exception("Trying to dereference pastEOF ChunkerIterator!");
|
||||||
|
return buf.value();
|
||||||
|
}
|
||||||
|
|
||||||
|
bool Chunker::ChunkerIterator::operator==(const Chunker::ChunkerIterator &rhs) const {
|
||||||
|
return pastEOF == rhs.pastEOF;
|
||||||
|
}
|
||||||
|
|
||||||
|
Chunker::ChunkerIterator::ChunkerIterator(Chunker *source)
|
||||||
|
: source(source), pastEOF(source == nullptr) {
|
||||||
|
if (source)
|
||||||
|
operator++();
|
||||||
|
}
|
||||||
74
src/chunkers/Chunker.h
Normal file
74
src/chunkers/Chunker.h
Normal file
@@ -0,0 +1,74 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 15.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_CHUNKER_H
|
||||||
|
#define SEMBACKUP_CHUNKER_H
|
||||||
|
|
||||||
|
#include <array>
|
||||||
|
#include <optional>
|
||||||
|
#include <streambuf>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
/// Abstract base class for a Chunker that takes a file and splits it into chunks to be backed up
|
||||||
|
class Chunker {
|
||||||
|
private:
|
||||||
|
/// Convenience iterator to allow using Chunker%s in range for loops
|
||||||
|
struct ChunkerIterator {
|
||||||
|
using value_type = std::pair<std::string, std::vector<char>>;
|
||||||
|
|
||||||
|
/// Creates a ChunkerIterator pointing to the first chunk or past-EOF
|
||||||
|
/// \param source Pointer to a Chunker, should be available during the entire iterator lifetime, or nullptr if this is pastEOF iterator
|
||||||
|
ChunkerIterator(Chunker *source);
|
||||||
|
|
||||||
|
/// Increments the iterator to the next chunk, or past-EOF
|
||||||
|
/// \throws Exception if iterator points past-EOF
|
||||||
|
ChunkerIterator &operator++();
|
||||||
|
|
||||||
|
/// Returns the current pointed-to chunk
|
||||||
|
/// \throws Exception if iterator points past-EOF
|
||||||
|
value_type operator*() const;
|
||||||
|
|
||||||
|
/// Returns true if both iterators are past-EOF
|
||||||
|
bool operator==(const ChunkerIterator &rhs) const;
|
||||||
|
|
||||||
|
/// Returns false if both iterators are past-EOF
|
||||||
|
bool operator!=(const ChunkerIterator &rhs) const;
|
||||||
|
|
||||||
|
private:
|
||||||
|
Chunker *const source; ///< Pointer to the underlying Chunker
|
||||||
|
std::optional<value_type> buf;///< Currently pointed to chunk
|
||||||
|
bool pastEOF = false; ///< Whether past EOF has been reached
|
||||||
|
};
|
||||||
|
|
||||||
|
public:
|
||||||
|
/// Returns the next chunk of the file
|
||||||
|
/// Returns a single empty chunk if a file is empty
|
||||||
|
/// \return Pair consisting of chunk's bytes and its MD5 hash
|
||||||
|
/// \throws Exception if EOF was already reached
|
||||||
|
virtual std::pair<std::string, std::vector<char>> getNext() = 0;
|
||||||
|
|
||||||
|
/// Returns True if EOF was reached, False otherwise
|
||||||
|
bool getEof() const;
|
||||||
|
|
||||||
|
/// Default virtual destructor
|
||||||
|
virtual ~Chunker();
|
||||||
|
|
||||||
|
/// Returns a ChunkerIterator pointing to the first chunk in a file
|
||||||
|
ChunkerIterator begin();
|
||||||
|
|
||||||
|
/// Returns a past-EOF ChunkerIterator
|
||||||
|
static ChunkerIterator end();
|
||||||
|
|
||||||
|
protected:
|
||||||
|
/// \param buf Pointer to a std::streambuf, should be available during the entire lifetime of a Chunker
|
||||||
|
/// \param maxBytes Maximal amount of bytes in returned chunks
|
||||||
|
Chunker(std::streambuf *buf, unsigned long long maxBytes);
|
||||||
|
|
||||||
|
std::streambuf *const buf; ///< Constant pointer to the source std::streambuf
|
||||||
|
bool eof = false; ///< Indicates whether EOF has been reached
|
||||||
|
const unsigned long long maxBytes;///< Max number of bytes in returned chunks
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_CHUNKER_H
|
||||||
19
src/chunkers/ChunkerFactory.cpp
Normal file
19
src/chunkers/ChunkerFactory.cpp
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 30.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "ChunkerFactory.h"
|
||||||
|
|
||||||
|
#include "../Exception.h"
|
||||||
|
#include "BuzhashChunker.h"
|
||||||
|
#include "ConstChunker.h"
|
||||||
|
|
||||||
|
std::unique_ptr<Chunker> ChunkerFactory::getChunker(const Config &config, std::streambuf *buf) {
|
||||||
|
if (config.getStr("chunker") == "const") {
|
||||||
|
return std::make_unique<ConstChunker>(buf, config.getInt("chunker-max") * 1024);
|
||||||
|
} else if (config.getStr("chunker") == "buzhash") {
|
||||||
|
return std::make_unique<BuzhashChunker>(buf, config.getInt("chunker-min") * 1024, config.getInt("chunker-max") * 1024, config.getInt("chunker-mask"));
|
||||||
|
} else {
|
||||||
|
throw Exception("Unknown chunker type!");
|
||||||
|
}
|
||||||
|
}
|
||||||
25
src/chunkers/ChunkerFactory.h
Normal file
25
src/chunkers/ChunkerFactory.h
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 30.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_CHUNKERFACTORY_H
|
||||||
|
#define SEMBACKUP_CHUNKERFACTORY_H
|
||||||
|
|
||||||
|
#include <memory>
|
||||||
|
#include <streambuf>
|
||||||
|
|
||||||
|
#include "../Config.h"
|
||||||
|
#include "Chunker.h"
|
||||||
|
|
||||||
|
/// Factory for Chunker%s
|
||||||
|
class ChunkerFactory {
|
||||||
|
public:
|
||||||
|
/// Creates a new Chunker based on provided \p config backed with \p buf
|
||||||
|
/// \param config Constant reference to Config
|
||||||
|
/// \param buf Pointer to a std::streambuf instance, should be avaliable during the Chunker lifetime
|
||||||
|
/// \return Unique pointer to the created Chunker
|
||||||
|
static std::unique_ptr<Chunker> getChunker(const Config &config, std::streambuf *buf);
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_CHUNKERFACTORY_H
|
||||||
27
src/chunkers/ConstChunker.cpp
Normal file
27
src/chunkers/ConstChunker.cpp
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 15.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "ConstChunker.h"
|
||||||
|
|
||||||
|
#include "../Exception.h"
|
||||||
|
#include "../crypto/MD5.h"
|
||||||
|
|
||||||
|
ConstChunker::ConstChunker(std::streambuf *buf, unsigned long long maxBytes) : Chunker(buf, maxBytes) {}
|
||||||
|
|
||||||
|
std::pair<std::string, std::vector<char>> ConstChunker::getNext() {
|
||||||
|
if (eof) throw Exception("Trying to read from a file that is finished!");
|
||||||
|
|
||||||
|
std::vector<char> rbuf(maxBytes);
|
||||||
|
|
||||||
|
auto read = static_cast<unsigned long>(buf->sgetn(rbuf.data(), (long) maxBytes));
|
||||||
|
|
||||||
|
if (read != maxBytes) {
|
||||||
|
eof = true;
|
||||||
|
rbuf.resize(read);
|
||||||
|
}
|
||||||
|
|
||||||
|
auto md5 = MD5::calculate(rbuf);
|
||||||
|
|
||||||
|
return {md5, rbuf};
|
||||||
|
}
|
||||||
24
src/chunkers/ConstChunker.h
Normal file
24
src/chunkers/ConstChunker.h
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 15.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_CONSTCHUNKER_H
|
||||||
|
#define SEMBACKUP_CONSTCHUNKER_H
|
||||||
|
|
||||||
|
#include <streambuf>
|
||||||
|
|
||||||
|
#include "Chunker.h"
|
||||||
|
|
||||||
|
/// Chunker implementation that splits the file into equally-sized chunks of maxBytes bytes
|
||||||
|
class ConstChunker : public Chunker {
|
||||||
|
public:
|
||||||
|
/// Constructs a ConstChunker
|
||||||
|
/// \copydoc Chunker::Chunker
|
||||||
|
ConstChunker(std::streambuf *buf, unsigned long long maxBytes);
|
||||||
|
|
||||||
|
/// \copydoc Chunker::getNext
|
||||||
|
std::pair<std::string, std::vector<char>> getNext() override;
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_CONSTCHUNKER_H
|
||||||
9
src/commands/Command.cpp
Normal file
9
src/commands/Command.cpp
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 23.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "Command.h"
|
||||||
|
|
||||||
|
Command::Command(std::string name) : name(std::move(name)) {}
|
||||||
|
|
||||||
|
Command::~Command() = default;
|
||||||
28
src/commands/Command.h
Normal file
28
src/commands/Command.h
Normal file
@@ -0,0 +1,28 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 23.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_COMMAND_H
|
||||||
|
#define SEMBACKUP_COMMAND_H
|
||||||
|
|
||||||
|
#include "../Context.h"
|
||||||
|
|
||||||
|
/// Abstract base class for some process running with some Context
|
||||||
|
class Command {
|
||||||
|
public:
|
||||||
|
/// Runs the command with Context \p ctx
|
||||||
|
virtual void run(Context ctx) = 0;
|
||||||
|
|
||||||
|
/// Default virtual destructor
|
||||||
|
virtual ~Command() = 0;
|
||||||
|
|
||||||
|
/// The name of the command
|
||||||
|
const std::string name;
|
||||||
|
|
||||||
|
protected:
|
||||||
|
/// Constructs a command with name \p name
|
||||||
|
Command(std::string name);
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_COMMAND_H
|
||||||
152
src/commands/CommandDiff.cpp
Normal file
152
src/commands/CommandDiff.cpp
Normal file
@@ -0,0 +1,152 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 23.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "CommandDiff.h"
|
||||||
|
|
||||||
|
#include "../BytesFormatter.h"
|
||||||
|
#include "../Diff.h"
|
||||||
|
#include "../Exception.h"
|
||||||
|
#include "../Progress.h"
|
||||||
|
#include "../RunningDiffAverage.h"
|
||||||
|
#include "../Signals.h"
|
||||||
|
#include "../ThreadPool.h"
|
||||||
|
#include "../change_detectors/ChangeDetectorFactory.h"
|
||||||
|
#include "../chunkers/ChunkerFactory.h"
|
||||||
|
#include "../repo/Serialize.h"
|
||||||
|
#include "../repo/objects/Archive.h"
|
||||||
|
#include "../repo/objects/Chunk.h"
|
||||||
|
|
||||||
|
using namespace CommandsCommon;
|
||||||
|
|
||||||
|
CommandDiff::CommandDiff() : Command("diff") {}
|
||||||
|
|
||||||
|
void CommandDiff::run(Context ctx) {
|
||||||
|
std::string diffMode = ctx.repo->getConfig().getStr("diff-mode");
|
||||||
|
|
||||||
|
Object::idType archive1;
|
||||||
|
if (!ctx.repo->getConfig().exists("aid")) {
|
||||||
|
auto archives = ctx.repo->getObjects(Object::ObjectType::Archive);
|
||||||
|
archive1 = std::max_element(archives.begin(), archives.end(), [](const auto &a1, const auto &a2) { return a1.second < a2.second; })->second;
|
||||||
|
} else {
|
||||||
|
archive1 = ctx.repo->getConfig().getInt("aid");
|
||||||
|
}
|
||||||
|
|
||||||
|
ThreadPool threadPool([&](const std::string &error) {
|
||||||
|
ctx.logger->write("Error: " + error, 0);
|
||||||
|
},
|
||||||
|
ctx.repo->getConfig().exists("threads") ? ctx.repo->getConfig().getInt("threads") : std::thread::hardware_concurrency());
|
||||||
|
|
||||||
|
auto archiveO1 = Serialize::deserialize<Archive>(ctx.repo->getObject(archive1));
|
||||||
|
std::mutex filesLock;
|
||||||
|
std::map<std::filesystem::path, File> files;///< Files in the first archive
|
||||||
|
for (auto id: archiveO1.files) {
|
||||||
|
auto file = Serialize::deserialize<File>(ctx.repo->getObject(id));
|
||||||
|
auto path = std::filesystem::u8path(file.name);
|
||||||
|
if (isSubpath(ctx.repo->getConfig().getStr("prefix"), path))
|
||||||
|
files.emplace(file.getKey(), std::move(file));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Container of ChangeDetectors built using the config of the repository
|
||||||
|
ChangeDetectorContainer changeDetector = ChangeDetectorFactory::getChangeDetectors(ctx.repo->getConfig());
|
||||||
|
|
||||||
|
/// Task to to compare the given file with the first archive
|
||||||
|
auto processFile = [&, this](ComparableFile p) {
|
||||||
|
auto relPath = p.path;
|
||||||
|
std::unique_lock lock(filesLock);
|
||||||
|
if (files.count(relPath) == 0) {
|
||||||
|
ctx.logger->write(relPath + " is new\n", 0);
|
||||||
|
lock.unlock();
|
||||||
|
} else {
|
||||||
|
File repoFile = files.at(relPath);
|
||||||
|
lock.unlock();
|
||||||
|
if (changeDetector.check({repoFile, ctx.repo}, p)) {
|
||||||
|
ctx.logger->write(relPath + " is different " + Diff::diff({repoFile, ctx.repo}, p) + "\n", 1);
|
||||||
|
} else {
|
||||||
|
if (diffMode == "file")
|
||||||
|
ctx.logger->write(relPath + " are same ", 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
lock.lock();
|
||||||
|
files.erase(relPath);
|
||||||
|
};
|
||||||
|
|
||||||
|
std::optional<Archive> archiveO2;
|
||||||
|
if (diffMode == "normal") {
|
||||||
|
/// If a second archive is given, run the task for each of its files, otherwise use the "from" config option
|
||||||
|
if (ctx.repo->getConfig().exists("aid2")) {
|
||||||
|
archiveO2.emplace(Serialize::deserialize<Archive>(ctx.repo->getObject(ctx.repo->getConfig().getInt("aid2"))));
|
||||||
|
|
||||||
|
threadPool.push([&]() {
|
||||||
|
for (auto id: archiveO2.value().files) {
|
||||||
|
/// Exit when asked to
|
||||||
|
if (Signals::shouldQuit) throw Exception("Quitting");
|
||||||
|
auto file = Serialize::deserialize<File>(ctx.repo->getObject(id));
|
||||||
|
if (isSubpath(ctx.repo->getConfig().getStr("prefix"), std::filesystem::u8path(file.name)))
|
||||||
|
threadPool.push([&, file]() {
|
||||||
|
processFile(ComparableFile{file, ctx.repo});
|
||||||
|
});
|
||||||
|
if (Signals::shouldQuit) break;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
std::filesystem::path from = ctx.repo->getConfig().getStr("from");
|
||||||
|
/// Start the diff with the root directory and empty ignore list
|
||||||
|
threadPool.push([&, from]() {
|
||||||
|
processDirWithIgnore(
|
||||||
|
from,
|
||||||
|
{},
|
||||||
|
[&](std::function<void()> f) { threadPool.push(std::move(f)); },
|
||||||
|
[processFile, from, prefix = ctx.repo->getConfig().getStr("prefix")](const std::filesystem::directory_entry &dirEntry) {
|
||||||
|
if (isSubpath(prefix, dirEntry.path().lexically_relative(from)))
|
||||||
|
processFile(ComparableFile{dirEntry, from});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} else if (diffMode == "file") {
|
||||||
|
if (files.count(ctx.repo->getConfig().getStr("prefix")) == 0) {
|
||||||
|
ctx.logger->write("Doesn't exist in the first archive", 0);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ctx.repo->getConfig().exists("aid2")) {
|
||||||
|
archiveO2.emplace(Serialize::deserialize<Archive>(ctx.repo->getObject(ctx.repo->getConfig().getInt("aid2"))));
|
||||||
|
std::map<std::filesystem::path, File> files2;///< Files in the first archive
|
||||||
|
for (auto id: archiveO2->files) {
|
||||||
|
auto file = Serialize::deserialize<File>(ctx.repo->getObject(id));
|
||||||
|
auto path = std::filesystem::u8path(file.name);
|
||||||
|
if (isSubpath(ctx.repo->getConfig().getStr("prefix"), path))
|
||||||
|
files2.emplace(file.getKey(), std::move(file));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (files2.count(ctx.repo->getConfig().getStr("prefix")) == 0) {
|
||||||
|
ctx.logger->write("Doesn't exist in the second archive", 0);
|
||||||
|
return;
|
||||||
|
} else {
|
||||||
|
processFile(ComparableFile{files2.at(ctx.repo->getConfig().getStr("prefix")), ctx.repo});
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
std::filesystem::path from = ctx.repo->getConfig().getStr("from");
|
||||||
|
if (!std::filesystem::exists(from / ctx.repo->getConfig().getStr("prefix"))) {
|
||||||
|
ctx.logger->write("Doesn't exist in the filesystem archive", 0);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
/// Start the diff with the root directory and empty ignore list
|
||||||
|
processFile(ComparableFile{from / ctx.repo->getConfig().getStr("prefix"), from});
|
||||||
|
}
|
||||||
|
|
||||||
|
} else {
|
||||||
|
throw Exception("Unknown diff-mode: " + diffMode);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Wait for diff to end
|
||||||
|
std::unique_lock finishedLock(threadPool.finishedLock);
|
||||||
|
threadPool.finished.wait(finishedLock, [&threadPool] { return threadPool.empty(); });
|
||||||
|
if (diffMode == "normal")
|
||||||
|
for (auto const &s: files) {
|
||||||
|
ctx.logger->write(s.first.u8string() + " is removed\n", 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
23
src/commands/CommandDiff.h
Normal file
23
src/commands/CommandDiff.h
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 23.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_COMMANDDIFF_H
|
||||||
|
#define SEMBACKUP_COMMANDDIFF_H
|
||||||
|
|
||||||
|
#include "Command.h"
|
||||||
|
|
||||||
|
#include "CommandsCommon.h"
|
||||||
|
|
||||||
|
/// Run the diff between:
|
||||||
|
/// 1. The latest archive and the `from` directory
|
||||||
|
/// 2. if `aid` is set the aid archive and the `from` directory
|
||||||
|
/// 3. if `aid` and `aid2` are set between `aid` and `aid2`
|
||||||
|
class CommandDiff : public Command {
|
||||||
|
public:
|
||||||
|
CommandDiff();
|
||||||
|
void run(Context ctx) override;
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_COMMANDDIFF_H
|
||||||
16
src/commands/CommandList.cpp
Normal file
16
src/commands/CommandList.cpp
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 23.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "CommandList.h"
|
||||||
|
|
||||||
|
CommandList::CommandList() : Command("list") {
|
||||||
|
}
|
||||||
|
|
||||||
|
void CommandList::run(Context ctx) {
|
||||||
|
auto list = ctx.repo->getObjects(Object::ObjectType::Archive);
|
||||||
|
std::sort(list.begin(), list.end(), [](const auto &l, const auto &r) { return l.second < r.second; });
|
||||||
|
for (auto const &aid: list) {
|
||||||
|
std::cout << "Name: " << aid.first << " Id: " << aid.second << std::endl;
|
||||||
|
}
|
||||||
|
}
|
||||||
20
src/commands/CommandList.h
Normal file
20
src/commands/CommandList.h
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 23.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_COMMANDLIST_H
|
||||||
|
#define SEMBACKUP_COMMANDLIST_H
|
||||||
|
|
||||||
|
#include "Command.h"
|
||||||
|
|
||||||
|
#include "CommandsCommon.h"
|
||||||
|
|
||||||
|
/// Lists available archives in a repository
|
||||||
|
class CommandList : public Command {
|
||||||
|
public:
|
||||||
|
CommandList();
|
||||||
|
void run(Context ctx) override;
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_COMMANDLIST_H
|
||||||
22
src/commands/CommandListFiles.cpp
Normal file
22
src/commands/CommandListFiles.cpp
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 23.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "CommandListFiles.h"
|
||||||
|
|
||||||
|
#include "../BytesFormatter.h"
|
||||||
|
#include "../repo/Serialize.h"
|
||||||
|
#include "../repo/objects/Archive.h"
|
||||||
|
#include "../repo/objects/Chunk.h"
|
||||||
|
#include "../repo/objects/File.h"
|
||||||
|
|
||||||
|
CommandListFiles::CommandListFiles() : Command("list-files") {
|
||||||
|
}
|
||||||
|
|
||||||
|
void CommandListFiles::run(Context ctx) {
|
||||||
|
auto archive = Serialize::deserialize<Archive>(ctx.repo->getObject(ctx.repo->getConfig().getInt("aid")));
|
||||||
|
for (auto const &fid: archive.files) {
|
||||||
|
auto file = Serialize::deserialize<File>(ctx.repo->getObject(fid));
|
||||||
|
std::cout << "Name: " << file.name << " type: " << File::TypeToStr.at(file.fileType) << " size: " << BytesFormatter::formatStr(file.bytes) << std::endl;
|
||||||
|
}
|
||||||
|
}
|
||||||
20
src/commands/CommandListFiles.h
Normal file
20
src/commands/CommandListFiles.h
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 23.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_COMMANDLISTFILES_H
|
||||||
|
#define SEMBACKUP_COMMANDLISTFILES_H
|
||||||
|
|
||||||
|
#include "Command.h"
|
||||||
|
|
||||||
|
#include "CommandsCommon.h"
|
||||||
|
|
||||||
|
/// Lists files in the selected Archive
|
||||||
|
class CommandListFiles : public Command {
|
||||||
|
public:
|
||||||
|
CommandListFiles();
|
||||||
|
void run(Context ctx) override;
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_COMMANDLISTFILES_H
|
||||||
125
src/commands/CommandRestore.cpp
Normal file
125
src/commands/CommandRestore.cpp
Normal file
@@ -0,0 +1,125 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 23.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "CommandRestore.h"
|
||||||
|
|
||||||
|
#include <fstream>
|
||||||
|
#include <sstream>
|
||||||
|
|
||||||
|
#include "../BytesFormatter.h"
|
||||||
|
#include "../Exception.h"
|
||||||
|
#include "../Progress.h"
|
||||||
|
#include "../RunningDiffAverage.h"
|
||||||
|
#include "../Signals.h"
|
||||||
|
#include "../ThreadPool.h"
|
||||||
|
#include "../chunkers/ChunkerFactory.h"
|
||||||
|
#include "../repo/Serialize.h"
|
||||||
|
#include "../repo/objects/Archive.h"
|
||||||
|
#include "../repo/objects/Chunk.h"
|
||||||
|
|
||||||
|
using namespace CommandsCommon;
|
||||||
|
|
||||||
|
CommandRestore::CommandRestore() : Command("restore") {
|
||||||
|
}
|
||||||
|
|
||||||
|
void CommandRestore::run(Context ctx) {
|
||||||
|
Object::idType archive = ctx.repo->getConfig().getInt("aid");
|
||||||
|
std::filesystem::path to = std::filesystem::u8path(ctx.repo->getConfig().getStr("to"));
|
||||||
|
|
||||||
|
std::atomic<unsigned long long> filesToRestoreCount = 0;
|
||||||
|
std::atomic<unsigned long long> bytesToRestore = 0;
|
||||||
|
|
||||||
|
WorkerStats workerStats;///< Backup statistics of the worker threads
|
||||||
|
|
||||||
|
/// Worker callback, bound to the local workerStats variable
|
||||||
|
workerStatsFunction workerCallback = [&workerStats](unsigned long long bytesWritten, unsigned long long bytesSkipped, unsigned long long filesWritten) {
|
||||||
|
CommandsCommon::workerCallback(bytesWritten, bytesSkipped, filesWritten, workerStats);
|
||||||
|
};
|
||||||
|
{
|
||||||
|
/// Calculate the average speed of backup
|
||||||
|
RunningDiffAverage avg(
|
||||||
|
[&]() { return workerStats.bytesWritten.load(); },
|
||||||
|
100, 100);
|
||||||
|
|
||||||
|
/// Show restore progress
|
||||||
|
Progress progress([this, ctx](const std::string &s, int l) { ctx.logger->write(s, l); },
|
||||||
|
{
|
||||||
|
[&workerStats]() { return std::to_string(workerStats.filesWritten.load()); },
|
||||||
|
"/",
|
||||||
|
[&filesToRestoreCount]() { return std::to_string(filesToRestoreCount); },
|
||||||
|
" files saved, ",
|
||||||
|
[&workerStats]() { return BytesFormatter::formatStr(workerStats.bytesWritten.load() + workerStats.bytesSkipped.load()); },
|
||||||
|
" / ",
|
||||||
|
[&bytesToRestore]() { return BytesFormatter::formatStr(bytesToRestore); },
|
||||||
|
" saved @ ",
|
||||||
|
[&avg]() { return BytesFormatter::formatStr(avg.get() * 10); },
|
||||||
|
"/s",
|
||||||
|
},
|
||||||
|
ctx.repo->getConfig());
|
||||||
|
|
||||||
|
/// Thread pool for restore tasks
|
||||||
|
ThreadPool threadPool([&](const std::string &error) {
|
||||||
|
progress.print("Error: " + error, 0);
|
||||||
|
},
|
||||||
|
ctx.repo->getConfig().exists("threads") ? ctx.repo->getConfig().getInt("threads") : std::thread::hardware_concurrency());
|
||||||
|
|
||||||
|
/// Add the main restore task
|
||||||
|
threadPool.push([&, this]() {
|
||||||
|
/// Get the archive and its file IDs
|
||||||
|
auto archiveO = Serialize::deserialize<Archive>(ctx.repo->getObject(archive));
|
||||||
|
std::vector<Object::idType> files = archiveO.files;
|
||||||
|
/// For each file...
|
||||||
|
for (const auto fid: files) {
|
||||||
|
/// Stop when asked to
|
||||||
|
if (Signals::shouldQuit) break;
|
||||||
|
|
||||||
|
auto file = Serialize::deserialize<File>(ctx.repo->getObject(fid));
|
||||||
|
filesToRestoreCount++;
|
||||||
|
bytesToRestore += file.bytes;
|
||||||
|
/// Spawn a restore task
|
||||||
|
threadPool.push([&, this, to, file]() {
|
||||||
|
backupRestoreFile(file, to, workerCallback, ctx);
|
||||||
|
progress.print("Restored " + file.name, 1);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
/// Wait for all tasks to finish
|
||||||
|
std::unique_lock finishedLock(threadPool.finishedLock);
|
||||||
|
threadPool.finished.wait(finishedLock, [&threadPool] { return threadPool.empty(); });
|
||||||
|
}
|
||||||
|
ctx.logger->write("\n", 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string CommandRestore::backupRestoreFile(const File &file, const std::filesystem::path &baseDir, workerStatsFunction &callback, Context ctx) {
|
||||||
|
auto fullpath = baseDir / std::filesystem::u8path(file.name);
|
||||||
|
|
||||||
|
std::filesystem::create_directories(fullpath.parent_path());
|
||||||
|
|
||||||
|
if (file.fileType == File::Type::Directory) {
|
||||||
|
std::filesystem::create_directory(fullpath);
|
||||||
|
callback(0, 0, 1);
|
||||||
|
return fullpath.u8string();
|
||||||
|
}
|
||||||
|
if (file.fileType == File::Type::Symlink) {
|
||||||
|
auto dest = Serialize::deserialize<Chunk>(ctx.repo->getObject(file.chunks[0]));
|
||||||
|
std::filesystem::create_symlink(std::filesystem::u8path(std::string{dest.data.begin(), dest.data.end()}), fullpath);
|
||||||
|
callback(0, 0, 1);
|
||||||
|
return fullpath.u8string();
|
||||||
|
}
|
||||||
|
|
||||||
|
std::ofstream ostream(fullpath, std::ios::binary | std::ios::out | std::ios::trunc);
|
||||||
|
for (const auto cid: file.chunks) {
|
||||||
|
if (Signals::shouldQuit) throw Exception("Quitting!");
|
||||||
|
|
||||||
|
Chunk c = Serialize::deserialize<Chunk>(ctx.repo->getObject(cid));
|
||||||
|
if (!c.data.empty()) {
|
||||||
|
ostream.rdbuf()->sputn(c.data.data(), c.data.size());
|
||||||
|
callback(c.data.size(), 0, 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
callback(0, 0, 1);
|
||||||
|
|
||||||
|
return fullpath.u8string();
|
||||||
|
}
|
||||||
30
src/commands/CommandRestore.h
Normal file
30
src/commands/CommandRestore.h
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 23.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_COMMANDRESTORE_H
|
||||||
|
#define SEMBACKUP_COMMANDRESTORE_H
|
||||||
|
|
||||||
|
#include "Command.h"
|
||||||
|
|
||||||
|
#include "../repo/objects/File.h"
|
||||||
|
|
||||||
|
#include "CommandsCommon.h"
|
||||||
|
|
||||||
|
/// Restores the archive with id \aid to path \p to (from config)
|
||||||
|
class CommandRestore : public Command {
|
||||||
|
public:
|
||||||
|
CommandRestore();
|
||||||
|
void run(Context ctx) override;
|
||||||
|
|
||||||
|
private:
|
||||||
|
/// Internal function to restore a file
|
||||||
|
/// \param file Constant reference to the File object
|
||||||
|
/// \param base Base directory to restore to
|
||||||
|
/// \param callback Stats callback
|
||||||
|
/// \return Name of the restored file
|
||||||
|
std::string backupRestoreFile(const File &file, const std::filesystem::path &base, CommandsCommon::workerStatsFunction &callback, Context ctx);
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_COMMANDRESTORE_H
|
||||||
239
src/commands/CommandRun.cpp
Normal file
239
src/commands/CommandRun.cpp
Normal file
@@ -0,0 +1,239 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 23.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "CommandRun.h"
|
||||||
|
|
||||||
|
#include <fstream>
|
||||||
|
#include <iomanip>
|
||||||
|
#include <sstream>
|
||||||
|
|
||||||
|
#include "../BytesFormatter.h"
|
||||||
|
#include "../Exception.h"
|
||||||
|
#include "../Progress.h"
|
||||||
|
#include "../RunningDiffAverage.h"
|
||||||
|
#include "../Signals.h"
|
||||||
|
#include "../ThreadPool.h"
|
||||||
|
#include "../change_detectors/ChangeDetectorFactory.h"
|
||||||
|
#include "../chunkers/ChunkerFactory.h"
|
||||||
|
#include "../crypto/MD5.h"
|
||||||
|
#include "../repo/Serialize.h"
|
||||||
|
#include "../repo/objects/Archive.h"
|
||||||
|
#include "../repo/objects/Chunk.h"
|
||||||
|
#include "../repo/objects/File.h"
|
||||||
|
|
||||||
|
#include "CommandsCommon.h"
|
||||||
|
|
||||||
|
using namespace CommandsCommon;
|
||||||
|
|
||||||
|
CommandRun::CommandRun() : Command("run") {
|
||||||
|
}
|
||||||
|
|
||||||
|
void CommandRun::run(Context ctx) {
|
||||||
|
WorkerStats workerStats;///< Backup statistics of the worker threads
|
||||||
|
RunnerStats runnerStats;///< Backup target metrics
|
||||||
|
|
||||||
|
std::filesystem::path from = ctx.repo->getConfig().getStr("from");///< Directory to back up from
|
||||||
|
bool fullBackup = ctx.repo->getConfig().getStr("type") == "full";
|
||||||
|
if (fullBackup) {
|
||||||
|
ctx.logger->write("Backup is full because of the config\n", 1);
|
||||||
|
}
|
||||||
|
/// For progtest task compliance
|
||||||
|
if (!fullBackup) {
|
||||||
|
/// If it's time for full backup as per config, force it
|
||||||
|
auto per = ctx.repo->getConfig().getInt("full-period");
|
||||||
|
auto list = ctx.repo->getObjects(Object::ObjectType::Archive);
|
||||||
|
std::sort(list.begin(), list.end(), [](const auto &l, const auto &r) { return l.second > r.second; });
|
||||||
|
int lastInc = 0;
|
||||||
|
for (auto const &a: list) {
|
||||||
|
auto archiveO = Serialize::deserialize<Archive>(ctx.repo->getObject(a.second));
|
||||||
|
if (!archiveO.isFull) {
|
||||||
|
lastInc++;
|
||||||
|
continue;
|
||||||
|
} else
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (lastInc >= per) {
|
||||||
|
fullBackup = true;
|
||||||
|
ctx.logger->write("Backup is full because of the interval\n", 1);
|
||||||
|
}
|
||||||
|
if (list.size() == 0) {
|
||||||
|
fullBackup = true;
|
||||||
|
ctx.logger->write("Backup is full because there are no backups\n", 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Worker callback, bound to the local workerStats variable
|
||||||
|
workerStatsFunction workerCallback = [&](unsigned long long bytesWritten, unsigned long long bytesSkipped, unsigned long long filesWritten) {
|
||||||
|
CommandsCommon::workerCallback(bytesWritten, bytesSkipped, filesWritten, workerStats);
|
||||||
|
};
|
||||||
|
|
||||||
|
std::vector<Object::idType> files;///< File ids so far added to the archive
|
||||||
|
std::mutex filesLock; ///< Files vector lock
|
||||||
|
/// Function to safely add new file ids to `files`
|
||||||
|
std::function addFile = [&](Object::idType id) {std::lock_guard lock(filesLock); files.emplace_back(id); };
|
||||||
|
|
||||||
|
/// Technically the progtest task says that only the files from the last backup should be compared against...
|
||||||
|
std::map<std::string, Object::idType> prevArchiveFiles;
|
||||||
|
{
|
||||||
|
auto prevArchiveFilesList = ctx.repo->getObjects(Object::ObjectType::File);
|
||||||
|
prevArchiveFiles = {prevArchiveFilesList.begin(), prevArchiveFilesList.end()};
|
||||||
|
}
|
||||||
|
ctx.repo->clearCache(Object::ObjectType::File);
|
||||||
|
|
||||||
|
{
|
||||||
|
/// Calculate the average speed of backup
|
||||||
|
RunningDiffAverage avg(
|
||||||
|
[&]() { return workerStats.bytesWritten.load(); },
|
||||||
|
100, 100);
|
||||||
|
|
||||||
|
/// Show the progress of backup
|
||||||
|
Progress progress([this, ctx](const std::string &s, int l) { ctx.logger->write(s, l); },
|
||||||
|
{[&]() { return std::to_string(workerStats.filesWritten.load()); },
|
||||||
|
"/",
|
||||||
|
[&]() { return std::to_string(runnerStats.filesToSaveCount); },
|
||||||
|
" files saved, ",
|
||||||
|
[&]() { return std::to_string(runnerStats.filesSkipped); },
|
||||||
|
" files skipped, ",
|
||||||
|
[&]() { return BytesFormatter::formatStr((workerStats.bytesWritten.load() + workerStats.bytesSkipped.load())); },
|
||||||
|
" / ",
|
||||||
|
[&]() { return BytesFormatter::formatStr(runnerStats.bytesToSave); },
|
||||||
|
" read @ ",
|
||||||
|
[&]() { return BytesFormatter::formatStr(avg.get() * 10); },
|
||||||
|
"/s"},
|
||||||
|
ctx.repo->getConfig());
|
||||||
|
|
||||||
|
/// Thread pool for backup tasks, prints to progress on any errors
|
||||||
|
ThreadPool threadPool([&](const std::string &error) {
|
||||||
|
progress.print("Error: " + error, 0);
|
||||||
|
},
|
||||||
|
ctx.repo->getConfig().exists("threads") ? ctx.repo->getConfig().getInt("threads") : std::thread::hardware_concurrency());
|
||||||
|
|
||||||
|
/// Container of ChangeDetectors built using the config of the repository
|
||||||
|
ChangeDetectorContainer changeDetector = ChangeDetectorFactory::getChangeDetectors(ctx.repo->getConfig());
|
||||||
|
|
||||||
|
/// Function to spawn a rechunking task
|
||||||
|
auto saveFile = [&, this](const std::filesystem::path &absPath, const std::filesystem::path &relPath) {
|
||||||
|
runnerStats.bytesToSave += File::getFileType(absPath) == File::Type::Normal ? std::filesystem::file_size(absPath) : 0;
|
||||||
|
runnerStats.filesToSaveCount++;
|
||||||
|
threadPool.push([&, relPath, absPath]() {
|
||||||
|
addFile(backupChunkFile(absPath, relPath.u8string(), workerCallback, ctx));
|
||||||
|
progress.print("Copied: " + relPath.u8string(), 1);
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
/// Task to process an individual file in the backup
|
||||||
|
std::function<void(std::filesystem::path)> processFile;
|
||||||
|
/// If it's a full backup, just save the file, otherwise re-chunk it only if it's changed
|
||||||
|
if (fullBackup)
|
||||||
|
processFile =
|
||||||
|
[&, this](const std::filesystem::path &p) {
|
||||||
|
saveFile(p, p.lexically_relative(from).u8string());
|
||||||
|
};
|
||||||
|
else
|
||||||
|
processFile =
|
||||||
|
[&, this](const std::filesystem::path &p) {
|
||||||
|
auto relPath = p.lexically_relative(from).u8string();
|
||||||
|
|
||||||
|
if (prevArchiveFiles.count(relPath) != 0) {
|
||||||
|
File repoFile = Serialize::deserialize<File>(ctx.repo->getObject(prevArchiveFiles.at(relPath)));
|
||||||
|
if (!changeDetector.check({repoFile, ctx.repo}, {p, from})) {
|
||||||
|
addFile(repoFile.id);
|
||||||
|
ctx.repo->addToCache(repoFile);
|
||||||
|
progress.print("Skipped: " + relPath, 1);
|
||||||
|
runnerStats.filesSkipped++;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
saveFile(p, relPath);
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
|
||||||
|
/// Start the backup with the root directory and empty ignore list
|
||||||
|
threadPool.push([&]() {
|
||||||
|
processDirWithIgnore(
|
||||||
|
from,
|
||||||
|
{},
|
||||||
|
[&](std::function<void()> f) { threadPool.push(std::move(f)); },
|
||||||
|
processFile);
|
||||||
|
});
|
||||||
|
|
||||||
|
/// Wait for all the tasks to finish
|
||||||
|
std::unique_lock finishedLock(threadPool.finishedLock);
|
||||||
|
threadPool.finished.wait(finishedLock, [&threadPool] { return threadPool.empty(); });
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx.logger->write("\n", 1);
|
||||||
|
|
||||||
|
auto written = BytesFormatter::format(workerStats.bytesWritten);
|
||||||
|
auto skipped = BytesFormatter::format(workerStats.bytesSkipped);
|
||||||
|
|
||||||
|
ctx.logger->write(written.prefix + " written: " + written.number + '\n', 1);
|
||||||
|
ctx.logger->write(skipped.prefix + " skipped: " + skipped.number + '\n', 1);
|
||||||
|
|
||||||
|
auto time = std::time(0);
|
||||||
|
auto ltime = std::localtime(&time);
|
||||||
|
std::stringstream s;
|
||||||
|
s << std::put_time(ltime, "%d-%m-%Y %H-%M-%S");
|
||||||
|
/// Avoid archive name collisions
|
||||||
|
while (ctx.repo->exists(Object::ObjectType::Archive, s.str())) s << "N";
|
||||||
|
Archive a(ctx.repo->getId(), s.str(), time, files, fullBackup);
|
||||||
|
ctx.repo->putObject(a);
|
||||||
|
}
|
||||||
|
|
||||||
|
Object::idType CommandRun::backupChunkFile(const std::filesystem::path &orig, const std::string &saveAs, workerStatsFunction &callback, Context ctx) {
|
||||||
|
/// If it's a symlink or directory, treat it specially
|
||||||
|
/// The order of checks is important, because is_directory follows the symlink
|
||||||
|
if (std::filesystem::is_symlink(orig) || std::filesystem::is_directory(orig)) {
|
||||||
|
auto contents = File::getFileContents(orig);
|
||||||
|
Chunk c(ctx.repo->getId(), MD5::calculate(contents), contents);
|
||||||
|
File f(ctx.repo->getId(), saveAs, c.length, File::getFileMtime(orig), c.md5, {c.id}, File::getFileType(orig));
|
||||||
|
ctx.repo->putObject(c);
|
||||||
|
ctx.repo->putObject(f);
|
||||||
|
return f.id;
|
||||||
|
}
|
||||||
|
if (!std::filesystem::is_regular_file(orig))
|
||||||
|
throw Exception(orig.u8string() + "is a special file, not saving");
|
||||||
|
|
||||||
|
std::ifstream ifstream(orig, std::ios::in | std::ios::binary);
|
||||||
|
if (!ifstream) throw Exception("Couldn't open " + orig.u8string() + " for reading");
|
||||||
|
std::unique_ptr<Chunker> chunker = ChunkerFactory::getChunker(ctx.repo->getConfig(), ifstream.rdbuf());
|
||||||
|
|
||||||
|
MD5 fileHash;
|
||||||
|
|
||||||
|
std::vector<Object::idType> fileChunks;
|
||||||
|
unsigned long long size = 0;
|
||||||
|
|
||||||
|
for (auto chunkp: *chunker) {
|
||||||
|
/// Exit when asked to
|
||||||
|
if (Signals::shouldQuit) break;
|
||||||
|
|
||||||
|
Object::idType chunkId;
|
||||||
|
size += chunkp.second.size();
|
||||||
|
if (ctx.repo->getConfig().getStr("dedup") == "on" && ctx.repo->exists(Object::ObjectType::Chunk, chunkp.first)) {
|
||||||
|
/// If the chunk already exists, reuse it
|
||||||
|
chunkId = ctx.repo->getObjectId(Object::ObjectType::Chunk, chunkp.first);
|
||||||
|
callback(0, chunkp.second.size(), 0);
|
||||||
|
} else {
|
||||||
|
/// Otherwise, write it
|
||||||
|
Chunk c(ctx.repo->getId(), chunkp.first, chunkp.second);
|
||||||
|
chunkId = c.id;
|
||||||
|
callback(c.data.size(), 0, 0);
|
||||||
|
ctx.repo->putObject(c);
|
||||||
|
}
|
||||||
|
fileHash.feedData(chunkp.second);
|
||||||
|
fileChunks.emplace_back(chunkId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// We might have exited in the loop before, so we don't save an incomplete file
|
||||||
|
if (Signals::shouldQuit) throw Exception("Quitting!");
|
||||||
|
if (size != File::getFileSize(orig)) {
|
||||||
|
throw Exception("Something really bad happened or file " + orig.u8string() + " changed during backup");
|
||||||
|
}
|
||||||
|
File f(ctx.repo->getId(), saveAs, size, File::getFileMtime(orig), fileHash.getHash(), fileChunks, File::getFileType(orig));
|
||||||
|
ctx.repo->putObject(f);
|
||||||
|
callback(0, 0, 1);
|
||||||
|
|
||||||
|
return f.id;
|
||||||
|
}
|
||||||
28
src/commands/CommandRun.h
Normal file
28
src/commands/CommandRun.h
Normal file
@@ -0,0 +1,28 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 23.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_COMMANDRUN_H
|
||||||
|
#define SEMBACKUP_COMMANDRUN_H
|
||||||
|
|
||||||
|
#include "Command.h"
|
||||||
|
|
||||||
|
#include "CommandsCommon.h"
|
||||||
|
|
||||||
|
/// Runs the backup according to the config in the Repository
|
||||||
|
class CommandRun : public Command {
|
||||||
|
public:
|
||||||
|
CommandRun();
|
||||||
|
void run(Context ctx) override;
|
||||||
|
|
||||||
|
private:
|
||||||
|
/// Internal function to chunk the file and save it
|
||||||
|
/// \param orig Absolute path to the file
|
||||||
|
/// \param saveAs UTF-8 encoded file name to save as
|
||||||
|
/// \param callback Stats callback
|
||||||
|
/// \return ID of the saved file
|
||||||
|
Object::idType backupChunkFile(const std::filesystem::path &orig, const std::string &saveAs, CommandsCommon::workerStatsFunction &callback, Context ctx);
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_COMMANDRUN_H
|
||||||
67
src/commands/CommandsCommon.cpp
Normal file
67
src/commands/CommandsCommon.cpp
Normal file
@@ -0,0 +1,67 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 23.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "CommandsCommon.h"
|
||||||
|
|
||||||
|
#include <fstream>
|
||||||
|
#include <regex>
|
||||||
|
|
||||||
|
#include "../Exception.h"
|
||||||
|
#include "../Signals.h"
|
||||||
|
|
||||||
|
void CommandsCommon::workerCallback(unsigned long long int bytesWritten, unsigned long long int bytesSkipped, unsigned long long int filesWritten, WorkerStats &to) {
|
||||||
|
to.bytesWritten += bytesWritten;
|
||||||
|
to.bytesSkipped += bytesSkipped;
|
||||||
|
to.filesWritten += filesWritten;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool CommandsCommon::isSubpath(const std::filesystem::path &prefix, const std::filesystem::path &p) {
|
||||||
|
if (prefix.u8string().size() > p.u8string().size()) return false;
|
||||||
|
for (int i = 0; i < prefix.u8string().size(); i++)
|
||||||
|
if (p.u8string()[i] != prefix.u8string()[i]) return false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
void CommandsCommon::processDirWithIgnore(const std::filesystem::path &dir, std::vector<std::string> ignore, std::function<void(std::function<void()>)> spawner, std::function<void(std::filesystem::directory_entry)> processFile) {
|
||||||
|
if (!std::filesystem::is_directory(dir)) throw Exception(dir.u8string() + " is not a directory!");
|
||||||
|
|
||||||
|
/// Don't process the directory if it has a ".nobackup" file
|
||||||
|
if (std::filesystem::exists(dir / ".nobackup")) return;
|
||||||
|
|
||||||
|
/// If it has an .ignore file, add every line of it into our ignore vector
|
||||||
|
if (std::filesystem::exists(dir / ".ignore")) {
|
||||||
|
std::ifstream ignorefile(dir / ".ignore", std::ios::in);
|
||||||
|
std::string line;
|
||||||
|
while (std::getline(ignorefile, line)) {
|
||||||
|
ignore.emplace_back(line);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// For each directory entry...
|
||||||
|
for (const auto &dirEntry: std::filesystem::directory_iterator(dir)) {
|
||||||
|
/// Break in case exit was requested by the user
|
||||||
|
if (Signals::shouldQuit) break;
|
||||||
|
|
||||||
|
/// Don't process the entry if it matches any of the ignore rules
|
||||||
|
if (std::any_of(ignore.begin(), ignore.end(), [dirEntry](auto pred) {
|
||||||
|
std::smatch m;
|
||||||
|
auto s = dirEntry.path().filename().u8string();
|
||||||
|
return std::regex_match(s, m, std::regex(pred));
|
||||||
|
})) continue;
|
||||||
|
|
||||||
|
/// If it's a directory, spawn a task to process the entries in it
|
||||||
|
if (!dirEntry.is_symlink() && dirEntry.is_directory()) {
|
||||||
|
spawner([dirEntry, ignore, spawner, processFile]() {
|
||||||
|
processDirWithIgnore(dirEntry.path(), ignore, spawner, processFile);
|
||||||
|
});
|
||||||
|
/// Don't save the dir if it has a .nobackup file
|
||||||
|
if (std::filesystem::exists(dirEntry.path() / ".nobackup")) continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Spawn a task to process each individual file
|
||||||
|
spawner([processFile, dirEntry]() {
|
||||||
|
processFile(dirEntry);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
48
src/commands/CommandsCommon.h
Normal file
48
src/commands/CommandsCommon.h
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 23.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_COMMANDSCOMMON_H
|
||||||
|
#define SEMBACKUP_COMMANDSCOMMON_H
|
||||||
|
|
||||||
|
#include <atomic>
|
||||||
|
#include <filesystem>
|
||||||
|
#include <functional>
|
||||||
|
|
||||||
|
namespace CommandsCommon {
|
||||||
|
// Bytes written, bytes skipped, files written
|
||||||
|
using workerStatsFunction = std::function<void(unsigned long long, unsigned long long, unsigned long long)>;
|
||||||
|
|
||||||
|
/// Internat function for recursive directory processing, taking into account ".ignore" and ".nobackup" files
|
||||||
|
/// \param dir Const reference to the path of directory to iterate through
|
||||||
|
/// \param ignore List of files to ignore
|
||||||
|
/// \param spawner Function to spawn other tasks
|
||||||
|
/// \param processFile Task to spawn on found files
|
||||||
|
void processDirWithIgnore(const std::filesystem::path &dir, std::vector<std::string> ignore, std::function<void(std::function<void()>)> spawner, std::function<void(std::filesystem::directory_entry)> processFile);
|
||||||
|
|
||||||
|
struct WorkerStats {
|
||||||
|
public:
|
||||||
|
std::atomic<unsigned long long> bytesWritten = 0;
|
||||||
|
std::atomic<unsigned long long> bytesSkipped = 0;
|
||||||
|
std::atomic<unsigned long long> filesWritten = 0;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct RunnerStats {
|
||||||
|
public:
|
||||||
|
std::atomic<unsigned long long> bytesToSave = 0;
|
||||||
|
std::atomic<unsigned long long> filesToSaveCount = 0;
|
||||||
|
std::atomic<unsigned long long> filesSkipped = 0;
|
||||||
|
};
|
||||||
|
|
||||||
|
/// Checks if \p p has \p prefix as prefix
|
||||||
|
/// \param prefix Constant reference to the prefix path
|
||||||
|
/// \param p Constant reference to the checked path
|
||||||
|
/// \return True if \p p contains \p prefix at its prefix, False otherwise
|
||||||
|
bool isSubpath(const std::filesystem::path &prefix, const std::filesystem::path &p);
|
||||||
|
|
||||||
|
void workerCallback(unsigned long long bytesWritten, unsigned long long bytesSkipped, unsigned long long filesWritten, WorkerStats &to);
|
||||||
|
|
||||||
|
};// namespace CommandsCommon
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_COMMANDSCOMMON_H
|
||||||
82
src/crypto/AES.cpp
Normal file
82
src/crypto/AES.cpp
Normal file
@@ -0,0 +1,82 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 30.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "AES.h"
|
||||||
|
|
||||||
|
#include <openssl/aes.h>
|
||||||
|
#include <openssl/evp.h>
|
||||||
|
#include <openssl/rand.h>
|
||||||
|
|
||||||
|
#include "../Exception.h"
|
||||||
|
|
||||||
|
std::vector<char> AES::encrypt(const std::vector<char> &in, const std::string &password, const std::string &salt) {
|
||||||
|
return AES::encrypt(in, AES::deriveKey(password, salt));
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<char> AES::decrypt(const std::vector<char> &in, const std::string &password, const std::string &salt) {
|
||||||
|
return AES::decrypt(in, AES::deriveKey(password, salt));
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<char> AES::encrypt(const std::vector<char> &in, const std::array<uint8_t, 32> &key) {
|
||||||
|
std::unique_ptr<EVP_CIPHER_CTX, decltype(&EVP_CIPHER_CTX_free)> ctx(EVP_CIPHER_CTX_new(), &EVP_CIPHER_CTX_free);
|
||||||
|
if (!ctx) throw Exception("Error initializing encryption context!");
|
||||||
|
|
||||||
|
std::vector<char> out(in.size() + AES_BLOCK_SIZE + 32);
|
||||||
|
if (!RAND_bytes(reinterpret_cast<unsigned char *>(out.data()), 32))
|
||||||
|
throw Exception("Error generating IV!");
|
||||||
|
|
||||||
|
if (!EVP_EncryptInit_ex(ctx.get(), EVP_aes_256_cbc(), nullptr, key.data(), reinterpret_cast<const unsigned char *>(out.data())))
|
||||||
|
throw Exception("Error encrypting!");
|
||||||
|
|
||||||
|
int outlen = static_cast<int>(out.size()) - 32;
|
||||||
|
|
||||||
|
if (!EVP_EncryptUpdate(ctx.get(), reinterpret_cast<unsigned char *>(out.data() + 32), &outlen, reinterpret_cast<const unsigned char *>(in.data()), static_cast<int>(in.size())))
|
||||||
|
throw Exception("Error encrypting!");
|
||||||
|
|
||||||
|
int finlen = 0;
|
||||||
|
if (!EVP_EncryptFinal_ex(ctx.get(), reinterpret_cast<unsigned char *>(out.data() + outlen + 32), &finlen))
|
||||||
|
throw Exception("Error encrypting!");
|
||||||
|
|
||||||
|
out.resize(outlen + finlen + 32);
|
||||||
|
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<char> AES::decrypt(const std::vector<char> &in, const std::array<uint8_t, 32> &key) {
|
||||||
|
if (in.size() < 32) throw Exception("Array to decrypt is too small!");
|
||||||
|
|
||||||
|
std::unique_ptr<EVP_CIPHER_CTX, decltype(&EVP_CIPHER_CTX_free)> ctx(EVP_CIPHER_CTX_new(), &EVP_CIPHER_CTX_free);
|
||||||
|
if (!ctx) throw Exception("Error initializing encryption context!");
|
||||||
|
|
||||||
|
std::vector<char> out(in.size() - 32);
|
||||||
|
int outlen = static_cast<int>(out.size());
|
||||||
|
|
||||||
|
if (!EVP_DecryptInit_ex(ctx.get(), EVP_aes_256_cbc(), nullptr, key.data(), reinterpret_cast<const unsigned char *>(in.data())))
|
||||||
|
throw Exception("Error decrypting!");
|
||||||
|
|
||||||
|
|
||||||
|
if (!EVP_DecryptUpdate(ctx.get(), reinterpret_cast<unsigned char *>(out.data()), &outlen, reinterpret_cast<const unsigned char *>(in.data() + 32), static_cast<int>(in.size() - 32)))
|
||||||
|
throw Exception("Error decrypting!");
|
||||||
|
|
||||||
|
int finlen = 0;
|
||||||
|
if (!EVP_DecryptFinal_ex(ctx.get(), (unsigned char *) (out.data() + outlen), &finlen))
|
||||||
|
throw Exception("Error decrypting!");
|
||||||
|
|
||||||
|
out.resize(outlen + finlen);
|
||||||
|
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::array<uint8_t, 32> AES::deriveKey(const std::string &password, const std::string &salt) {
|
||||||
|
std::array<uint8_t, 32> key;//NOLINT
|
||||||
|
if (!PKCS5_PBKDF2_HMAC_SHA1(password.data(),
|
||||||
|
static_cast<int>(password.length()),
|
||||||
|
reinterpret_cast<const unsigned char *>(salt.data()),
|
||||||
|
static_cast<int>(salt.length()),
|
||||||
|
10000,
|
||||||
|
32,
|
||||||
|
key.data()))
|
||||||
|
throw Exception("Error deriving key!");
|
||||||
|
return key;
|
||||||
|
}
|
||||||
59
src/crypto/AES.h
Normal file
59
src/crypto/AES.h
Normal file
@@ -0,0 +1,59 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 30.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_AES_H
|
||||||
|
#define SEMBACKUP_AES_H
|
||||||
|
|
||||||
|
#include <array>
|
||||||
|
#include <cstdint>
|
||||||
|
#include <memory>
|
||||||
|
#include <string>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
/// Utility class to handle encryption/decryption of byte vectors
|
||||||
|
/**
|
||||||
|
* Based on: https://wiki.openssl.org/index.php/EVP_Symmetric_Encryption_and_Decryption
|
||||||
|
*/
|
||||||
|
class AES {
|
||||||
|
public:
|
||||||
|
/// Encrypts the provided \p in vector using \p password and \p salt
|
||||||
|
/// \param in Constant reference to to-be-encrypted vector
|
||||||
|
/// \param password Constant reference to the password
|
||||||
|
/// \param salt Constant reference to the salt
|
||||||
|
/// \return Encrypted vector of size at most original + 48 (16 for possible padding, 32 for the IV)
|
||||||
|
/// \throws Exception on any error
|
||||||
|
static std::vector<char> encrypt(const std::vector<char> &in, const std::string &password, const std::string &salt);
|
||||||
|
|
||||||
|
/// Decrypts the provided \p in vector using \p password and \p salt
|
||||||
|
/// \param in Constant reference to to-be-decrypted vector
|
||||||
|
/// \param password Constant reference to the password
|
||||||
|
/// \param salt Constant reference to the salt
|
||||||
|
/// \return Decrypted vector
|
||||||
|
/// \throws Exception on any error
|
||||||
|
static std::vector<char> decrypt(const std::vector<char> &in, const std::string &password, const std::string &salt);
|
||||||
|
|
||||||
|
/// Encrypts the provided \p in vector using \p key
|
||||||
|
/// \param in Constant reference to to-be-encrypted vector
|
||||||
|
/// \param key Constant reference to the key
|
||||||
|
/// \return Encrypted vector of size at most original + 48 (16 for possible padding, 32 for the IV)
|
||||||
|
/// \throws Exception on any error
|
||||||
|
static std::vector<char> encrypt(const std::vector<char> &in, const std::array<uint8_t, 32> &key);
|
||||||
|
|
||||||
|
/// Decrypts the provided \p in vector using \p key
|
||||||
|
/// \param in Constant reference to to-be-decrypted vector
|
||||||
|
/// \param key Constant reference to the key
|
||||||
|
/// \return Decrypted vector
|
||||||
|
/// \throws Exception on any error
|
||||||
|
static std::vector<char> decrypt(const std::vector<char> &in, const std::array<uint8_t, 32> &key);
|
||||||
|
|
||||||
|
/// Generates a key for the encryption using \p password and \p salt using PKCS5_PBKDF2_HMAC_SHA1
|
||||||
|
/// \param password Constant reference to the password
|
||||||
|
/// \param salt Constant reference to the salt
|
||||||
|
/// \return Derived key
|
||||||
|
/// \throws Exception on any error
|
||||||
|
static std::array<uint8_t, 32> deriveKey(const std::string &password, const std::string &salt);
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_AES_H
|
||||||
11
src/crypto/CRC32.cpp
Normal file
11
src/crypto/CRC32.cpp
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 12.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "CRC32.h"
|
||||||
|
|
||||||
|
CRC32::crcType CRC32::calculate(const std::vector<char> &in) {
|
||||||
|
crcType res = crc32(0L, nullptr, 0);
|
||||||
|
res = crc32(res, reinterpret_cast<const Bytef *>(in.data()), in.size());
|
||||||
|
return res;
|
||||||
|
}
|
||||||
25
src/crypto/CRC32.h
Normal file
25
src/crypto/CRC32.h
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 12.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_CRC32_H
|
||||||
|
#define SEMBACKUP_CRC32_H
|
||||||
|
|
||||||
|
#include <cstdint>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
#include <zlib.h>
|
||||||
|
|
||||||
|
/// Utility class to compute CRC32 values of vectors of chars
|
||||||
|
class CRC32 {
|
||||||
|
public:
|
||||||
|
using crcType = uLong;
|
||||||
|
|
||||||
|
/// Calculates the CRC32 of given vector
|
||||||
|
/// \param in Constant reference to a vector of chars
|
||||||
|
/// \return CRC32 result
|
||||||
|
static crcType calculate(const std::vector<char> &in);
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_CRC32_H
|
||||||
48
src/crypto/MD5.cpp
Normal file
48
src/crypto/MD5.cpp
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 15.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "MD5.h"
|
||||||
|
|
||||||
|
#include "../Exception.h"
|
||||||
|
|
||||||
|
std::string MD5::calculate(const std::vector<char> &in) {
|
||||||
|
MD5 hasher;
|
||||||
|
hasher.feedData(in);
|
||||||
|
return hasher.getHash();
|
||||||
|
}
|
||||||
|
|
||||||
|
MD5::MD5() {
|
||||||
|
if (!mdctx)
|
||||||
|
throw Exception("Can't create hashing context!");
|
||||||
|
|
||||||
|
if (!EVP_DigestInit_ex(mdctx.get(), EVP_md5(), nullptr))
|
||||||
|
throw Exception("Can't create hashing context!");
|
||||||
|
}
|
||||||
|
|
||||||
|
void MD5::feedData(const std::vector<char> &in) {
|
||||||
|
if (in.empty()) return;
|
||||||
|
if (!EVP_DigestUpdate(mdctx.get(), in.data(), in.size()))
|
||||||
|
throw Exception("Error hashing!");
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string MD5::getHash() {
|
||||||
|
std::array<char, 16> out;
|
||||||
|
unsigned int s = 0;
|
||||||
|
|
||||||
|
if (!EVP_DigestFinal_ex(mdctx.get(), reinterpret_cast<unsigned char *>(out.data()), &s))
|
||||||
|
throw Exception("Error hashing!");
|
||||||
|
|
||||||
|
if (s != out.size())
|
||||||
|
throw Exception("Error hashing!");
|
||||||
|
|
||||||
|
if (!EVP_MD_CTX_reset(mdctx.get()))
|
||||||
|
throw Exception("Error hashing!");
|
||||||
|
|
||||||
|
return {out.begin(), out.end()};
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string MD5::calculate(const std::string &in) {
|
||||||
|
std::vector<char> tmp(in.begin(), in.end());
|
||||||
|
return MD5::calculate(tmp);
|
||||||
|
}
|
||||||
48
src/crypto/MD5.h
Normal file
48
src/crypto/MD5.h
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 15.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_MD5_H
|
||||||
|
#define SEMBACKUP_MD5_H
|
||||||
|
|
||||||
|
#include <array>
|
||||||
|
#include <memory>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
#include <openssl/evp.h>
|
||||||
|
|
||||||
|
/// Class to handle MD5 hashing
|
||||||
|
/**
|
||||||
|
* Based on: https://wiki.openssl.org/index.php/EVP_Message_Digests
|
||||||
|
*/
|
||||||
|
class MD5 {
|
||||||
|
public:
|
||||||
|
/// Constructs an empty MD5 hasher instance
|
||||||
|
/// \throws Exception on initialization error
|
||||||
|
MD5();
|
||||||
|
|
||||||
|
/// Calculates the hash for a given \p in char vector
|
||||||
|
/// \param in Constant reference to an input vector
|
||||||
|
/// \return MD5 hash of \p in
|
||||||
|
static std::string calculate(const std::vector<char> &in);
|
||||||
|
|
||||||
|
/// Calculates the hash for a given \p in string
|
||||||
|
/// \param in Constant reference to an input string
|
||||||
|
/// \return MD5 hash of \p in
|
||||||
|
static std::string calculate(const std::string &in);
|
||||||
|
|
||||||
|
/// Append a vector of chars to the current hash
|
||||||
|
/// \param in Constant reference to an input vector
|
||||||
|
/// \throws Exception on any error
|
||||||
|
void feedData(const std::vector<char> &in);
|
||||||
|
|
||||||
|
/// Returns the hash, resets the hashing context
|
||||||
|
/// \throws Exception on any error
|
||||||
|
std::string getHash();
|
||||||
|
|
||||||
|
private:
|
||||||
|
const std::unique_ptr<EVP_MD_CTX, decltype(&EVP_MD_CTX_free)> mdctx{EVP_MD_CTX_new(), &EVP_MD_CTX_free};///< Current hashing context
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_MD5_H
|
||||||
47
src/filters/CheckFilter.cpp
Normal file
47
src/filters/CheckFilter.cpp
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 12.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "CheckFilter.h"
|
||||||
|
#include "../crypto/CRC32.h"
|
||||||
|
#include "../repo/Serialize.h"
|
||||||
|
|
||||||
|
std::vector<char> CheckFilter::filterWrite(std::vector<char> from) const {
|
||||||
|
return filterWriteStatic(std::move(from));
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<char> CheckFilter::filterRead(std::vector<char> from) const {
|
||||||
|
return filterReadStatic(std::move(from));
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<char> CheckFilter::filterWriteStatic(std::vector<char> from) {
|
||||||
|
auto out = magic;
|
||||||
|
|
||||||
|
Serialize::serialize(from, out);
|
||||||
|
|
||||||
|
auto crc = CRC32::calculate(from);
|
||||||
|
|
||||||
|
Serialize::serialize(crc, out);
|
||||||
|
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<char> CheckFilter::filterReadStatic(std::vector<char> from) {
|
||||||
|
if (from.size() < magic.size()) throw Exception("Input is corrupted (too small)!");
|
||||||
|
|
||||||
|
for (size_t i = 0; i < magic.size(); i++) {
|
||||||
|
if (from[i] != magic[i]) throw Exception("Magic prefix is wrong!");
|
||||||
|
}
|
||||||
|
|
||||||
|
auto fromIt = from.cbegin() + magic.size();
|
||||||
|
|
||||||
|
auto out = Serialize::deserialize<std::vector<char>>(fromIt, from.cend());
|
||||||
|
|
||||||
|
auto crc = CRC32::calculate(out);
|
||||||
|
|
||||||
|
auto crcRecorded = Serialize::deserialize<CRC32::crcType>(fromIt, from.cend());
|
||||||
|
|
||||||
|
if (crc != crcRecorded) throw Exception("CRC mismatch!");
|
||||||
|
|
||||||
|
return out;
|
||||||
|
}
|
||||||
36
src/filters/CheckFilter.h
Normal file
36
src/filters/CheckFilter.h
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 12.05.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_CHECKFILTER_H
|
||||||
|
#define SEMBACKUP_CHECKFILTER_H
|
||||||
|
|
||||||
|
#include "Filter.h"
|
||||||
|
|
||||||
|
/// Filter implementation that checks the input for corruption using CRC
|
||||||
|
/**
|
||||||
|
* Additionally, it has static methods for work outside FilterContainer%s
|
||||||
|
*/
|
||||||
|
class CheckFilter : public Filter {
|
||||||
|
public:
|
||||||
|
/// \copydoc Filter::filterWrite
|
||||||
|
/// \copydoc CheckFilter::filterWriteS
|
||||||
|
std::vector<char> filterWrite(std::vector<char> from) const override;
|
||||||
|
|
||||||
|
/// \copydoc Filter::filterRead
|
||||||
|
/// \copydoc CheckFilter::filterReadS
|
||||||
|
std::vector<char> filterRead(std::vector<char> from) const override;
|
||||||
|
|
||||||
|
/// Adds CRC hash and magic string to the the \p from vector
|
||||||
|
static std::vector<char> filterWriteStatic(std::vector<char> from);
|
||||||
|
|
||||||
|
/// Checks the \p from vector and removes the metadata
|
||||||
|
/// \throws Exception on any error
|
||||||
|
static std::vector<char> filterReadStatic(std::vector<char> from);
|
||||||
|
|
||||||
|
private:
|
||||||
|
static const inline std::vector<char> magic{'s', 'e', 'm', 'b', 'a'};
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_CHECKFILTER_H
|
||||||
6
src/filters/Filter.cpp
Normal file
6
src/filters/Filter.cpp
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 22.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "Filter.h"
|
||||||
|
Filter::~Filter() = default;
|
||||||
30
src/filters/Filter.h
Normal file
30
src/filters/Filter.h
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 22.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_FILTER_H
|
||||||
|
#define SEMBACKUP_FILTER_H
|
||||||
|
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
/// Interface class for I/O filters
|
||||||
|
class Filter {
|
||||||
|
public:
|
||||||
|
/// Applies the filter to \p from vector and returns the result
|
||||||
|
/// Note: the vector is passed by value, as it allows to avoid copying with std::move in case the filter modifies the \p in vector in-place
|
||||||
|
/// \param from Source vector of chars
|
||||||
|
/// \return Filtered vector of chars
|
||||||
|
virtual std::vector<char> filterWrite(std::vector<char> from) const = 0;
|
||||||
|
|
||||||
|
/// Reverses the applied filter from \p from vector and returns the result
|
||||||
|
/// Note: the vector is passed by value, as it allows to avoid copying with std::move in case the filter modifies the \p in vector in-place
|
||||||
|
/// \param from Source vector of chars
|
||||||
|
/// \return Filtered vector of chars
|
||||||
|
virtual std::vector<char> filterRead(std::vector<char> from) const = 0;
|
||||||
|
|
||||||
|
/// Default virtual destructor
|
||||||
|
virtual ~Filter();
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_FILTER_H
|
||||||
17
src/filters/FilterAES.cpp
Normal file
17
src/filters/FilterAES.cpp
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 23.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "FilterAES.h"
|
||||||
|
|
||||||
|
#include "../crypto/AES.h"
|
||||||
|
|
||||||
|
std::vector<char> FilterAES::filterWrite(std::vector<char> from) const {
|
||||||
|
return AES::encrypt(from, key);
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<char> FilterAES::filterRead(std::vector<char> from) const {
|
||||||
|
return AES::decrypt(from, key);
|
||||||
|
}
|
||||||
|
|
||||||
|
FilterAES::FilterAES(const std::string &password, const std::string &salt) : key(AES::deriveKey(password, salt)) {}
|
||||||
37
src/filters/FilterAES.h
Normal file
37
src/filters/FilterAES.h
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 23.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_FILTERAES_H
|
||||||
|
#define SEMBACKUP_FILTERAES_H
|
||||||
|
|
||||||
|
#include <array>
|
||||||
|
#include <cstdint>
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
#include "Filter.h"
|
||||||
|
|
||||||
|
/// Filter implementation that encrypts/decrypts data using provided password and salt
|
||||||
|
class FilterAES : public Filter {
|
||||||
|
public:
|
||||||
|
/// Constructs the filter, using \p password and \p salt to generate the encryption key
|
||||||
|
/// \param password Constant reference to password string
|
||||||
|
/// \param salt Constant reference to salt string
|
||||||
|
FilterAES(const std::string &password, const std::string &salt);
|
||||||
|
|
||||||
|
/// Encrypts the \p from vector
|
||||||
|
/// \copydoc Filter::filterWrite
|
||||||
|
/// \throws Exception on any error
|
||||||
|
std::vector<char> filterWrite(std::vector<char> from) const override;
|
||||||
|
|
||||||
|
/// Decrypts the \p from vector
|
||||||
|
/// \copydoc Filter::filterRead
|
||||||
|
/// \throws Exception on any error
|
||||||
|
std::vector<char> filterRead(std::vector<char> from) const override;
|
||||||
|
|
||||||
|
private:
|
||||||
|
const std::array<uint8_t, 32> key;///< Key used for encryption, derived from \p password and \p salt
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_FILTERAES_H
|
||||||
23
src/filters/FilterContainer.cpp
Normal file
23
src/filters/FilterContainer.cpp
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 22.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "FilterContainer.h"
|
||||||
|
|
||||||
|
FilterContainer::FilterContainer() = default;
|
||||||
|
|
||||||
|
FilterContainer &FilterContainer::addFilter(std::unique_ptr<Filter> &&f) {
|
||||||
|
filters.emplace_back(std::move(f));
|
||||||
|
return *this;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<char> FilterContainer::filterWrite(std::vector<char> from) const {
|
||||||
|
for (auto const &f: filters) from = f->filterWrite(std::move(from));
|
||||||
|
return from;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<char> FilterContainer::filterRead(std::vector<char> from) const {
|
||||||
|
for (auto f = filters.crbegin(); f != filters.crend(); f++)
|
||||||
|
from = (*f)->filterRead(std::move(from));
|
||||||
|
return from;
|
||||||
|
}
|
||||||
37
src/filters/FilterContainer.h
Normal file
37
src/filters/FilterContainer.h
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 22.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_FILTERCONTAINER_H
|
||||||
|
#define SEMBACKUP_FILTERCONTAINER_H
|
||||||
|
|
||||||
|
#include <memory>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
#include "Filter.h"
|
||||||
|
|
||||||
|
/// Convenience Filter implementation, that applies multiple Filter%s in succession
|
||||||
|
class FilterContainer : public Filter {
|
||||||
|
public:
|
||||||
|
/// Constructs an empty FilterContainer
|
||||||
|
FilterContainer();
|
||||||
|
|
||||||
|
/// Adds a Filter into itself
|
||||||
|
/// \param f Rvalue reference to a unique pointer to Filter
|
||||||
|
/// \return Reference to itself
|
||||||
|
FilterContainer &addFilter(std::unique_ptr<Filter> &&f);
|
||||||
|
|
||||||
|
/// Applies the filters in order of insertion
|
||||||
|
/// \copydoc Filter::filterWrite
|
||||||
|
std::vector<char> filterWrite(std::vector<char> from) const override;
|
||||||
|
|
||||||
|
/// Applies the filters in reverse order of insertion
|
||||||
|
/// \copydoc Filter::filterRead
|
||||||
|
std::vector<char> filterRead(std::vector<char> from) const override;
|
||||||
|
|
||||||
|
private:
|
||||||
|
std::vector<std::unique_ptr<Filter>> filters;///< Vector of unique pointers to Filter%s
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_FILTERCONTAINER_H
|
||||||
32
src/filters/FilterFactory.cpp
Normal file
32
src/filters/FilterFactory.cpp
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 22.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "FilterFactory.h"
|
||||||
|
|
||||||
|
#include "../Exception.h"
|
||||||
|
#include "CheckFilter.h"
|
||||||
|
#include "FilterAES.h"
|
||||||
|
#include "FilterShift.h"
|
||||||
|
#include "FilterShiftSecret.h"
|
||||||
|
#include "FilterZlib.h"
|
||||||
|
|
||||||
|
std::unique_ptr<Filter> FilterFactory::makeFilter(const std::string &type, const Config &config) {
|
||||||
|
if (type == "none") throw Exception("Trying to make a \"none\" filter!");
|
||||||
|
|
||||||
|
if (type == "aes") {
|
||||||
|
return std::make_unique<FilterAES>(config.getStr("password"), config.getStr("salt"));
|
||||||
|
} else if (type == "zlib") {
|
||||||
|
return std::make_unique<FilterZlib>(config.getInt("compression-level"));
|
||||||
|
} else if (type == "crc") {
|
||||||
|
return std::make_unique<CheckFilter>();
|
||||||
|
}
|
||||||
|
#ifdef TEST
|
||||||
|
else if (type == "shiftC") {
|
||||||
|
return std::make_unique<FilterShift>(config.getInt("compression-level"));
|
||||||
|
} else if (type == "shiftE")
|
||||||
|
return std::make_unique<FilterShiftSecret>(config.getStr("password"), config.getStr("salt"));
|
||||||
|
#endif
|
||||||
|
|
||||||
|
throw Exception("Unknown filter value");
|
||||||
|
}
|
||||||
25
src/filters/FilterFactory.h
Normal file
25
src/filters/FilterFactory.h
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 22.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_FILTERFACTORY_H
|
||||||
|
#define SEMBACKUP_FILTERFACTORY_H
|
||||||
|
|
||||||
|
#include <memory>
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
#include "../Config.h"
|
||||||
|
#include "Filter.h"
|
||||||
|
|
||||||
|
/// Utility factory class for Filter%s
|
||||||
|
class FilterFactory {
|
||||||
|
public:
|
||||||
|
/// Constructs a Filter of type \p type according to \p config
|
||||||
|
/// \param type Constant reference to a string containing the type of filter to construct
|
||||||
|
/// \param config Constant reference to Config which will be used to determine constructed Filter%'s parameters
|
||||||
|
/// \return Unique pointer to the constructed Filter
|
||||||
|
static std::unique_ptr<Filter> makeFilter(const std::string &type, const Config &config);
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_FILTERFACTORY_H
|
||||||
18
src/filters/FilterShift.cpp
Normal file
18
src/filters/FilterShift.cpp
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 22.04.2023.
|
||||||
|
//
|
||||||
|
#ifdef TEST
|
||||||
|
#include "FilterShift.h"
|
||||||
|
|
||||||
|
std::vector<char> FilterShift::filterWrite(std::vector<char> from) const {
|
||||||
|
for (auto &c: from) c += shiftVal;
|
||||||
|
return from;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<char> FilterShift::filterRead(std::vector<char> from) const {
|
||||||
|
for (auto &c: from) c -= shiftVal;
|
||||||
|
return from;
|
||||||
|
}
|
||||||
|
|
||||||
|
FilterShift::FilterShift(int level) : shiftVal(level) {}
|
||||||
|
#endif
|
||||||
30
src/filters/FilterShift.h
Normal file
30
src/filters/FilterShift.h
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 22.04.2023.
|
||||||
|
//
|
||||||
|
#ifdef TEST
|
||||||
|
#ifndef SEMBACKUP_FILTERSHIFT_H
|
||||||
|
#define SEMBACKUP_FILTERSHIFT_H
|
||||||
|
|
||||||
|
#include "Filter.h"
|
||||||
|
|
||||||
|
/// Filter implementation that shifts every byte in input vector using provided value
|
||||||
|
/// \warning For testing purposes only!
|
||||||
|
class FilterShift : public Filter {
|
||||||
|
public:
|
||||||
|
/// Constructs the filter using \p level as shift value
|
||||||
|
/// \param level Number that will be added to each input byte
|
||||||
|
FilterShift(int level);
|
||||||
|
|
||||||
|
/// \copydoc Filter::filterWrite
|
||||||
|
std::vector<char> filterWrite(std::vector<char> from) const override;
|
||||||
|
|
||||||
|
/// \copydoc Filter::filterRead
|
||||||
|
std::vector<char> filterRead(std::vector<char> from) const override;
|
||||||
|
|
||||||
|
private:
|
||||||
|
int shiftVal;///< Value to add to input bytes
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_FILTERSHIFT_H
|
||||||
|
#endif//TEST
|
||||||
23
src/filters/FilterShiftSecret.cpp
Normal file
23
src/filters/FilterShiftSecret.cpp
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 23.04.2023.
|
||||||
|
//
|
||||||
|
#ifdef TEST
|
||||||
|
|
||||||
|
#include "FilterShiftSecret.h"
|
||||||
|
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
std::vector<char> FilterShiftSecret::filterWrite(std::vector<char> from) const {
|
||||||
|
for (auto &c: from) c += shiftVal;
|
||||||
|
return from;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<char> FilterShiftSecret::filterRead(std::vector<char> from) const {
|
||||||
|
for (auto &c: from) c -= shiftVal;
|
||||||
|
return from;
|
||||||
|
}
|
||||||
|
|
||||||
|
FilterShiftSecret::FilterShiftSecret(const std::string &password, const std::string &salt) {
|
||||||
|
shiftVal = password[0] + salt[0];
|
||||||
|
}
|
||||||
|
#endif
|
||||||
33
src/filters/FilterShiftSecret.h
Normal file
33
src/filters/FilterShiftSecret.h
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 23.04.2023.
|
||||||
|
//
|
||||||
|
#ifdef TEST
|
||||||
|
#ifndef SEMBACKUP_FILTERSHIFTSECRET_H
|
||||||
|
#define SEMBACKUP_FILTERSHIFTSECRET_H
|
||||||
|
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
#include "Filter.h"
|
||||||
|
|
||||||
|
/// Filter implementation that shifts every byte in input vector using two provided value
|
||||||
|
/// \warning For testing purposes only!
|
||||||
|
class FilterShiftSecret : public Filter {
|
||||||
|
public:
|
||||||
|
/// Constructs the filter using the sum of first bytes of \p password and \p salt to initialize shiftVal
|
||||||
|
/// \param password Constant reference to "password" string
|
||||||
|
/// \param salt Constant reference to "salt" string
|
||||||
|
FilterShiftSecret(const std::string &password, const std::string &salt);
|
||||||
|
|
||||||
|
/// \copydoc Filter::filterWrite
|
||||||
|
std::vector<char> filterWrite(std::vector<char> from) const override;
|
||||||
|
|
||||||
|
/// \copydoc Filter::filterRead
|
||||||
|
std::vector<char> filterRead(std::vector<char> from) const override;
|
||||||
|
|
||||||
|
private:
|
||||||
|
int shiftVal = 0;///< Value to add to input bytes
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_FILTERSHIFTSECRET_H
|
||||||
|
#endif//TEST
|
||||||
50
src/filters/FilterZlib.cpp
Normal file
50
src/filters/FilterZlib.cpp
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 23.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "FilterZlib.h"
|
||||||
|
|
||||||
|
#include <zlib.h>
|
||||||
|
|
||||||
|
#include "../repo/Serialize.h"
|
||||||
|
|
||||||
|
std::vector<char> FilterZlib::filterWrite(std::vector<char> from) const {
|
||||||
|
uLongf outSize = compressBound(from.size());
|
||||||
|
|
||||||
|
std::vector<char> out;
|
||||||
|
Serialize::serialize('C', out);
|
||||||
|
Serialize::serialize(static_cast<unsigned long long>(from.size()), out);
|
||||||
|
|
||||||
|
uLongf sizeSize = out.size();
|
||||||
|
|
||||||
|
out.resize(sizeSize + outSize);
|
||||||
|
|
||||||
|
if (compress2(reinterpret_cast<Bytef *>(out.data() + sizeSize), &outSize, reinterpret_cast<const Bytef *>(from.data()), from.size(), level) !=
|
||||||
|
Z_OK)
|
||||||
|
throw Exception("Error compressing!");
|
||||||
|
|
||||||
|
out.resize(outSize + sizeSize);
|
||||||
|
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<char> FilterZlib::filterRead(std::vector<char> from) const {
|
||||||
|
auto desI = from.cbegin();
|
||||||
|
|
||||||
|
char C = Serialize::deserialize<char>(desI, from.cend());
|
||||||
|
if (C != 'C') throw Exception("Bad compression prefix!");
|
||||||
|
|
||||||
|
uLongf size = Serialize::deserialize<unsigned long long>(desI, from.cend());
|
||||||
|
|
||||||
|
std::vector<char> out(size);
|
||||||
|
|
||||||
|
if (desI >= from.cend()) throw Exception("Unexpected end of archive!");
|
||||||
|
|
||||||
|
if (uncompress(reinterpret_cast<Bytef *>(out.data()), &size, reinterpret_cast<const Bytef *>(&(*desI)), std::distance(desI, from.cend())) !=
|
||||||
|
Z_OK)
|
||||||
|
throw Exception("Error decompressing!");
|
||||||
|
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
FilterZlib::FilterZlib(int level) : level(level) {}
|
||||||
31
src/filters/FilterZlib.h
Normal file
31
src/filters/FilterZlib.h
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 23.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_FILTERZLIB_H
|
||||||
|
#define SEMBACKUP_FILTERZLIB_H
|
||||||
|
|
||||||
|
#include "Filter.h"
|
||||||
|
|
||||||
|
/// Filter implementation that uses Zlib to compress data
|
||||||
|
class FilterZlib : public Filter {
|
||||||
|
public:
|
||||||
|
/// Creates the filter using \p level as compression level
|
||||||
|
/// \param level
|
||||||
|
FilterZlib(int level);
|
||||||
|
|
||||||
|
/// Compresses the \p from vector
|
||||||
|
/// \copydoc Filter::filterWrite
|
||||||
|
/// \throws Exception on any error
|
||||||
|
std::vector<char> filterWrite(std::vector<char> from) const override;
|
||||||
|
|
||||||
|
/// Decompresses the \p from vector
|
||||||
|
/// \copydoc Filter::filterRead
|
||||||
|
/// \throws Exception on any error
|
||||||
|
std::vector<char> filterRead(std::vector<char> from) const override;
|
||||||
|
|
||||||
|
private:
|
||||||
|
int level = -1;///< Compression level to use, -1 is the Zlib default
|
||||||
|
};
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_FILTERZLIB_H
|
||||||
129
src/main.cpp
Normal file
129
src/main.cpp
Normal file
@@ -0,0 +1,129 @@
|
|||||||
|
|
||||||
|
#include <iostream>
|
||||||
|
|
||||||
|
#include "BytesFormatter.h"
|
||||||
|
#include "Config.h"
|
||||||
|
#include "Context.h"
|
||||||
|
#include "Exception.h"
|
||||||
|
#include "Logger.h"
|
||||||
|
#include "Signals.h"
|
||||||
|
#include "commands/Command.h"
|
||||||
|
#include "commands/CommandDiff.h"
|
||||||
|
#include "commands/CommandList.h"
|
||||||
|
#include "commands/CommandListFiles.h"
|
||||||
|
#include "commands/CommandRestore.h"
|
||||||
|
#include "commands/CommandRun.h"
|
||||||
|
#include "repo/FileRepository.h"
|
||||||
|
#include "repo/Repository.h"
|
||||||
|
#include "repo/Serialize.h"
|
||||||
|
#include "repo/objects/Archive.h"
|
||||||
|
#include "repo/objects/File.h"
|
||||||
|
|
||||||
|
Config getConf(int argc, char *argv[]) {
|
||||||
|
Config out;
|
||||||
|
for (int i = 0; i < argc; i++) {
|
||||||
|
std::string key = argv[i];
|
||||||
|
if (key.substr(0, 2) != "--") throw Exception("Options should start with --");
|
||||||
|
key = key.substr(2);
|
||||||
|
if (++i == argc) throw Exception("Option not specified for " + key);
|
||||||
|
std::string val = argv[i];
|
||||||
|
out.add(key, val);
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
int help() {
|
||||||
|
for (auto const &o: Config::keys) {
|
||||||
|
std::cout << "--" << o.first << " <" << Config::KeyTypeToStr.at(o.second.type) << ">" << std::endl;
|
||||||
|
if (o.second.defaultval.has_value())
|
||||||
|
std::cout << " Default: " << o.second.defaultval.value() << std::endl;
|
||||||
|
std::cout << " Is saved in repository: " << (o.second.remember ? "yes" : "no") << std::endl;
|
||||||
|
std::cout << " Info: " << o.second.info << std::endl;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::unique_ptr<Repository> openRepo(Config &conf) {
|
||||||
|
try {
|
||||||
|
auto repo = std::make_unique<FileRepository>(conf);
|
||||||
|
repo->open();
|
||||||
|
return repo;
|
||||||
|
} catch (std::exception &e) {
|
||||||
|
std::cout << "Error opening repo: " << e.what() << std::endl;
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
int init(Config &conf) {
|
||||||
|
auto repo = std::make_unique<FileRepository>(conf);
|
||||||
|
if (repo == nullptr) return -1;
|
||||||
|
|
||||||
|
try {
|
||||||
|
repo->init();
|
||||||
|
} catch (std::exception &e) {
|
||||||
|
std::cout << "Error initializing repo: " << e.what() << std::endl;
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
int main(int argc, char *argv[]) {
|
||||||
|
try {
|
||||||
|
Signals::setup();
|
||||||
|
|
||||||
|
if (argc < 2) {
|
||||||
|
std::cerr << "No argument specified" << std::endl;
|
||||||
|
help();
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string opt = argv[1];
|
||||||
|
if (opt == "help") {
|
||||||
|
return help();
|
||||||
|
}
|
||||||
|
|
||||||
|
Config conf;
|
||||||
|
|
||||||
|
try {
|
||||||
|
conf = getConf(argc - 2, argv + 2);
|
||||||
|
} catch (std::exception &e) {
|
||||||
|
std::cerr << "Error reading config!" << std::endl
|
||||||
|
<< e.what() << std::endl;
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (opt == "init") {
|
||||||
|
return init(conf);
|
||||||
|
}
|
||||||
|
|
||||||
|
auto repo = openRepo(conf);
|
||||||
|
|
||||||
|
if (repo == nullptr) {
|
||||||
|
std::cerr << "Can't open repo!" << std::endl;
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
Logger logger(conf.getInt("verbose"));
|
||||||
|
Context ctx{&logger, repo.get()};
|
||||||
|
|
||||||
|
std::unordered_map<std::string, std::unique_ptr<Command>> commands;
|
||||||
|
commands.emplace(CommandDiff().name, std::make_unique<CommandDiff>());
|
||||||
|
commands.emplace(CommandRestore().name, std::make_unique<CommandRestore>());
|
||||||
|
commands.emplace(CommandRun().name, std::make_unique<CommandRun>());
|
||||||
|
commands.emplace(CommandListFiles().name, std::make_unique<CommandListFiles>());
|
||||||
|
commands.emplace(CommandList().name, std::make_unique<CommandList>());
|
||||||
|
|
||||||
|
if (commands.count(opt) == 0) {
|
||||||
|
std::cerr << "Unknown argument" << std::endl;
|
||||||
|
return -1;
|
||||||
|
} else {
|
||||||
|
commands.at(opt)->run(ctx);
|
||||||
|
}
|
||||||
|
} catch (std::exception &e) {
|
||||||
|
std::cerr << "Error!" << std::endl
|
||||||
|
<< e.what() << std::endl;
|
||||||
|
} catch (...) {
|
||||||
|
std::cerr << "Something very bad happened!" << std::endl;
|
||||||
|
}
|
||||||
|
}
|
||||||
246
src/repo/FileRepository.cpp
Normal file
246
src/repo/FileRepository.cpp
Normal file
@@ -0,0 +1,246 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 14.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "FileRepository.h"
|
||||||
|
|
||||||
|
#include <exception>
|
||||||
|
#include <iterator>
|
||||||
|
#include <mutex>
|
||||||
|
|
||||||
|
#include "../filters/CheckFilter.h"
|
||||||
|
#include "../filters/FilterFactory.h"
|
||||||
|
#include "Object.h"
|
||||||
|
#include "Serialize.h"
|
||||||
|
|
||||||
|
FileRepository::FileRepository(Config config) : Repository(std::move(config)), root(std::filesystem::path(this->config.getStr("repo"))), writeCacheMax(config.getInt("repo-target") * 1024 * 1024) {}
|
||||||
|
|
||||||
|
bool FileRepository::exists() {
|
||||||
|
return std::filesystem::is_directory(root) && std::filesystem::exists(root / "info");
|
||||||
|
}
|
||||||
|
|
||||||
|
bool FileRepository::flush() {
|
||||||
|
flushWriteCache(std::unique_lock(writeCacheLock));
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool FileRepository::open() {
|
||||||
|
if (!exists()) throw Exception("Repository doesn't exist!");
|
||||||
|
|
||||||
|
auto readConf = Serialize::deserialize<Config>(CheckFilter::filterReadStatic(readFile(root / "info")));
|
||||||
|
std::swap(config, readConf);
|
||||||
|
config.merge(readConf);
|
||||||
|
|
||||||
|
if (config.getStr("compression") != "none") filters.addFilter(FilterFactory::makeFilter(config.getStr("compression"), config));
|
||||||
|
if (config.getStr("encryption") != "none") filters.addFilter(FilterFactory::makeFilter(config.getStr("encryption"), config));
|
||||||
|
filters.addFilter(FilterFactory::makeFilter("crc", config));
|
||||||
|
|
||||||
|
ready = true;
|
||||||
|
try {
|
||||||
|
std::tie(maxFileId, offsetIndex) = Serialize::deserialize<std::pair<decltype(maxFileId), decltype(offsetIndex)>>(filters.filterRead(readFile(root / "offsets")));
|
||||||
|
std::tie(keyIndex, largestUnusedId) = Serialize::deserialize<std::pair<decltype(keyIndex), decltype(largestUnusedId)>>(filters.filterRead(readFile(root / "index")));
|
||||||
|
} catch (const std::exception &e) {
|
||||||
|
ready = false;
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool FileRepository::init() {
|
||||||
|
if (ready) throw Exception("Trying to initialize already initialized repository!");
|
||||||
|
if (exists()) throw Exception("Trying to initialize already existing repository!");
|
||||||
|
|
||||||
|
if (!std::filesystem::is_directory(root) && !std::filesystem::create_directories(root))
|
||||||
|
throw Exception("Can't create directory " + root.u8string());
|
||||||
|
|
||||||
|
writeFile(root / "info", CheckFilter::filterWriteStatic(Serialize::serialize(config)));
|
||||||
|
|
||||||
|
if (config.getStr("compression") != "none") filters.addFilter(FilterFactory::makeFilter(config.getStr("compression"), config));
|
||||||
|
if (config.getStr("encryption") != "none") filters.addFilter(FilterFactory::makeFilter(config.getStr("encryption"), config));
|
||||||
|
filters.addFilter(FilterFactory::makeFilter("crc", config));
|
||||||
|
|
||||||
|
ready = true;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
FileRepository::~FileRepository() {
|
||||||
|
if (ready) {
|
||||||
|
ready = false;
|
||||||
|
flushWriteCache(std::unique_lock(writeCacheLock));
|
||||||
|
|
||||||
|
writeFile(root / "offsets", filters.filterWrite(Serialize::serialize(std::make_pair(maxFileId, offsetIndex))));
|
||||||
|
writeFile(root / "index", filters.filterWrite(Serialize::serialize(std::make_pair(keyIndex, largestUnusedId))));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<char> FileRepository::getObject(Object::idType id) const {
|
||||||
|
if (!ready) throw Exception("Tried working with uninitialized repo!");
|
||||||
|
|
||||||
|
std::unique_lock lock(repoLock);
|
||||||
|
if (offsetIndex.count(id) == 0)
|
||||||
|
throw Exception("Object with id " + std::to_string(id) + " doesn't exist!");
|
||||||
|
auto entry = offsetIndex.at(id);
|
||||||
|
lock.unlock();
|
||||||
|
|
||||||
|
return filters.filterRead(readFile(root / std::to_string(entry.fileId), entry.offset, entry.length));
|
||||||
|
}
|
||||||
|
|
||||||
|
bool FileRepository::writeObject(const Object &obj) {
|
||||||
|
if (!ready) throw Exception("Tried working with uninitialized repo!");
|
||||||
|
auto tmp = filters.filterWrite(Serialize::serialize(obj));
|
||||||
|
{
|
||||||
|
std::unique_lock lockW(writeCacheLock);
|
||||||
|
writeCacheSize += tmp.size();
|
||||||
|
writeCache[obj.id] = std::move(tmp);
|
||||||
|
|
||||||
|
// If we have reached the target file size, flush the cache
|
||||||
|
if (writeCacheSize >= writeCacheMax) {
|
||||||
|
flushWriteCache(std::move(lockW));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
void FileRepository::flushWriteCache(std::unique_lock<std::mutex> &&lockW) {
|
||||||
|
if (writeCache.empty()) {
|
||||||
|
lockW.unlock();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Swap the cache for a new one and unlock the mutex so other threads can continue working
|
||||||
|
decltype(writeCache) objs;
|
||||||
|
std::swap(writeCache, objs);
|
||||||
|
writeCacheSize = 0;
|
||||||
|
|
||||||
|
decltype(maxFileId) currentFileId;
|
||||||
|
{
|
||||||
|
std::lock_guard lockI(repoLock);
|
||||||
|
currentFileId = maxFileId;
|
||||||
|
maxFileId++;
|
||||||
|
}
|
||||||
|
lockW.unlock();
|
||||||
|
|
||||||
|
unsigned long long offset = 0;
|
||||||
|
std::ofstream ofstream(root / std::to_string(currentFileId), std::ios::binary | std::ios::trunc | std::ios::out);
|
||||||
|
|
||||||
|
for (auto &i: objs) {
|
||||||
|
{
|
||||||
|
std::lock_guard lockI(repoLock);
|
||||||
|
offsetIndex.emplace(i.first, OffsetEntry(currentFileId, offset, i.second.size()));
|
||||||
|
}
|
||||||
|
offset += i.second.size();
|
||||||
|
ofstream.rdbuf()->sputn(i.second.data(), i.second.size());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool FileRepository::putObject(const Object &obj) {
|
||||||
|
// Put the object into index, and then write it to the storage
|
||||||
|
{
|
||||||
|
std::lock_guard lock(repoLock);
|
||||||
|
keyIndex[obj.type][obj.getKey()] = obj.id;
|
||||||
|
}
|
||||||
|
writeObject(obj);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool FileRepository::deleteObject(const Object &obj) {
|
||||||
|
if (!ready) throw Exception("Tried working with uninitialized repo!");
|
||||||
|
throw Exception("Deletion not implemented!");
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<char> FileRepository::readFile(const std::filesystem::path &file, unsigned long long offset, unsigned long long size) const {
|
||||||
|
if (size > absoluteMaxFileLimit) throw Exception("Tried to read " + std::to_string(size) +
|
||||||
|
" bytes from " + file.u8string() +
|
||||||
|
" which is more than absoluteMaxFileLimit");
|
||||||
|
|
||||||
|
std::ifstream ifstream(file, std::ios::binary | std::ios::in);
|
||||||
|
if (!ifstream.is_open()) throw Exception("Can't open file " + file.u8string() + " for reading!");
|
||||||
|
|
||||||
|
std::vector<char> buf(size);
|
||||||
|
|
||||||
|
if (ifstream.rdbuf()->pubseekpos(offset) == std::streampos(std::streamoff(-1))) throw Exception("Unexpected end of file " + file.u8string());
|
||||||
|
if (ifstream.rdbuf()->sgetn(buf.data(), size) != size) throw Exception("Unexpected end of file " + file.u8string());
|
||||||
|
|
||||||
|
return buf;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<char> FileRepository::readFile(const std::filesystem::path &file) const {
|
||||||
|
if (!std::filesystem::is_regular_file(file)) throw Exception("File " + file.u8string() + " is not a regular file!");
|
||||||
|
auto fileSize = std::filesystem::file_size(file);
|
||||||
|
if (fileSize == 0) return {};
|
||||||
|
return readFile(file, 0, fileSize);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool FileRepository::writeFile(const std::filesystem::path &file, const std::vector<char> &data) {
|
||||||
|
std::ofstream ofstream(file, std::ios::binary | std::ios::trunc | std::ios::out);
|
||||||
|
if (!ofstream.is_open()) throw Exception("Can't open file " + file.u8string() + " for writing!");
|
||||||
|
|
||||||
|
if (ofstream.rdbuf()->sputn(data.data(), data.size()) != data.size())
|
||||||
|
throw Exception("Couldn't write all the data for " + file.u8string());
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<char> FileRepository::getObject(Object::ObjectType type, const std::string &key) const {
|
||||||
|
return getObject(getObjectId(type, key));
|
||||||
|
}
|
||||||
|
|
||||||
|
Object::idType FileRepository::getObjectId(Object::ObjectType type, const std::string &key) const {
|
||||||
|
std::lock_guard lock(repoLock);
|
||||||
|
if (keyIndex.count(type) == 0) throw Exception("No objects of requested type!");
|
||||||
|
return keyIndex.at(type).at(key);
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<std::pair<std::string, Object::idType>> FileRepository::getObjects(Object::ObjectType type) const {
|
||||||
|
std::lock_guard lock(repoLock);
|
||||||
|
std::vector<std::pair<std::string, Object::idType>> out;
|
||||||
|
if (keyIndex.count(type) == 0) return {};
|
||||||
|
for (auto const &i: keyIndex.at(type))
|
||||||
|
out.emplace_back(i);
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool FileRepository::exists(Object::ObjectType type, const std::string &key) const {
|
||||||
|
std::lock_guard lock(repoLock);
|
||||||
|
if (keyIndex.count(type) == 0) return false;
|
||||||
|
return keyIndex.at(type).count(key) > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
Object::idType FileRepository::getId() {
|
||||||
|
std::lock_guard lock(repoLock);
|
||||||
|
return largestUnusedId++;
|
||||||
|
}
|
||||||
|
|
||||||
|
FileRepository::OffsetEntry::OffsetEntry(std::vector<char, std::allocator<char>>::const_iterator &in, const std::vector<char, std::allocator<char>>::const_iterator &end)
|
||||||
|
: fileId(Serialize::deserialize<decltype(fileId)>(in, end)),
|
||||||
|
offset(Serialize::deserialize<decltype(offset)>(in, end)),
|
||||||
|
length(Serialize::deserialize<decltype(length)>(in, end)) {
|
||||||
|
}
|
||||||
|
|
||||||
|
void FileRepository::OffsetEntry::serialize(std::vector<char> &out) const {
|
||||||
|
Serialize::serialize(fileId, out);
|
||||||
|
Serialize::serialize(offset, out);
|
||||||
|
Serialize::serialize(length, out);
|
||||||
|
}
|
||||||
|
|
||||||
|
FileRepository::OffsetEntry::OffsetEntry(unsigned long long int fileId, unsigned long long int offset, unsigned long long int length)
|
||||||
|
: fileId(fileId), offset(offset), length(length) {}
|
||||||
|
|
||||||
|
bool FileRepository::clearCache(Object::ObjectType type) {
|
||||||
|
keyIndex[type] = {};
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool FileRepository::addToCache(const Object &obj) {
|
||||||
|
{
|
||||||
|
std::unique_lock lock(repoLock);
|
||||||
|
if (offsetIndex.count(obj.id) == 0)
|
||||||
|
throw Exception("Object with id " + std::to_string(obj.id) + " doesn't exist!");
|
||||||
|
}
|
||||||
|
{
|
||||||
|
std::lock_guard lock(repoLock);
|
||||||
|
keyIndex[obj.type][obj.getKey()] = obj.id;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
125
src/repo/FileRepository.h
Normal file
125
src/repo/FileRepository.h
Normal file
@@ -0,0 +1,125 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 14.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef SEMBACKUP_FILEREPOSITORY_H
|
||||||
|
#define SEMBACKUP_FILEREPOSITORY_H
|
||||||
|
|
||||||
|
#include <filesystem>
|
||||||
|
#include <fstream>
|
||||||
|
#include <map>
|
||||||
|
#include <mutex>
|
||||||
|
|
||||||
|
#include "Object.h"
|
||||||
|
#include "Repository.h"
|
||||||
|
|
||||||
|
/// Repository implementation in the local filesystem
|
||||||
|
/**
|
||||||
|
* `root` Config value is used as a root
|
||||||
|
* Objects are stored concatenated in files with approximate size of `repo-target` MB (from Config)
|
||||||
|
* The object key/object id index is stored as a hash map, as a `index` file out of the object storage structure
|
||||||
|
* Hints for the location of objects inside of files are also stored as a hash map in the `offsets` file
|
||||||
|
* Config is stored in the `info` file, merged with the supplied Config on open()
|
||||||
|
*
|
||||||
|
* Thread safe, approx. max memory usage is `number of threads` * `repo-target`,
|
||||||
|
* as every thread can be flushing its write cache at the same time
|
||||||
|
*/
|
||||||
|
class FileRepository final : public Repository {
|
||||||
|
public:
|
||||||
|
/// Constructs a new FileRepository
|
||||||
|
/// \param config Config to use
|
||||||
|
FileRepository(Config config);
|
||||||
|
|
||||||
|
bool exists() override;
|
||||||
|
bool open() override;
|
||||||
|
bool init() override;
|
||||||
|
bool flush() override;
|
||||||
|
|
||||||
|
std::vector<char> getObject(Object::idType id) const override;
|
||||||
|
bool putObject(const Object &obj) override;
|
||||||
|
bool deleteObject(const Object &obj) override;
|
||||||
|
|
||||||
|
std::vector<char> getObject(Object::ObjectType type, const std::string &key) const override;
|
||||||
|
Object::idType getObjectId(Object::ObjectType type, const std::string &key) const override;
|
||||||
|
std::vector<std::pair<std::string, Object::idType>> getObjects(Object::ObjectType type) const override;
|
||||||
|
|
||||||
|
bool clearCache(Object::ObjectType type) override;
|
||||||
|
bool addToCache(const Object &obj) override;
|
||||||
|
|
||||||
|
bool exists(Object::ObjectType type, const std::string &key) const override;
|
||||||
|
Object::idType getId() override;
|
||||||
|
|
||||||
|
/// FileRepository destructor
|
||||||
|
/// Flushes write cache, and writes the metadata
|
||||||
|
~FileRepository() override;
|
||||||
|
FileRepository(const FileRepository &r) = delete;
|
||||||
|
FileRepository &operator=(const FileRepository &r) = delete;
|
||||||
|
|
||||||
|
private:
|
||||||
|
const std::filesystem::path root;///< Root of the repository in the filesystem
|
||||||
|
|
||||||
|
/// Puts the Object raw data into write cache
|
||||||
|
bool writeObject(const Object &obj);
|
||||||
|
|
||||||
|
bool ready = false;/// < Indicates whether the FileRepository was open or initialized
|
||||||
|
|
||||||
|
/// Reads the file and returns its raw data
|
||||||
|
/// \param file Constant reference to the absolute path of the file
|
||||||
|
/// \return Vector of bytes of the file
|
||||||
|
std::vector<char> readFile(const std::filesystem::path &file) const;
|
||||||
|
|
||||||
|
|
||||||
|
/// Reads the \psize bytes of the file from \p offset and returns its raw data
|
||||||
|
/// \param file Constant reference to the absolute path of the file
|
||||||
|
/// \param offset First byte of the file to read
|
||||||
|
/// \param size Amount of bytes to read (no more than absoluteMaxFileLimit)
|
||||||
|
/// \return Vector of bytes of the file
|
||||||
|
/// \throws Exception on any error, or when absoluteMaxFileLimit is reached
|
||||||
|
std::vector<char> readFile(const std::filesystem::path &file, unsigned long long offset, unsigned long long size) const;
|
||||||
|
static constexpr unsigned long long absoluteMaxFileLimit{4ULL * 1024 * 1024 * 1024};///<Max file read size (4GB)
|
||||||
|
|
||||||
|
/// Writes \p data to \p file
|
||||||
|
/// \param file Constant reference to the absolute path of the file
|
||||||
|
/// \param data Constant reference to the vector of bytes to write
|
||||||
|
/// \return True
|
||||||
|
/// \throws Exception on any error
|
||||||
|
bool writeFile(const std::filesystem::path &file, const std::vector<char> &data);
|
||||||
|
|
||||||
|
mutable std::mutex repoLock;///< Lock for any operations on the Repository
|
||||||
|
|
||||||
|
/// Helper struct to store the location of objects in the filesystem
|
||||||
|
struct OffsetEntry {
|
||||||
|
unsigned long long fileId;///< ID of file where the object is located
|
||||||
|
unsigned long long offset;///< Offset in the file where the object starts
|
||||||
|
unsigned long long length;///< Length of the object
|
||||||
|
using serializable = std::true_type;
|
||||||
|
|
||||||
|
/// Default constructor
|
||||||
|
OffsetEntry(unsigned long long fileId, unsigned long long offset, unsigned long long length);
|
||||||
|
|
||||||
|
/// Deserialization constrictor
|
||||||
|
OffsetEntry(std::vector<char>::const_iterator &in, const std::vector<char>::const_iterator &end);
|
||||||
|
|
||||||
|
/// Serializes the entry to \p out
|
||||||
|
void serialize(std::vector<char> &out) const;
|
||||||
|
};
|
||||||
|
|
||||||
|
unsigned long long maxFileId = 1; ///< Largest ID of object storage file
|
||||||
|
std::unordered_map<Object::idType, OffsetEntry> offsetIndex;///< Used to locate Object%s in the filesystem
|
||||||
|
|
||||||
|
std::mutex writeCacheLock; ///< Write cache lock
|
||||||
|
std::map<Object::idType, std::vector<char>> writeCache;///< Write cache, map of Object ids and their serialized data
|
||||||
|
unsigned long long writeCacheSize = 0; ///< Current byte size of the write cache
|
||||||
|
const unsigned long long writeCacheMax; ///< Target size of the write cache, it is automatically flushed after this is reached
|
||||||
|
|
||||||
|
/// Flushes the write cache
|
||||||
|
/// Takes the cache lock, swaps the cache with an empty one and unlocks it
|
||||||
|
/// \param lockW Write cache lock
|
||||||
|
void flushWriteCache(std::unique_lock<std::mutex> &&lockW);
|
||||||
|
|
||||||
|
Object::idType largestUnusedId = 1; ///< Largest available objectID
|
||||||
|
std::unordered_map<Object::ObjectType, std::unordered_map<std::string, Object::idType>> keyIndex;///< Maps Object%'s keys to their ID's
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
#endif//SEMBACKUP_FILEREPOSITORY_H
|
||||||
21
src/repo/Object.cpp
Normal file
21
src/repo/Object.cpp
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
//
|
||||||
|
// Created by Stepan Usatiuk on 14.04.2023.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "Object.h"
|
||||||
|
|
||||||
|
#include "Serialize.h"
|
||||||
|
|
||||||
|
Object::Object(idType id, ObjectType type) : id(id), type(type) {}
|
||||||
|
|
||||||
|
Object::Object(std::vector<char>::const_iterator &in, const std::vector<char>::const_iterator &end)
|
||||||
|
: id(Serialize::deserialize<idType>(in, end)),
|
||||||
|
type(Serialize::deserialize<ObjectType>(in, end)) {
|
||||||
|
}
|
||||||
|
|
||||||
|
void Object::serialize(std::vector<char> &out) const {
|
||||||
|
Serialize::serialize(id, out);
|
||||||
|
Serialize::serialize(type, out);
|
||||||
|
}
|
||||||
|
|
||||||
|
Object::~Object() = default;
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user