stepus53/dhfs
1
0
Fork 0
mirror of https://github.com/usatiuk/dhfs.git synced 2025-10-29 04:57:48 +01:00
Code Issues Packages Projects Releases Wiki Activity

use mmaped files

Browse Source
This commit is contained in:
Stepan Usatiuk
2024-07-29 23:23:32 +02:00
parent b9852b68d6
commit c88d8cfbb5
2 changed files with 23 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/persistence/FileObjectPersistentStore.java
View File

@@ -1,5 +1,6 @@
package com.usatiuk.dhfs.objects.repository.persistence;
import com.google.protobuf.CodedOutputStream;
import com.google.protobuf.Message;
import com.google.protobuf.UnsafeByteOperations;
import com.usatiuk.dhfs.objects.persistence.JObjectDataP;
@@ -17,6 +18,7 @@ import org.eclipse.microprofile.config.inject.ConfigProperty;
import javax.annotation.Nonnull;
import java.io.*;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
@@ -33,6 +35,8 @@ public class FileObjectPersistentStore implements ObjectPersistentStore {
private final Path metaPath;
private final Path dataPath;
private final static long mmapThreshold = 65536;
public FileObjectPersistentStore(@ConfigProperty(name = "dhfs.objects.persistence.files.root") String root) {
this.root = root;
this.metaPath = Paths.get(root, "meta");
@@ -108,9 +112,15 @@ public class FileObjectPersistentStore implements ObjectPersistentStore {
}
private <T extends Message> T readObjectImpl(T defaultInstance, Path path) {
try (var fsb = new FileInputStream(path.toFile())) {
var file = fsb.readAllBytes();
return (T) defaultInstance.getParserForType().parseFrom(UnsafeByteOperations.unsafeWrap(file));
try (var rf = new RandomAccessFile(path.toFile(), "r")) {
var len = rf.length();
if (len > mmapThreshold)
return (T) defaultInstance.getParserForType().parseFrom(UnsafeByteOperations.unsafeWrap(rf.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, len)));
else {
var arr = new byte[(int) len];
rf.readFully(arr, 0, (int) len);
return (T) defaultInstance.getParserForType().parseFrom(UnsafeByteOperations.unsafeWrap(arr));
}
} catch (FileNotFoundException | NoSuchFileException fx) {
throw new StatusRuntimeExceptionNoStacktrace(Status.NOT_FOUND);
} catch (IOException e) {
@@ -133,6 +143,15 @@ public class FileObjectPersistentStore implements ObjectPersistentStore {
private void writeObjectImpl(Path path, Message data) {
try {
var len = data.getSerializedSize();
if (len > mmapThreshold)
try (var rf = new RandomAccessFile(path.toFile(), "rw")) {
rf.setLength(len);
var mapped = rf.getChannel().map(FileChannel.MapMode.READ_WRITE, 0, len);
data.writeTo(CodedOutputStream.newInstance(mapped));
return;
}
try (var fsb = new FileOutputStream(path.toFile(), false);
var buf = new BufferedOutputStream(fsb, Math.min(65536, data.getSerializedSize()))) {
data.writeTo(buf);

2
dhfs-parent/server/src/main/resources/application.properties
View File

@@ -13,7 +13,7 @@ dhfs.fuse.root=${HOME}/dhfs_default/fuse
dhfs.fuse.debug=false
dhfs.fuse.enabled=true
dhfs.files.allow_recursive_delete=false
dhfs.files.target_chunk_size=524288
dhfs.files.target_chunk_size=2097152
# Writes strictly smaller than this will try to merge with blocks nearby
dhfs.files.write_merge_threshold=0.8
# If a merge would result in a block of greater size than this, stop merging