type iterators?

.github/workflows/server.yml

@@ -49,7 +49,7 @@ jobs:
           cache: maven
 
       - name: Test with Maven
-        run: cd dhfs-parent && mvn -T $(nproc) --batch-mode --update-snapshots package verify
+        run: cd dhfs-parent && mvn --batch-mode --update-snapshots package verify
 
       # - name: Build with Maven
       #   run: cd dhfs-parent && mvn --batch-mode --update-snapshots package # -Dquarkus.log.category.\"com.usatiuk.dhfs\".min-level=DEBUG
@@ -57,7 +57,7 @@ jobs:
       - uses: actions/upload-artifact@v4
         with:
           name: DHFS Server Package
-          path: dhfs-parent/dhfs-app/target/quarkus-app
+          path: dhfs-parent/server/target/quarkus-app
 
       - uses: actions/upload-artifact@v4
         if: ${{ always() }}

dhfs-parent/.run/Main 2.run.xml

@@ -1,8 +1,8 @@
 <component name="ProjectRunConfigurationManager">
   <configuration default="false" name="Main 2" type="QsApplicationConfigurationType" factoryName="QuarkusApplication">
-    <option name="MAIN_CLASS_NAME" value="com.usatiuk.dhfs.app.Main" />
-    <module name="dhfs-app" />
-    <option name="VM_PARAMETERS" value="-XX:+UnlockDiagnosticVMOptions -XX:+DebugNonSafepoints --add-exports java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-exports java.base/jdk.internal.access=ALL-UNNAMED -ea -Dcom.usatiuk.dhfs.supportlib.native-path=$ProjectFileDir$/target/classes/native -Xmx2G -Ddhfs.webui.root=$ProjectFileDir$/../webui/dist -Ddhfs.fuse.root=${HOME}/dhfs_test/2/fuse -Ddhfs.objects.persistence.files.root=${HOME}/dhfs_test/2/data -Ddhfs.objects.persistence.stuff.root=${HOME}/dhfs_test/2/data/stuff -Ddhfs.objects.peerdiscovery.broadcast=false -Dquarkus.http.port=9020 -Dquarkus.http.ssl-port=9021 -Ddhfs.peerdiscovery.preset-uuid=22000000-0000-0000-0000-000000000000 -Ddhfs.peerdiscovery.static-peers=11000000-0000-0000-0000-000000000000:127.0.0.1:9010:9011" />
+    <option name="MAIN_CLASS_NAME" value="com.usatiuk.dhfs.Main" />
+    <module name="server" />
+    <option name="VM_PARAMETERS" value="--add-exports java.base/sun.nio.ch=ALL-UNNAMED --add-exports java.base/jdk.internal.access=ALL-UNNAMED -ea -Dcom.usatiuk.dhfs.supportlib.native-path=$ProjectFileDir$/target/classes/native -Xmx2G -Ddhfs.webui.root=$ProjectFileDir$/../webui/dist -Ddhfs.fuse.root=${HOME}/dhfs_test/2/fuse -Ddhfs.objects.persistence.files.root=${HOME}/dhfs_test/2/data -Ddhfs.objects.persistence.stuff.root=${HOME}/dhfs_test/2/data/stuff -Ddhfs.objects.peerdiscovery.broadcast=false -Dquarkus.http.port=9020 -Dquarkus.http.ssl-port=9021 -Ddhfs.peerdiscovery.preset-uuid=22000000-0000-0000-0000-000000000000 -Ddhfs.peerdiscovery.static-peers=11000000-0000-0000-0000-000000000000:127.0.0.1:9010:9011" />
     <extension name="coverage">
       <pattern>
         <option name="PATTERN" value="com.usatiuk.dhfs.*" />

dhfs-parent/.run/Main.run.xml

@@ -1,8 +1,8 @@
 <component name="ProjectRunConfigurationManager">
   <configuration default="false" name="Main" type="QsApplicationConfigurationType" factoryName="QuarkusApplication" nameIsGenerated="true">
-    <option name="MAIN_CLASS_NAME" value="com.usatiuk.dhfs.app.Main" />
-    <module name="dhfs-app" />
-    <option name="VM_PARAMETERS" value="-XX:+UnlockDiagnosticVMOptions -XX:+DebugNonSafepoints --add-exports java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-exports java.base/jdk.internal.access=ALL-UNNAMED -ea -Dcom.usatiuk.dhfs.supportlib.native-path=$ProjectFileDir$/target/classes/native -Xmx2G -Ddhfs.webui.root=$ProjectFileDir$/../webui/dist -Ddhfs.fuse.root=${HOME}/dhfs_test/1/fuse -Ddhfs.objects.persistence.files.root=${HOME}/dhfs_test/1/data -Ddhfs.objects.persistence.stuff.root=${HOME}/dhfs_test/1/data/stuff -Ddhfs.objects.peerdiscovery.broadcast=true -Dquarkus.http.port=8080 -Dquarkus.http.ssl-port=9011 -Ddhfs.peerdiscovery.preset-uuid=11000000-0000-0000-0000-000000000000 -Ddhfs.peerdiscovery.static-peers=22000000-0000-0000-0000-000000000000:127.0.0.1:9020:9021 -Dquarkus.http.host=0.0.0.0" />
+    <option name="MAIN_CLASS_NAME" value="com.usatiuk.dhfs.Main" />
+    <module name="server" />
+    <option name="VM_PARAMETERS" value="--add-exports java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-exports java.base/jdk.internal.access=ALL-UNNAMED -ea -Dcom.usatiuk.dhfs.supportlib.native-path=$ProjectFileDir$/target/classes/native -Xmx2G -Ddhfs.webui.root=$ProjectFileDir$/../webui/dist -Ddhfs.fuse.root=${HOME}/dhfs_test/1/fuse -Ddhfs.objects.persistence.files.root=${HOME}/dhfs_test/1/data -Ddhfs.objects.persistence.stuff.root=${HOME}/dhfs_test/1/data/stuff -Ddhfs.objects.peerdiscovery.broadcast=false -Dquarkus.http.port=9010 -Dquarkus.http.ssl-port=9011 -Ddhfs.peerdiscovery.preset-uuid=11000000-0000-0000-0000-000000000000 -Ddhfs.peerdiscovery.static-peers=22000000-0000-0000-0000-000000000000:127.0.0.1:9020:9021" />
     <extension name="coverage">
       <pattern>
         <option name="PATTERN" value="com.usatiuk.dhfs.*" />

dhfs-parent/autoprotomap/deployment/pom.xml

@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>com.usatiuk</groupId>
+        <artifactId>autoprotomap-parent</artifactId>
+        <version>1.0-SNAPSHOT</version>
+    </parent>
+    <artifactId>autoprotomap-deployment</artifactId>
+    <name>Autoprotomap - Deployment</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-arc-deployment</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.usatiuk</groupId>
+            <artifactId>autoprotomap</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-junit5-internal</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-grpc-deployment</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-collections4</artifactId>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>default-compile</id>
+                        <configuration>
+                            <annotationProcessorPaths>
+                                <path>
+                                    <groupId>io.quarkus</groupId>
+                                    <artifactId>quarkus-extension-processor</artifactId>
+                                    <version>${quarkus.platform.version}</version>
+                                </path>
+                            </annotationProcessorPaths>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>

dhfs-parent/autoprotomap/deployment/src/main/java/com/usatiuk/autoprotomap/deployment/AutoprotomapProcessor.java

@@ -0,0 +1,78 @@
+package com.usatiuk.autoprotomap.deployment;
+
+import com.usatiuk.autoprotomap.runtime.ProtoMirror;
+import com.usatiuk.autoprotomap.runtime.ProtoSerializer;
+import io.quarkus.arc.deployment.GeneratedBeanBuildItem;
+import io.quarkus.arc.deployment.GeneratedBeanGizmoAdaptor;
+import io.quarkus.deployment.annotations.BuildProducer;
+import io.quarkus.deployment.annotations.BuildStep;
+import io.quarkus.deployment.builditem.ApplicationIndexBuildItem;
+import io.quarkus.gizmo.ClassCreator;
+import io.quarkus.gizmo.SignatureBuilder;
+import jakarta.inject.Singleton;
+import org.jboss.jandex.ClassType;
+import org.jboss.jandex.Type;
+
+class AutoprotomapProcessor {
+    @BuildStep
+    ProtoIndexBuildItem index(ApplicationIndexBuildItem jandex) {
+        var ret = new ProtoIndexBuildItem();
+        var annot = jandex.getIndex().getAnnotations(ProtoMirror.class);
+        for (var a : annot) {
+            var protoTarget = jandex.getIndex().getClassByName(((ClassType) a.value().value()).name());
+//            if (!messageImplementors.contains(protoTarget))
+//                throw new IllegalArgumentException("Expected " + protoTarget + " to be a proto message");
+            System.out.println("Found: " + a.name().toString() + " at " + protoTarget.name().toString() + " of " + a.target().asClass().name().toString());
+            ret.protoMsgToObj.put(protoTarget, a.target().asClass());
+        }
+        return ret;
+    }
+
+    @BuildStep
+    void generateProtoSerializer(ApplicationIndexBuildItem jandex,
+                                 ProtoIndexBuildItem protoIndex,
+                                 BuildProducer<GeneratedBeanBuildItem> generatedClasses) {
+        try {
+            for (var o : protoIndex.protoMsgToObj.entrySet()) {
+                System.out.println("Generating " + o.getKey().toString() + " -> " + o.getValue().toString());
+                var gizmoAdapter = new GeneratedBeanGizmoAdaptor(generatedClasses);
+
+                var msgType = io.quarkus.gizmo.Type.classType(o.getKey().name());
+                var objType = io.quarkus.gizmo.Type.classType(o.getValue().name());
+
+                var type = io.quarkus.gizmo.Type.ParameterizedType.parameterizedType(
+                        io.quarkus.gizmo.Type.classType(ProtoSerializer.class),
+                        msgType, objType);
+
+                var msgJType = Type.create(o.getKey().name(), Type.Kind.CLASS);
+                var objJType = Type.create(o.getValue().name(), Type.Kind.CLASS);
+
+                try (ClassCreator classCreator = ClassCreator.builder()
+                        .className("com.usatiuk.autoprotomap.generated.for" + o.getKey().simpleName())
+                        .signature(SignatureBuilder.forClass().addInterface(type))
+                        .classOutput(gizmoAdapter)
+                        .setFinal(true)
+                        .build()) {
+                    classCreator.addAnnotation(Singleton.class);
+
+                    var generator = new ProtoSerializerGenerator(
+                            jandex.getIndex(),
+                            protoIndex,
+                            classCreator,
+                            msgJType,
+                            objJType
+                    );
+
+                    generator.generate();
+                }
+            }
+        } catch (Throwable e) {
+            StringBuilder sb = new StringBuilder();
+            sb.append(e + "\n");
+            for (var el : e.getStackTrace()) {
+                sb.append(el.toString() + "\n");
+            }
+            System.out.println(sb);
+        }
+    }
+}

dhfs-parent/autoprotomap/deployment/src/main/java/com/usatiuk/autoprotomap/deployment/Constants.java

@@ -0,0 +1,18 @@
+package com.usatiuk.autoprotomap.deployment;
+
+public class Constants {
+    public static final String FIELD_PREFIX = "_";
+
+    public static String capitalize(String str) {
+        return str.substring(0, 1).toUpperCase() + str.substring(1);
+    }
+
+    public static String stripPrefix(String str, String prefix) {
+        if (str.startsWith(prefix)) {
+            return str.substring(prefix.length());
+        }
+        return str;
+    }
+
+
+}

dhfs-parent/autoprotomap/deployment/src/main/java/com/usatiuk/autoprotomap/deployment/Effect.java

@@ -0,0 +1,6 @@
+package com.usatiuk.autoprotomap.deployment;
+
+@FunctionalInterface
+public interface Effect {
+    void apply();
+}

dhfs-parent/autoprotomap/deployment/src/main/java/com/usatiuk/autoprotomap/deployment/ProtoIndexBuildItem.java

@@ -0,0 +1,10 @@
+package com.usatiuk.autoprotomap.deployment;
+
+import io.quarkus.builder.item.SimpleBuildItem;
+import org.apache.commons.collections4.BidiMap;
+import org.apache.commons.collections4.bidimap.DualHashBidiMap;
+import org.jboss.jandex.ClassInfo;
+
+public final class ProtoIndexBuildItem extends SimpleBuildItem {
+    BidiMap<ClassInfo, ClassInfo> protoMsgToObj = new DualHashBidiMap<>();
+}

dhfs-parent/autoprotomap/deployment/src/main/java/com/usatiuk/autoprotomap/deployment/ProtoSerializerGenerator.java

@@ -0,0 +1,342 @@
+package com.usatiuk.autoprotomap.deployment;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Message;
+import com.usatiuk.autoprotomap.runtime.ProtoSerializer;
+import io.quarkus.gizmo.*;
+import jakarta.inject.Inject;
+import org.apache.commons.lang3.tuple.Pair;
+import org.jboss.jandex.Type;
+import org.jboss.jandex.*;
+import org.objectweb.asm.Opcodes;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Objects;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.function.IntConsumer;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static com.usatiuk.autoprotomap.deployment.Constants.*;
+
+public class ProtoSerializerGenerator {
+    private final Index index;
+    private final ProtoIndexBuildItem protoIndex;
+    private final ClassCreator classCreator;
+    private final HashSet<Pair<ClassInfo, ClassInfo>> externalSerializers = new HashSet<>();
+    private final Type topMessageType;
+    private final Type topObjectType;
+
+    public ProtoSerializerGenerator(Index index, ProtoIndexBuildItem protoIndex, ClassCreator classCreator, Type topMessageType, Type topObjectType) {
+        this.index = index;
+        this.protoIndex = protoIndex;
+        this.classCreator = classCreator;
+        this.topMessageType = topMessageType;
+        this.topObjectType = topObjectType;
+    }
+
+    private FieldDescriptor getOutsideSerializer(ClassInfo messageClass, ClassInfo objectClass) {
+        var name = messageClass.name().withoutPackagePrefix() + objectClass.name().withoutPackagePrefix() + "serializer";
+        var msgType = io.quarkus.gizmo.Type.classType(messageClass.name());
+        var objType = io.quarkus.gizmo.Type.classType(objectClass.name());
+        var type = io.quarkus.gizmo.Type.ParameterizedType.parameterizedType(
+                io.quarkus.gizmo.Type.classType(ProtoSerializer.class),
+                msgType, objType);
+        var sig = SignatureBuilder.forField().setType(type).build();
+        var fd = FieldDescriptor.of(classCreator.getClassName(), name, ProtoSerializer.class);
+        if (externalSerializers.add(Pair.of(messageClass, objectClass))) {
+            var fc = classCreator.getFieldCreator(fd);
+            fc.addAnnotation(Inject.class);
+            fc.setSignature(sig);
+            fc.setModifiers(Opcodes.ACC_PUBLIC);
+        }
+        return fd;
+    }
+
+    private void traverseHierarchy(Index index, ClassInfo klass, Consumer<ClassInfo> visitor) {
+        var cur = klass;
+        while (true) {
+            visitor.accept(cur);
+
+            var next = cur.superClassType().name();
+            if (next.equals(DotName.OBJECT_NAME) || next.equals(DotName.RECORD_NAME)) break;
+            cur = index.getClassByName(next);
+        }
+    }
+
+    private ArrayList<FieldInfo> findAllFields(Index index, ClassInfo klass) {
+        ArrayList<FieldInfo> ret = new ArrayList<>();
+        traverseHierarchy(index, klass, cur -> {
+            ret.addAll(cur.fields());
+        });
+        return ret;
+    }
+
+    private void generateBuilderUse(BytecodeCreator bytecodeCreator,
+                                    ResultHandle builder,
+                                    Type messageType, Type objectType,
+                                    ResultHandle object) {
+        var builderType = Type.create(DotName.createComponentized(messageType.name(), "Builder", true), Type.Kind.CLASS);
+
+        var objectClass = index.getClassByName(objectType.name().toString());
+
+        Function<String, String> getterGetter = objectClass.isRecord()
+                ? Function.identity()
+                : s -> "get" + capitalize(stripPrefix(s, FIELD_PREFIX));
+
+        for (var f : findAllFields(index, objectClass)) {
+            var consideredFieldName = stripPrefix(f.name(), FIELD_PREFIX);
+
+            Supplier<ResultHandle> get = () -> {
+                if ((f.flags() & Opcodes.ACC_PUBLIC) != 0)
+                    return bytecodeCreator.readInstanceField(f, object);
+                else {
+                    var fieldGetter = getterGetter.apply(f.name());
+                    return bytecodeCreator.invokeVirtualMethod(
+                            MethodDescriptor.ofMethod(objectType.toString(), fieldGetter, f.type().name().toString()), object);
+                }
+            };
+
+            Effect doSimpleCopy = () -> {
+                var setter = MethodDescriptor.ofMethod(builderType.name().toString(), "set" + capitalize(consideredFieldName),
+                        builderType.name().toString(), f.type().toString());
+
+                var val = get.get();
+                bytecodeCreator.invokeVirtualMethod(setter, builder, val);
+            };
+
+            switch (f.type().kind()) {
+                case CLASS -> {
+                    if (f.type().equals(Type.create(String.class)) || f.type().equals(Type.create(ByteString.class))) {
+                        doSimpleCopy.apply();
+                    } else {
+                        var builderGetter = "get" + capitalize(f.name()) + "Builder";
+                        var protoType = protoIndex.protoMsgToObj.inverseBidiMap().get(index.getClassByName(f.type().name()));
+                        var nestedBuilderType = Type.create(DotName.createComponentized(protoType.name(), "Builder", true), Type.Kind.CLASS);
+                        var nestedBuilder = bytecodeCreator.invokeVirtualMethod(
+                                MethodDescriptor.ofMethod(builderType.toString(), builderGetter, nestedBuilderType.name().toString()), builder);
+
+                        var val = get.get();
+
+                        generateBuilderUse(bytecodeCreator, nestedBuilder, Type.create(protoType.name(), Type.Kind.CLASS), f.type(), val);
+                    }
+                }
+                case PRIMITIVE -> {
+                    doSimpleCopy.apply();
+                }
+                case WILDCARD_TYPE -> throw new UnsupportedOperationException("Wildcards not supported yet");
+                case PARAMETERIZED_TYPE ->
+                        throw new UnsupportedOperationException("Parametrized types not supported yet");
+                case ARRAY -> throw new UnsupportedOperationException("Arrays not supported yet");
+                default -> throw new IllegalStateException("Unexpected type: " + f.type());
+            }
+        }
+    }
+
+    private ResultHandle generateConstructorUse(
+            BytecodeCreator bytecodeCreator,
+            ClassCreator classCreator,
+            Type messageType, Type objectType,
+            ResultHandle message
+    ) {
+        var constructor = findAllArgsConstructor(index, index.getClassByName(objectType.name()));
+        if (constructor == null) {
+            throw new IllegalStateException("No constructor found for type: " + objectType.name());
+        }
+        var argMap = new ResultHandle[constructor.parametersCount()];
+
+        for (int i = 0; i < argMap.length; i++) {
+            var type = constructor.parameterType(i);
+            var strippedName = stripPrefix(constructor.parameterName(i), FIELD_PREFIX);
+
+            IntConsumer doSimpleCopy = (arg) -> {
+                var call = MethodDescriptor.ofMethod(messageType.name().toString(), "get" + capitalize(strippedName),
+                        type.name().toString());
+                argMap[arg] = bytecodeCreator.invokeVirtualMethod(call, message);
+            };
+
+            switch (type.kind()) {
+                case CLASS -> {
+                    if (type.equals(Type.create(String.class)) || type.equals(Type.create(ByteString.class))) {
+                        doSimpleCopy.accept(i);
+                    } else {
+                        var nestedProtoType = protoIndex.protoMsgToObj.inverseBidiMap().get(index.getClassByName(type.name()));
+                        var call = MethodDescriptor.ofMethod(messageType.name().toString(), "get" + capitalize(strippedName),
+                                nestedProtoType.name().toString());
+                        var nested = bytecodeCreator.invokeVirtualMethod(call, message);
+                        argMap[i] = generateConstructorUse(bytecodeCreator, classCreator, Type.create(nestedProtoType.name(), Type.Kind.CLASS), type, nested);
+                    }
+                }
+                case PRIMITIVE -> {
+                    doSimpleCopy.accept(i);
+                }
+                case WILDCARD_TYPE -> throw new UnsupportedOperationException("Wildcards not supported yet");
+                case PARAMETERIZED_TYPE ->
+                        throw new UnsupportedOperationException("Parametrized types not supported yet");
+                case ARRAY -> throw new UnsupportedOperationException("Arrays not supported yet");
+                default -> throw new IllegalStateException("Unexpected type: " + type);
+            }
+        }
+
+        return bytecodeCreator.newInstance(constructor, argMap);
+    }
+
+    private MethodInfo findAllArgsConstructor(Index index, ClassInfo klass) {
+        ArrayList<FieldInfo> fields = findAllFields(index, klass);
+
+        var fieldCount = fields.size();
+        var fieldNames = fields.stream().map(f -> stripPrefix(f.name(), FIELD_PREFIX)).sorted().toList();
+        var fieldNameToType = fields.stream().collect(Collectors.toMap(f -> stripPrefix(f.name(), FIELD_PREFIX), FieldInfo::type));
+
+        for (var m : klass.constructors()) {
+            if (m.parametersCount() != fieldCount) continue;
+            var parameterNames = m.parameters().stream().map(n -> stripPrefix(n.name(), FIELD_PREFIX)).sorted().toList();
+            if (!Objects.equals(fieldNames, parameterNames)) continue;
+
+            for (var p : m.parameters()) {
+                if (!Objects.equals(fieldNameToType.get(stripPrefix(p.name(), FIELD_PREFIX)), p.type())) continue;
+            }
+
+            return m;
+        }
+
+        return null;
+    }
+
+    public void generateAbstract() {
+        var kids = Stream.concat(index.getAllKnownSubclasses(topObjectType.name()).stream(),
+                        index.getAllKnownImplementors(topObjectType.name()).stream())
+                .filter(k -> !k.isAbstract() && !k.isInterface()).toList();
+
+        try (MethodCreator method = classCreator.getMethodCreator("serialize",
+                Message.class, Object.class)) {
+
+            method.setModifiers(Opcodes.ACC_PUBLIC);
+
+            var builderType = Type.create(DotName.createComponentized(topMessageType.name(), "Builder", true), Type.Kind.CLASS);
+
+            var builder = method.invokeStaticMethod(MethodDescriptor.ofMethod(topMessageType.name().toString(), "newBuilder", builderType.name().toString()));
+
+            var arg = method.getMethodParam(0);
+
+            for (var nestedObjClass : kids) {
+                System.out.println("Generating " + nestedObjClass.name() + " serializer for " + topObjectType.name());
+                var nestedObjType = Type.create(nestedObjClass.name(), Type.Kind.CLASS);
+                var nestedMessageClass = protoIndex.protoMsgToObj.inverseBidiMap().get(nestedObjClass);
+                boolean doExternalCall = false;
+                if (nestedMessageClass == null) {
+                    var msgInfo = index.getClassByName(topMessageType.name());
+                    nestedMessageClass = index.getClassByName(msgInfo.method("get" + capitalize(nestedObjType.name().withoutPackagePrefix())).returnType().name());
+                    doExternalCall = true;
+                }
+                var nestedMessageType = Type.create(nestedMessageClass.name(), Type.Kind.CLASS);
+
+                var statement = method.ifTrue(method.instanceOf(arg, nestedObjClass.name().toString()));
+
+                try (var branch = statement.trueBranch()) {
+                    if (doExternalCall) {
+                        var externalSerializer = getOutsideSerializer(nestedMessageClass, nestedObjClass);
+                        var serializerLoaded = branch.readInstanceField(externalSerializer, branch.getThis());
+                        var serialized = branch.invokeInterfaceMethod(
+                                MethodDescriptor.ofMethod(ProtoSerializer.class,
+                                        "serialize", Message.class, Object.class),
+                                serializerLoaded, arg);
+                        branch.invokeVirtualMethod(MethodDescriptor.ofMethod(builderType.name().toString(),
+                                "set" + capitalize(nestedObjType.name().withoutPackagePrefix()),
+                                builderType.name().toString(), nestedMessageType.name().toString()), builder, serialized);
+                    } else {
+                        var nestedBuilderType = Type.create(DotName.createComponentized(nestedMessageType.name(), "Builder", true), Type.Kind.CLASS);
+                        var nestedBuilder = branch.invokeVirtualMethod(MethodDescriptor.ofMethod(builderType.name().toString(),
+                                "get" + capitalize(nestedObjType.name().withoutPackagePrefix()) + "Builder",
+                                nestedBuilderType.name().toString()), builder);
+                        generateBuilderUse(branch, nestedBuilder, nestedMessageType, nestedObjType, arg);
+                    }
+                    var result = branch.invokeVirtualMethod(MethodDescriptor.ofMethod(builderType.name().toString(), "build", topMessageType.name().toString()), builder);
+                    branch.returnValue(result);
+                }
+            }
+            method.throwException(IllegalArgumentException.class, "Unknown object type");
+        }
+
+        try (MethodCreator method = classCreator.getMethodCreator("deserialize",
+                Object.class, Message.class)) {
+            method.setModifiers(Opcodes.ACC_PUBLIC);
+            var arg = method.getMethodParam(0);
+
+            for (var nestedObjClass : kids) {
+                System.out.println("Generating " + nestedObjClass.name() + " deserializer for " + topObjectType.name());
+                var nestedObjType = Type.create(nestedObjClass.name(), Type.Kind.CLASS);
+
+                var nestedMessageClass = protoIndex.protoMsgToObj.inverseBidiMap().get(nestedObjClass);
+                boolean doExternalCall = false;
+                if (nestedMessageClass == null) {
+                    var msgInfo = index.getClassByName(topMessageType.name());
+                    nestedMessageClass = index.getClassByName(msgInfo.method("get" + capitalize(nestedObjType.name().withoutPackagePrefix())).returnType().name());
+                    doExternalCall = true;
+                }
+
+                var nestedMessageType = Type.create(nestedMessageClass.name(), Type.Kind.CLASS);
+
+                var typeCheck = method.invokeVirtualMethod(MethodDescriptor.ofMethod(topMessageType.name().toString(),
+                        "has" + capitalize(nestedObjType.name().withoutPackagePrefix()), boolean.class), arg);
+
+                var statement = method.ifTrue(typeCheck);
+
+                try (var branch = statement.trueBranch()) {
+                    var nestedMessage = branch.invokeVirtualMethod(MethodDescriptor.ofMethod(topMessageType.name().toString(),
+                            "get" + capitalize(nestedObjType.name().withoutPackagePrefix()), nestedMessageType.name().toString()), arg);
+                    if (doExternalCall) {
+                        var externalSerializer = getOutsideSerializer(nestedMessageClass, nestedObjClass);
+                        var serializerLoaded = branch.readInstanceField(externalSerializer, branch.getThis());
+                        branch.returnValue(branch.invokeInterfaceMethod(
+                                MethodDescriptor.ofMethod(ProtoSerializer.class,
+                                        "deserialize", Object.class, Message.class),
+                                serializerLoaded, nestedMessage));
+                    } else {
+                        branch.returnValue(generateConstructorUse(branch, classCreator, nestedMessageType, nestedObjType, nestedMessage));
+                    }
+                }
+            }
+            method.throwException(IllegalArgumentException.class, "Unknown object type");
+        }
+    }
+
+    public void generate() {
+        var objInfo = index.getClassByName(topObjectType.name());
+        if (objInfo.isAbstract() || objInfo.isInterface()) {
+            generateAbstract();
+            return;
+        }
+
+        try (MethodCreator method = classCreator.getMethodCreator("serialize",
+                Message.class, Object.class)) {
+
+            method.setModifiers(Opcodes.ACC_PUBLIC);
+
+            var builderType = Type.create(DotName.createComponentized(topMessageType.name(), "Builder", true), Type.Kind.CLASS);
+
+            var builder = method.invokeStaticMethod(MethodDescriptor.ofMethod(topMessageType.name().toString(), "newBuilder", builderType.name().toString()));
+
+            var arg = method.getMethodParam(0);
+
+            generateBuilderUse(method, builder, topMessageType, topObjectType, arg);
+
+            var result = method.invokeVirtualMethod(MethodDescriptor.ofMethod(builderType.name().toString(), "build", topMessageType.name().toString()), builder);
+
+            method.returnValue(result);
+        }
+
+        try (MethodCreator method = classCreator.getMethodCreator("deserialize",
+                Object.class, Message.class)) {
+            method.setModifiers(Opcodes.ACC_PUBLIC);
+
+            var arg = method.getMethodParam(0);
+
+            method.returnValue(generateConstructorUse(method, classCreator, topMessageType, topObjectType, arg));
+        }
+
+    }
+}

dhfs-parent/autoprotomap/deployment/src/test/java/com/usatiuk/autoprotomap/test/AutoprotomapDevModeTest.java

@@ -0,0 +1,22 @@
+package com.usatiuk.autoprotomap.test;
+
+import io.quarkus.test.QuarkusDevModeTest;
+import org.jboss.shrinkwrap.api.ShrinkWrap;
+import org.jboss.shrinkwrap.api.spec.JavaArchive;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+public class AutoprotomapDevModeTest {
+
+    // Start hot reload (DevMode) test with your extension loaded
+    @RegisterExtension
+    static final QuarkusDevModeTest devModeTest = new QuarkusDevModeTest()
+            .setArchiveProducer(() -> ShrinkWrap.create(JavaArchive.class));
+
+    @Test
+    public void writeYourOwnDevModeTest() {
+        // Write your dev mode tests here - see the testing extension guide https://quarkus.io/guides/writing-extensions#testing-hot-reload for more information
+        Assertions.assertTrue(true, "Add dev mode assertions to " + getClass().getName());
+    }
+}

dhfs-parent/autoprotomap/deployment/src/test/java/com/usatiuk/autoprotomap/test/AutoprotomapTest.java

@@ -0,0 +1,22 @@
+package com.usatiuk.autoprotomap.test;
+
+import io.quarkus.test.QuarkusUnitTest;
+import org.jboss.shrinkwrap.api.ShrinkWrap;
+import org.jboss.shrinkwrap.api.spec.JavaArchive;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+public class AutoprotomapTest {
+
+    // Start unit test with your extension loaded
+    @RegisterExtension
+    static final QuarkusUnitTest unitTest = new QuarkusUnitTest()
+            .setArchiveProducer(() -> ShrinkWrap.create(JavaArchive.class));
+
+    @Test
+    public void writeYourOwnUnitTest() {
+        // Write your unit tests here - see the testing extension guide https://quarkus.io/guides/writing-extensions#testing-extensions for more information
+        Assertions.assertTrue(true, "Add some assertions to " + getClass().getName());
+    }
+}

dhfs-parent/autoprotomap/integration-tests/pom.xml

@@ -0,0 +1,107 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>com.usatiuk</groupId>
+        <artifactId>autoprotomap-parent</artifactId>
+        <version>1.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>autoprotomap-integration-tests</artifactId>
+    <name>Autoprotomap - Integration Tests</name>
+
+    <properties>
+        <skipITs>true</skipITs>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.usatiuk</groupId>
+            <artifactId>autoprotomap</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.usatiuk</groupId>
+            <artifactId>autoprotomap-deployment</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-junit5</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-grpc</artifactId>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>io.quarkus</groupId>
+                <artifactId>quarkus-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>build</goal>
+                            <goal>generate-code</goal>
+                            <goal>generate-code-tests</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <artifactId>maven-failsafe-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>integration-test</goal>
+                            <goal>verify</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <systemPropertyVariables>
+                        <native.image.path>${project.build.directory}/${project.build.finalName}-runner
+                        </native.image.path>
+                        <java.util.logging.manager>org.jboss.logmanager.LogManager</java.util.logging.manager>
+                        <maven.home>${maven.home}</maven.home>
+                    </systemPropertyVariables>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+    <profiles>
+        <profile>
+            <id>native-image</id>
+            <activation>
+                <property>
+                    <name>native</name>
+                </property>
+            </activation>
+            <build>
+                <plugins>
+                    <plugin>
+                        <artifactId>maven-surefire-plugin</artifactId>
+                        <configuration>
+                            <skipTests>${native.surefire.skip}</skipTests>
+                        </configuration>
+                    </plugin>
+                </plugins>
+            </build>
+            <properties>
+                <skipITs>false</skipITs>
+                <quarkus.native.enabled>true</quarkus.native.enabled>
+            </properties>
+        </profile>
+    </profiles>
+</project>

dhfs-parent/autoprotomap/integration-tests/src/main/java/com/usatiuk/autoprotomap/it/AbstractObject.java

@@ -0,0 +1,7 @@
+package com.usatiuk.autoprotomap.it;
+
+import com.usatiuk.autoprotomap.runtime.ProtoMirror;
+
+@ProtoMirror(AbstractProto.class)
+public abstract class AbstractObject {
+}

dhfs-parent/autoprotomap/integration-tests/src/main/java/com/usatiuk/autoprotomap/it/CustomObject.java

@@ -0,0 +1,10 @@
+package com.usatiuk.autoprotomap.it;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+
+@AllArgsConstructor
+@Getter
+public class CustomObject extends AbstractObject {
+    public int testNum = 0;
+}

dhfs-parent/autoprotomap/integration-tests/src/main/java/com/usatiuk/autoprotomap/it/CustomObjectSerializer.java

@@ -0,0 +1,17 @@
+package com.usatiuk.autoprotomap.it;
+
+import com.usatiuk.autoprotomap.runtime.ProtoSerializer;
+import jakarta.inject.Singleton;
+
+@Singleton
+public class CustomObjectSerializer implements ProtoSerializer<CustomObjectProto, CustomObject> {
+    @Override
+    public CustomObject deserialize(CustomObjectProto message) {
+        return new CustomObject(2);
+    }
+
+    @Override
+    public CustomObjectProto serialize(CustomObject object) {
+        return CustomObjectProto.newBuilder().setTest(1).build();
+    }
+}

dhfs-parent/autoprotomap/integration-tests/src/main/java/com/usatiuk/autoprotomap/it/InterfaceObject.java

@@ -0,0 +1,8 @@
+package com.usatiuk.autoprotomap.it;
+
+import com.usatiuk.autoprotomap.runtime.ProtoMirror;
+
+@ProtoMirror(InterfaceObjectProto.class)
+public interface InterfaceObject {
+    String key();
+}

dhfs-parent/autoprotomap/integration-tests/src/main/java/com/usatiuk/autoprotomap/it/NestedObject.java

@@ -0,0 +1,15 @@
+package com.usatiuk.autoprotomap.it;
+
+import com.google.protobuf.ByteString;
+import com.usatiuk.autoprotomap.runtime.ProtoMirror;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+
+@ProtoMirror(NestedObjectProto.class)
+@AllArgsConstructor
+@Getter
+public class NestedObject extends AbstractObject {
+    public SimpleObject object;
+    public String _nestedName;
+    public ByteString _nestedSomeBytes;
+}

dhfs-parent/autoprotomap/integration-tests/src/main/java/com/usatiuk/autoprotomap/it/RecordObject.java

@@ -0,0 +1,7 @@
+package com.usatiuk.autoprotomap.it;
+
+import com.usatiuk.autoprotomap.runtime.ProtoMirror;
+
+@ProtoMirror(RecordObjectProto.class)
+public record RecordObject(String key) implements InterfaceObject {
+}

dhfs-parent/autoprotomap/integration-tests/src/main/java/com/usatiuk/autoprotomap/it/RecordObject2.java

@@ -0,0 +1,7 @@
+package com.usatiuk.autoprotomap.it;
+
+import com.usatiuk.autoprotomap.runtime.ProtoMirror;
+
+@ProtoMirror(RecordObject2Proto.class)
+public record RecordObject2(String key, int value) implements InterfaceObject {
+}

dhfs-parent/autoprotomap/integration-tests/src/main/java/com/usatiuk/autoprotomap/it/SimpleObject.java

@@ -0,0 +1,15 @@
+package com.usatiuk.autoprotomap.it;
+
+import com.google.protobuf.ByteString;
+import com.usatiuk.autoprotomap.runtime.ProtoMirror;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+
+@ProtoMirror(SimpleObjectProto.class)
+@AllArgsConstructor
+@Getter
+public class SimpleObject extends AbstractObject {
+    public int numfield = 0;
+    private String name;
+    public ByteString someBytes;
+}

dhfs-parent/autoprotomap/integration-tests/src/main/proto/autoprotomap_test.proto

@@ -0,0 +1,47 @@
+syntax = "proto3";
+
+option java_multiple_files = true;
+option java_package = "com.usatiuk.autoprotomap.it";
+option java_outer_classname = "TestProto";
+
+package autoprotomap.test;
+
+message SimpleObjectProto {
+  int32 numfield = 1;
+  string name = 2;
+  bytes someBytes = 3;
+}
+
+message NestedObjectProto {
+  SimpleObjectProto object = 1;
+  string nestedName = 2;
+  bytes nestedSomeBytes = 3;
+}
+
+message CustomObjectProto {
+  int64 test = 1;
+}
+
+message AbstractProto {
+  oneof obj {
+    NestedObjectProto nestedObject = 1;
+    SimpleObjectProto simpleObject = 2;
+    CustomObjectProto customObject = 3;
+  }
+}
+
+message RecordObjectProto {
+  string key = 1;
+}
+
+message RecordObject2Proto {
+  string key = 1;
+  int32 value = 2;
+}
+
+message InterfaceObjectProto {
+  oneof obj {
+    RecordObjectProto recordObject = 1;
+    RecordObject2Proto recordObject2 = 2;
+  }
+}

dhfs-parent/autoprotomap/integration-tests/src/main/resources/application.properties

@@ -0,0 +1 @@
+quarkus.package.jar.decompiler.enabled=true

dhfs-parent/autoprotomap/integration-tests/src/test/java/com/usatiuk/autoprotomap/it/AutoprotomapResourceIT.java

@@ -0,0 +1,7 @@
+package com.usatiuk.autoprotomap.it;
+
+import io.quarkus.test.junit.QuarkusIntegrationTest;
+
+@QuarkusIntegrationTest
+public class AutoprotomapResourceIT extends AutoprotomapResourceTest {
+}

dhfs-parent/autoprotomap/integration-tests/src/test/java/com/usatiuk/autoprotomap/it/AutoprotomapResourceTest.java

@@ -0,0 +1,113 @@
+package com.usatiuk.autoprotomap.it;
+
+import com.google.protobuf.ByteString;
+import com.usatiuk.autoprotomap.runtime.ProtoSerializer;
+import io.quarkus.test.junit.QuarkusTest;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+
+@QuarkusTest
+public class AutoprotomapResourceTest {
+    @Inject
+    ProtoSerializer<SimpleObjectProto, SimpleObject> simpleProtoSerializer;
+    @Inject
+    ProtoSerializer<NestedObjectProto, NestedObject> nestedProtoSerializer;
+    @Inject
+    ProtoSerializer<AbstractProto, AbstractObject> abstractProtoSerializer;
+    @Inject
+    ProtoSerializer<InterfaceObjectProto, InterfaceObject> interfaceProtoSerializer;
+
+    @Test
+    public void testSimple() {
+        var ret = simpleProtoSerializer.serialize(new SimpleObject(1234, "simple test", ByteString.copyFrom(new byte[]{1, 2, 3})));
+        Assertions.assertEquals(1234, ret.getNumfield());
+        Assertions.assertEquals("simple test", ret.getName());
+        Assertions.assertEquals(ByteString.copyFrom(new byte[]{1, 2, 3}), ret.getSomeBytes());
+
+        var des = simpleProtoSerializer.deserialize(ret);
+        Assertions.assertEquals(1234, des.getNumfield());
+        Assertions.assertEquals("simple test", des.getName());
+        Assertions.assertEquals(ByteString.copyFrom(new byte[]{1, 2, 3}), des.getSomeBytes());
+    }
+
+    @Test
+    public void testNested() {
+        var ret = nestedProtoSerializer.serialize(
+                new NestedObject(
+                        new SimpleObject(333, "nested so", ByteString.copyFrom(new byte[]{1, 2, 3})),
+                        "nested obj", ByteString.copyFrom(new byte[]{4, 5, 6})));
+        Assertions.assertEquals(333, ret.getObject().getNumfield());
+        Assertions.assertEquals("nested so", ret.getObject().getName());
+        Assertions.assertEquals(ByteString.copyFrom(new byte[]{1, 2, 3}), ret.getObject().getSomeBytes());
+        Assertions.assertEquals("nested obj", ret.getNestedName());
+        Assertions.assertEquals(ByteString.copyFrom(new byte[]{4, 5, 6}), ret.getNestedSomeBytes());
+
+        var des = nestedProtoSerializer.deserialize(ret);
+        Assertions.assertEquals(333, des.object.numfield);
+        Assertions.assertEquals(333, des.getObject().getNumfield());
+        Assertions.assertEquals("nested so", des.getObject().getName());
+        Assertions.assertEquals(ByteString.copyFrom(new byte[]{1, 2, 3}), des.getObject().getSomeBytes());
+        Assertions.assertEquals("nested obj", des.get_nestedName());
+        Assertions.assertEquals(ByteString.copyFrom(new byte[]{4, 5, 6}), des.get_nestedSomeBytes());
+    }
+
+    @Test
+    public void testAbstractSimple() {
+        var ret = abstractProtoSerializer.serialize(new SimpleObject(1234, "simple test", ByteString.copyFrom(new byte[]{1, 2, 3})));
+        Assertions.assertEquals(1234, ret.getSimpleObject().getNumfield());
+        Assertions.assertEquals("simple test", ret.getSimpleObject().getName());
+        Assertions.assertEquals(ByteString.copyFrom(new byte[]{1, 2, 3}), ret.getSimpleObject().getSomeBytes());
+
+        var des = (SimpleObject) abstractProtoSerializer.deserialize(ret);
+        Assertions.assertEquals(1234, des.getNumfield());
+        Assertions.assertEquals("simple test", des.getName());
+        Assertions.assertEquals(ByteString.copyFrom(new byte[]{1, 2, 3}), des.getSomeBytes());
+    }
+
+    @Test
+    public void testAbstractCustom() {
+        var ret = abstractProtoSerializer.serialize(new CustomObject(1234));
+        Assertions.assertEquals(1, ret.getCustomObject().getTest());
+
+        var des = (CustomObject) abstractProtoSerializer.deserialize(ret);
+        Assertions.assertEquals(2, des.getTestNum());
+    }
+
+    @Test
+    public void testAbstractNested() {
+        var ret = abstractProtoSerializer.serialize(
+                new NestedObject(
+                        new SimpleObject(333, "nested so", ByteString.copyFrom(new byte[]{1, 2, 3})),
+                        "nested obj", ByteString.copyFrom(new byte[]{4, 5, 6})));
+        Assertions.assertEquals(333, ret.getNestedObject().getObject().getNumfield());
+        Assertions.assertEquals("nested so", ret.getNestedObject().getObject().getName());
+        Assertions.assertEquals(ByteString.copyFrom(new byte[]{1, 2, 3}), ret.getNestedObject().getObject().getSomeBytes());
+        Assertions.assertEquals("nested obj", ret.getNestedObject().getNestedName());
+        Assertions.assertEquals(ByteString.copyFrom(new byte[]{4, 5, 6}), ret.getNestedObject().getNestedSomeBytes());
+
+        var des = (NestedObject) abstractProtoSerializer.deserialize(ret);
+        Assertions.assertEquals(333, des.object.numfield);
+        Assertions.assertEquals(333, des.getObject().getNumfield());
+        Assertions.assertEquals("nested so", des.getObject().getName());
+        Assertions.assertEquals(ByteString.copyFrom(new byte[]{1, 2, 3}), des.getObject().getSomeBytes());
+        Assertions.assertEquals("nested obj", des.get_nestedName());
+        Assertions.assertEquals(ByteString.copyFrom(new byte[]{4, 5, 6}), des.get_nestedSomeBytes());
+    }
+
+    @Test
+    public void testInterface() {
+        var ret = interfaceProtoSerializer.serialize(new RecordObject("record test"));
+        Assertions.assertEquals("record test", ret.getRecordObject().getKey());
+        var des = (RecordObject) interfaceProtoSerializer.deserialize(ret);
+        Assertions.assertEquals("record test", des.key());
+
+        var ret2 = interfaceProtoSerializer.serialize(new RecordObject2("record test 2", 1234));
+        Assertions.assertEquals("record test 2", ret2.getRecordObject2().getKey());
+        Assertions.assertEquals(1234, ret2.getRecordObject2().getValue());
+        var des2 = (RecordObject2) interfaceProtoSerializer.deserialize(ret2);
+        Assertions.assertEquals("record test 2", des2.key());
+        Assertions.assertEquals(1234, des2.value());
+    }
+}

dhfs-parent/autoprotomap/pom.xml

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>com.usatiuk.dhfs</groupId>
+        <artifactId>parent</artifactId>
+        <version>1.0-SNAPSHOT</version>
+    </parent>
+
+    <groupId>com.usatiuk</groupId>
+    <artifactId>autoprotomap-parent</artifactId>
+    <version>1.0-SNAPSHOT</version>
+    <packaging>pom</packaging>
+    <name>Autoprotomap - Parent</name>
+
+    <modules>
+        <module>deployment</module>
+        <module>runtime</module>
+        <module>integration-tests</module>
+    </modules>
+
+</project>

dhfs-parent/autoprotomap/runtime/pom.xml

@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>com.usatiuk</groupId>
+        <artifactId>autoprotomap-parent</artifactId>
+        <version>1.0-SNAPSHOT</version>
+    </parent>
+    <artifactId>autoprotomap</artifactId>
+    <name>Autoprotomap - Runtime</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-arc</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-grpc</artifactId>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>io.quarkus</groupId>
+                <artifactId>quarkus-extension-maven-plugin</artifactId>
+                <version>${quarkus.platform.version}</version>
+                <executions>
+                    <execution>
+                        <phase>compile</phase>
+                        <goals>
+                            <goal>extension-descriptor</goal>
+                        </goals>
+                        <configuration>
+                            <deployment>${project.groupId}:${project.artifactId}-deployment:${project.version}
+                            </deployment>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>default-compile</id>
+                        <configuration>
+                            <annotationProcessorPaths>
+                                <path>
+                                    <groupId>io.quarkus</groupId>
+                                    <artifactId>quarkus-extension-processor</artifactId>
+                                    <version>${quarkus.platform.version}</version>
+                                </path>
+                            </annotationProcessorPaths>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>

dhfs-parent/autoprotomap/runtime/src/main/java/com/usatiuk/autoprotomap/runtime/ProtoMirror.java

@@ -0,0 +1,12 @@
+package com.usatiuk.autoprotomap.runtime;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Retention(RetentionPolicy.CLASS)
+@Target(ElementType.TYPE)
+public @interface ProtoMirror {
+    Class<?> value() default Object.class;
+}

dhfs-parent/autoprotomap/runtime/src/main/java/com/usatiuk/autoprotomap/runtime/ProtoSerializer.java

@@ -1,4 +1,4 @@
-package com.usatiuk.dhfs;
+package com.usatiuk.autoprotomap.runtime;
 
 import com.google.protobuf.Message;
 

dhfs-parent/autoprotomap/runtime/src/main/resources/META-INF/quarkus-extension.yaml

@@ -0,0 +1,9 @@
+name: Autoprotomap
+#description: Do something useful.
+metadata:
+#  keywords:
+#    - autoprotomap
+#  guide: ... # To create and publish this guide, see https://github.com/quarkiverse/quarkiverse/wiki#documenting-your-extension
+#  categories:
+#    - "miscellaneous"
+#  status: "preview"

dhfs-parent/dhfs-app/pom.xml

@@ -1,215 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-    <groupId>com.usatiuk.dhfs</groupId>
-    <artifactId>dhfs-app</artifactId>
-    <version>1.0-SNAPSHOT</version>
-
-    <parent>
-        <groupId>com.usatiuk.dhfs</groupId>
-        <artifactId>parent</artifactId>
-        <version>1.0-SNAPSHOT</version>
-    </parent>
-
-    <dependencies>
-        <dependency>
-            <groupId>org.testcontainers</groupId>
-            <artifactId>testcontainers</artifactId>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.awaitility</groupId>
-            <artifactId>awaitility</artifactId>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.bouncycastle</groupId>
-            <artifactId>bcprov-jdk18on</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.bouncycastle</groupId>
-            <artifactId>bcpkix-jdk18on</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>io.quarkus</groupId>
-            <artifactId>quarkus-security</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>net.openhft</groupId>
-            <artifactId>zero-allocation-hashing</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>io.quarkus</groupId>
-            <artifactId>quarkus-grpc</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>io.quarkus</groupId>
-            <artifactId>quarkus-arc</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>io.quarkus</groupId>
-            <artifactId>quarkus-rest</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>io.quarkus</groupId>
-            <artifactId>quarkus-rest-client</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>io.quarkus</groupId>
-            <artifactId>quarkus-rest-client-jsonb</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>io.quarkus</groupId>
-            <artifactId>quarkus-rest-jsonb</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>io.quarkus</groupId>
-            <artifactId>quarkus-scheduler</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>io.quarkus</groupId>
-            <artifactId>quarkus-junit5</artifactId>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.github.SerCeMan</groupId>
-            <artifactId>jnr-fuse</artifactId>
-            <version>44ed40f8ce</version>
-        </dependency>
-        <dependency>
-            <groupId>com.github.jnr</groupId>
-            <artifactId>jnr-ffi</artifactId>
-            <version>2.2.16</version>
-        </dependency>
-        <dependency>
-            <groupId>com.github.jnr</groupId>
-            <artifactId>jnr-posix</artifactId>
-            <version>3.1.19</version>
-        </dependency>
-        <dependency>
-            <groupId>com.github.jnr</groupId>
-            <artifactId>jnr-constants</artifactId>
-            <version>0.10.4</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.commons</groupId>
-            <artifactId>commons-lang3</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>commons-io</groupId>
-            <artifactId>commons-io</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.jboss.slf4j</groupId>
-            <artifactId>slf4j-jboss-logmanager</artifactId>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>commons-codec</groupId>
-            <artifactId>commons-codec</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.commons</groupId>
-            <artifactId>commons-collections4</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.pcollections</groupId>
-            <artifactId>pcollections</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.commons</groupId>
-            <artifactId>commons-math3</artifactId>
-            <version>3.6.1</version>
-        </dependency>
-        <dependency>
-            <groupId>com.usatiuk</groupId>
-            <artifactId>kleppmanntree</artifactId>
-            <version>1.0-SNAPSHOT</version>
-        </dependency>
-        <dependency>
-            <groupId>com.usatiuk.dhfs</groupId>
-            <artifactId>supportlib</artifactId>
-            <version>1.0-SNAPSHOT</version>
-        </dependency>
-        <dependency>
-            <groupId>com.usatiuk.dhfs</groupId>
-            <artifactId>objects</artifactId>
-            <version>1.0-SNAPSHOT</version>
-        </dependency>
-        <dependency>
-            <groupId>com.usatiuk.dhfs</groupId>
-            <artifactId>dhfs-fs</artifactId>
-            <version>1.0-SNAPSHOT</version>
-        </dependency>
-        <dependency>
-            <groupId>com.usatiuk.dhfs</groupId>
-            <artifactId>dhfs-fuse</artifactId>
-            <version>1.0-SNAPSHOT</version>
-        </dependency>
-        <dependency>
-            <groupId>com.usatiuk.dhfs</groupId>
-            <artifactId>sync-base</artifactId>
-            <version>1.0-SNAPSHOT</version>
-        </dependency>
-        <dependency>
-            <groupId>com.usatiuk.dhfs</groupId>
-            <artifactId>utils</artifactId>
-            <version>1.0-SNAPSHOT</version>
-        </dependency>
-    </dependencies>
-
-    <build>
-        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-surefire-plugin</artifactId>
-                <configuration>
-                    <forkCount>1C</forkCount>
-                    <reuseForks>false</reuseForks>
-                    <parallel>classes</parallel>
-                    <systemPropertyVariables>
-                        <junit.jupiter.execution.parallel.enabled>
-                            false
-                        </junit.jupiter.execution.parallel.enabled>
-                    </systemPropertyVariables>
-                </configuration>
-            </plugin>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-failsafe-plugin</artifactId>
-                <configuration>
-                    <systemPropertyVariables>
-                        <junit.jupiter.execution.parallel.enabled>
-                            true
-                        </junit.jupiter.execution.parallel.enabled>
-                        <junit.jupiter.execution.parallel.mode.default>
-                            concurrent
-                        </junit.jupiter.execution.parallel.mode.default>
-                        <junit.jupiter.execution.parallel.config.dynamic.factor>
-                            0.5
-                        </junit.jupiter.execution.parallel.config.dynamic.factor>
-                        <junit.platform.output.capture.stdout>true</junit.platform.output.capture.stdout>
-                        <junit.platform.output.capture.stderr>true</junit.platform.output.capture.stderr>
-                    </systemPropertyVariables>
-                </configuration>
-            </plugin>
-            <plugin>
-                <groupId>${quarkus.platform.group-id}</groupId>
-                <artifactId>quarkus-maven-plugin</artifactId>
-                <version>${quarkus.platform.version}</version>
-                <extensions>true</extensions>
-                <executions>
-                    <execution>
-                        <id>quarkus-plugin</id>
-                        <goals>
-                            <goal>build</goal>
-                            <goal>generate-code</goal>
-                            <goal>generate-code-tests</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-</project>

dhfs-parent/dhfs-app/src/main/resources/application.properties

@@ -1,34 +0,0 @@
-quarkus.grpc.server.use-separate-server=false
-dhfs.objects.peerdiscovery.port=42069
-dhfs.objects.peerdiscovery.interval=4s
-dhfs.objects.peerdiscovery.broadcast=true
-dhfs.objects.sync.timeout=30
-dhfs.objects.sync.ping.timeout=5
-dhfs.objects.invalidation.threads=16
-dhfs.objects.invalidation.delay=1000
-dhfs.objects.reconnect_interval=5s
-dhfs.objects.write_log=false
-dhfs.objects.periodic-push-op-interval=5m
-dhfs.fuse.root=${HOME}/dhfs_default/fuse
-dhfs.objects.persistence.stuff.root=${HOME}/dhfs_default/data/stuff
-dhfs.fuse.debug=false
-dhfs.fuse.enabled=true
-dhfs.files.allow_recursive_delete=false
-dhfs.files.target_chunk_size=2097152
-dhfs.files.target_chunk_alignment=19
-dhfs.objects.deletion.delay=1000
-dhfs.objects.deletion.can-delete-retry-delay=10000
-dhfs.objects.ref_verification=true
-dhfs.files.use_hash_for_chunks=false
-dhfs.objects.autosync.threads=16
-dhfs.objects.autosync.download-all=false
-dhfs.objects.move-processor.threads=16
-dhfs.objects.ref-processor.threads=16
-dhfs.objects.opsender.batch-size=100
-dhfs.objects.lock_timeout_secs=2
-dhfs.local-discovery=true
-dhfs.peerdiscovery.timeout=10000
-quarkus.log.category."com.usatiuk".min-level=TRACE
-quarkus.log.category."com.usatiuk".level=TRACE
-quarkus.http.insecure-requests=enabled
-quarkus.http.ssl.client-auth=required

dhfs-parent/dhfs-app/src/test/java/com/usatiuk/dhfs/TestDataCleaner.java

@@ -1,44 +0,0 @@
-package com.usatiuk.dhfs;
-
-import io.quarkus.logging.Log;
-import io.quarkus.runtime.ShutdownEvent;
-import io.quarkus.runtime.StartupEvent;
-import jakarta.annotation.Priority;
-import jakarta.enterprise.context.ApplicationScoped;
-import jakarta.enterprise.event.Observes;
-import org.eclipse.microprofile.config.inject.ConfigProperty;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.util.Objects;
-
-@ApplicationScoped
-public class TestDataCleaner {
-    @ConfigProperty(name = "dhfs.objects.persistence.files.root")
-    String tempDirectory;
-
-    void init(@Observes @Priority(1) StartupEvent event) throws IOException {
-        try {
-            purgeDirectory(Path.of(tempDirectory).toFile());
-        } catch (Exception ignored) {
-            Log.warn("Couldn't cleanup test data on init");
-        }
-    }
-
-    void shutdown(@Observes @Priority(1000000000) ShutdownEvent event) throws IOException {
-        purgeDirectory(Path.of(tempDirectory).toFile());
-    }
-
-    public static void purgeDirectory(File dir) {
-        try {
-            for (File file : Objects.requireNonNull(dir.listFiles())) {
-                if (file.isDirectory())
-                    purgeDirectory(file);
-                file.delete();
-            }
-        } catch (Exception e) {
-            Log.error("Couldn't purge directory " + dir, e);
-        }
-    }
-}

dhfs-parent/dhfs-app/src/test/java/com/usatiuk/dhfs/integration/KillIT.java

@@ -1,182 +0,0 @@
-package com.usatiuk.dhfs.integration;
-
-import com.github.dockerjava.api.model.Device;
-import com.usatiuk.dhfs.TestDataCleaner;
-import io.quarkus.logging.Log;
-import org.junit.jupiter.api.*;
-import org.slf4j.LoggerFactory;
-import org.testcontainers.DockerClientFactory;
-import org.testcontainers.containers.GenericContainer;
-import org.testcontainers.containers.Network;
-import org.testcontainers.containers.output.Slf4jLogConsumer;
-import org.testcontainers.containers.output.WaitingConsumer;
-import org.testcontainers.containers.wait.strategy.Wait;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.time.Duration;
-import java.util.Objects;
-import java.util.UUID;
-import java.util.concurrent.CyclicBarrier;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.stream.Stream;
-
-import static org.awaitility.Awaitility.await;
-
-public class KillIT {
-    GenericContainer<?> container1;
-    GenericContainer<?> container2;
-
-    WaitingConsumer waitingConsumer1;
-    WaitingConsumer waitingConsumer2;
-
-    String c1uuid;
-    String c2uuid;
-
-    File data1;
-    File data2;
-
-    @BeforeEach
-    void setup(TestInfo testInfo) throws IOException, InterruptedException, TimeoutException {
-        data1 = Files.createTempDirectory("").toFile();
-        data2 = Files.createTempDirectory("").toFile();
-
-        Network network = Network.newNetwork();
-
-        container1 = new GenericContainer<>(DhfsImage.getInstance())
-                .withPrivilegedMode(true)
-                .withCreateContainerCmdModifier(cmd -> Objects.requireNonNull(cmd.getHostConfig()).withDevices(Device.parse("/dev/fuse")))
-                .waitingFor(Wait.forLogMessage(".*Listening.*", 1).withStartupTimeout(Duration.ofSeconds(60))).withNetwork(network)
-                .withFileSystemBind(data1.getAbsolutePath(), "/dhfs_test/data");
-        container2 = new GenericContainer<>(DhfsImage.getInstance())
-                .withPrivilegedMode(true)
-                .withCreateContainerCmdModifier(cmd -> Objects.requireNonNull(cmd.getHostConfig()).withDevices(Device.parse("/dev/fuse")))
-                .waitingFor(Wait.forLogMessage(".*Listening.*", 1).withStartupTimeout(Duration.ofSeconds(60))).withNetwork(network)
-                .withFileSystemBind(data2.getAbsolutePath(), "/dhfs_test/data");
-
-        Stream.of(container1, container2).parallel().forEach(GenericContainer::start);
-
-        waitingConsumer1 = new WaitingConsumer();
-        var loggingConsumer1 = new Slf4jLogConsumer(LoggerFactory.getLogger(KillIT.class)).withPrefix("1-" + testInfo.getDisplayName());
-        container1.followOutput(loggingConsumer1.andThen(waitingConsumer1));
-        waitingConsumer2 = new WaitingConsumer();
-        var loggingConsumer2 = new Slf4jLogConsumer(LoggerFactory.getLogger(KillIT.class)).withPrefix("2-" + testInfo.getDisplayName());
-        container2.followOutput(loggingConsumer2.andThen(waitingConsumer2));
-
-        c1uuid = container1.execInContainer("/bin/sh", "-c", "cat /dhfs_test/data/stuff/self_uuid").getStdout();
-        c2uuid = container2.execInContainer("/bin/sh", "-c", "cat /dhfs_test/data/stuff/self_uuid").getStdout();
-
-        Assertions.assertDoesNotThrow(() -> UUID.fromString(c1uuid));
-        Assertions.assertDoesNotThrow(() -> UUID.fromString(c2uuid));
-
-        waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("New address"), 60, TimeUnit.SECONDS);
-        waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("New address"), 60, TimeUnit.SECONDS);
-
-        var c1curl = container1.execInContainer("/bin/sh", "-c",
-                "curl --header \"Content-Type: application/json\" " +
-                        "  --request PUT " +
-                        "  --data '{\"uuid\":\"" + c2uuid + "\"}' " +
-                        "  http://localhost:8080/peers-manage/known-peers");
-
-        var c2curl = container2.execInContainer("/bin/sh", "-c",
-                "curl --header \"Content-Type: application/json\" " +
-                        "  --request PUT " +
-                        "  --data '{\"uuid\":\"" + c1uuid + "\"}' " +
-                        "  http://localhost:8080/peers-manage/known-peers");
-
-        waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS);
-        waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS);
-    }
-
-    @AfterEach
-    void stop() {
-        Stream.of(container1, container2).parallel().forEach(GenericContainer::stop);
-        TestDataCleaner.purgeDirectory(data1);
-        TestDataCleaner.purgeDirectory(data2);
-    }
-
-    @Test
-    void killTest(TestInfo testInfo) throws Exception {
-        var executor = Executors.newFixedThreadPool(2);
-        var barrier = new CyclicBarrier(2);
-        var ret1 = executor.submit(() -> {
-            try {
-                Log.info("Writing to container 1");
-                barrier.await();
-                container1.execInContainer("/bin/sh", "-c", "counter=0; while true; do counter=`expr $counter + 1`; echo $counter >> /dhfs_test/fuse/test1; done");
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-        });
-        barrier.await();
-        Thread.sleep(10000);
-        var client = DockerClientFactory.instance().client();
-        client.killContainerCmd(container1.getContainerId()).exec();
-        container1.stop();
-        waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Lost connection to"), 60, TimeUnit.SECONDS);
-        container1.start();
-        waitingConsumer1 = new WaitingConsumer();
-        var loggingConsumer1 = new Slf4jLogConsumer(LoggerFactory.getLogger(KillIT.class)).withPrefix("1-" + testInfo.getDisplayName());
-        container1.followOutput(loggingConsumer1.andThen(waitingConsumer1));
-        waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS);
-        waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS);
-
-        await().atMost(45, TimeUnit.SECONDS).until(() -> {
-            Log.info("Listing consistency");
-            var ls1 = container1.execInContainer("/bin/sh", "-c", "ls /dhfs_test/fuse");
-            var cat1 = container1.execInContainer("/bin/sh", "-c", "cat /dhfs_test/fuse/*");
-            var ls2 = container2.execInContainer("/bin/sh", "-c", "ls /dhfs_test/fuse");
-            var cat2 = container2.execInContainer("/bin/sh", "-c", "cat /dhfs_test/fuse/*");
-            Log.info(ls1);
-            Log.info(cat1);
-            Log.info(ls2);
-            Log.info(cat2);
-
-            return ls1.equals(ls2) && cat1.equals(cat2);
-        });
-    }
-
-    @Test
-    void killTestDirs(TestInfo testInfo) throws Exception {
-        var executor = Executors.newFixedThreadPool(2);
-        var barrier = new CyclicBarrier(2);
-        var ret1 = executor.submit(() -> {
-            try {
-                Log.info("Writing to container 1");
-                barrier.await();
-                container1.execInContainer("/bin/sh", "-c", "counter=0; while true; do counter=`expr $counter + 1`; echo $counter >> /dhfs_test/fuse/test$counter; done");
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-        });
-        barrier.await();
-        Thread.sleep(10000);
-        var client = DockerClientFactory.instance().client();
-        client.killContainerCmd(container1.getContainerId()).exec();
-        container1.stop();
-        waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Lost connection to"), 60, TimeUnit.SECONDS);
-        container1.start();
-        waitingConsumer1 = new WaitingConsumer();
-        var loggingConsumer1 = new Slf4jLogConsumer(LoggerFactory.getLogger(KillIT.class)).withPrefix("1-" + testInfo.getDisplayName());
-        container1.followOutput(loggingConsumer1.andThen(waitingConsumer1));
-        waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS);
-        waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS);
-
-        await().atMost(45, TimeUnit.SECONDS).until(() -> {
-            Log.info("Listing consistency");
-            var ls1 = container1.execInContainer("/bin/sh", "-c", "ls /dhfs_test/fuse");
-            var cat1 = container1.execInContainer("/bin/sh", "-c", "cat /dhfs_test/fuse/*");
-            var ls2 = container2.execInContainer("/bin/sh", "-c", "ls /dhfs_test/fuse");
-            var cat2 = container2.execInContainer("/bin/sh", "-c", "cat /dhfs_test/fuse/*");
-            Log.info(ls1);
-            Log.info(cat1);
-            Log.info(ls2);
-            Log.info(cat2);
-
-            return ls1.equals(ls2) && cat1.equals(cat2);
-        });
-    }
-}

dhfs-parent/dhfs-fs/docker-compose.yml

@@ -1,43 +0,0 @@
-version: "3.2"
-
-services:
-  dhfs1:
-    build: .
-    privileged: true
-    devices:
-      - /dev/fuse
-    volumes:
-      - $HOME/dhfs/dhfs1:/dhfs_root
-      - $HOME/dhfs/dhfs1_f:/dhfs_root/fuse:rshared
-      - ./target/quarkus-app:/app
-    command: "java --add-exports java.base/sun.nio.ch=ALL-UNNAMED
-     -Ddhfs.objects.persistence.files.root=/dhfs_root/p 
-     -Ddhfs.objects.root=/dhfs_root/d 
-     -Ddhfs.fuse.root=/dhfs_root/fuse -Dquarkus.http.host=0.0.0.0 
-     -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005
-     -jar /app/quarkus-run.jar"
-    ports:
-      - 8080:8080
-      - 8081:8443
-      - 5005:5005
-  dhfs2:
-    build: .
-    privileged: true
-    devices:
-      - /dev/fuse
-    volumes:
-      - $HOME/dhfs/dhfs2:/dhfs_root
-      - $HOME/dhfs/dhfs2_f:/dhfs_root/fuse:rshared
-      - ./target/quarkus-app:/app
-    command: "java --add-exports java.base/sun.nio.ch=ALL-UNNAMED
-     --add-exports java.base/jdk.internal.access=ALL-UNNAMED
-     --add-opens=java.base/java.nio=ALL-UNNAMED
-     -Ddhfs.objects.persistence.files.root=/dhfs_root/p 
-     -Ddhfs.objects.root=/dhfs_root/d 
-     -Ddhfs.fuse.root=/dhfs_root/fuse -Dquarkus.http.host=0.0.0.0
-     -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5010
-     -jar /app/quarkus-run.jar"
-    ports:
-      - 8090:8080
-      - 8091:8443
-      - 5010:5010

dhfs-parent/dhfs-fs/src/main/java/com/usatiuk/dhfs/files/objects/ChunkDataProtoSerializer.java

@@ -1,26 +0,0 @@
-package com.usatiuk.dhfs.files.objects;
-
-import com.usatiuk.dhfs.ProtoSerializer;
-import com.usatiuk.objects.JObjectKey;
-import com.usatiuk.dhfs.persistence.ChunkDataP;
-import com.usatiuk.dhfs.persistence.JObjectKeyP;
-import jakarta.inject.Singleton;
-
-@Singleton
-public class ChunkDataProtoSerializer implements ProtoSerializer<ChunkDataP, ChunkData> {
-    @Override
-    public ChunkData deserialize(ChunkDataP message) {
-        return new ChunkData(
-                JObjectKey.of(message.getKey().getName()),
-                message.getData()
-        );
-    }
-
-    @Override
-    public ChunkDataP serialize(ChunkData object) {
-        return ChunkDataP.newBuilder()
-                .setKey(JObjectKeyP.newBuilder().setName(object.key().value()).build())
-                .setData(object.data())
-                .build();
-    }
-}

dhfs-parent/dhfs-fs/src/main/java/com/usatiuk/dhfs/files/objects/FileDto.java

@@ -1,15 +0,0 @@
-package com.usatiuk.dhfs.files.objects;
-
-import com.usatiuk.dhfs.JDataRemote;
-import com.usatiuk.objects.JObjectKey;
-import com.usatiuk.dhfs.repository.JDataRemoteDto;
-import org.apache.commons.lang3.tuple.Pair;
-
-import java.util.List;
-
-public record FileDto(File file, List<Pair<Long, JObjectKey>> chunks) implements JDataRemoteDto {
-    @Override
-    public Class<? extends JDataRemote> objClass() {
-        return File.class;
-    }
-}

dhfs-parent/dhfs-fs/src/main/java/com/usatiuk/dhfs/files/objects/FileDtoMapper.java

@@ -1,24 +0,0 @@
-package com.usatiuk.dhfs.files.objects;
-
-import com.usatiuk.dhfs.jmap.JMapHelper;
-import com.usatiuk.dhfs.repository.syncmap.DtoMapper;
-import jakarta.enterprise.context.ApplicationScoped;
-import jakarta.inject.Inject;
-
-@ApplicationScoped
-public class FileDtoMapper implements DtoMapper<File, FileDto> {
-    @Inject
-    JMapHelper jMapHelper;
-    @Inject
-    FileHelper fileHelper;
-
-    @Override
-    public FileDto toDto(File obj) {
-        return new FileDto(obj, fileHelper.getChunks(obj));
-    }
-
-    @Override
-    public File fromDto(FileDto dto) {
-        throw new UnsupportedOperationException();
-    }
-}

dhfs-parent/dhfs-fs/src/main/java/com/usatiuk/dhfs/files/objects/FileHelper.java

@@ -1,36 +0,0 @@
-package com.usatiuk.dhfs.files.objects;
-
-import com.usatiuk.objects.JObjectKey;
-import com.usatiuk.dhfs.jmap.JMapHelper;
-import com.usatiuk.dhfs.jmap.JMapLongKey;
-import jakarta.enterprise.context.ApplicationScoped;
-import jakarta.inject.Inject;
-import org.apache.commons.lang3.tuple.Pair;
-
-import java.util.ArrayList;
-import java.util.List;
-
-@ApplicationScoped
-public class FileHelper {
-    @Inject
-    JMapHelper jMapHelper;
-
-    public List<Pair<Long, JObjectKey>> getChunks(File file) {
-        ArrayList<Pair<Long, JObjectKey>> chunks = new ArrayList<>();
-        try (var it = jMapHelper.getIterator(file)) {
-            while (it.hasNext()) {
-                var cur = it.next();
-                chunks.add(Pair.of(cur.getKey().key(), cur.getValue().ref()));
-            }
-        }
-        return List.copyOf(chunks);
-    }
-
-    public void replaceChunks(File file, List<Pair<Long, JObjectKey>> chunks) {
-        jMapHelper.deleteAll(file);
-
-        for (var f : chunks) {
-            jMapHelper.put(file, JMapLongKey.of(f.getLeft()), f.getRight());
-        }
-    }
-}

dhfs-parent/dhfs-fs/src/main/java/com/usatiuk/dhfs/files/objects/FileProtoSerializer.java

@@ -1,25 +0,0 @@
-package com.usatiuk.dhfs.files.objects;
-
-import com.usatiuk.dhfs.ProtoSerializer;
-import com.usatiuk.dhfs.persistence.FileDtoP;
-import com.usatiuk.dhfs.utils.SerializationHelper;
-import jakarta.inject.Singleton;
-
-import java.io.IOException;
-
-@Singleton
-public class FileProtoSerializer implements ProtoSerializer<FileDtoP, FileDto> {
-    @Override
-    public FileDto deserialize(FileDtoP message) {
-        try (var is = message.getSerializedData().newInput()) {
-            return SerializationHelper.deserialize(is);
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    @Override
-    public FileDtoP serialize(FileDto object) {
-        return FileDtoP.newBuilder().setSerializedData(SerializationHelper.serialize(object)).build();
-    }
-}

dhfs-parent/dhfs-fs/src/main/java/com/usatiuk/dhfs/files/objects/FileSyncHandler.java

@@ -1,241 +0,0 @@
-package com.usatiuk.dhfs.files.objects;
-
-import com.usatiuk.dhfs.PeerId;
-import com.usatiuk.dhfs.RemoteObjectDataWrapper;
-import com.usatiuk.dhfs.RemoteObjectMeta;
-import com.usatiuk.dhfs.RemoteTransaction;
-import com.usatiuk.dhfs.files.service.DhfsFileService;
-import com.usatiuk.dhfs.jkleppmanntree.JKleppmannTreeManager;
-import com.usatiuk.dhfs.jkleppmanntree.structs.JKleppmannTreeNodeMetaFile;
-import com.usatiuk.dhfs.jmap.JMapHelper;
-import com.usatiuk.dhfs.repository.ObjSyncHandler;
-import com.usatiuk.dhfs.repository.PersistentPeerDataService;
-import com.usatiuk.dhfs.repository.SyncHelper;
-import com.usatiuk.objects.JObjectKey;
-import com.usatiuk.objects.transaction.LockingStrategy;
-import com.usatiuk.objects.transaction.Transaction;
-import com.usatiuk.kleppmanntree.AlreadyExistsException;
-import io.grpc.Status;
-import io.grpc.StatusRuntimeException;
-import io.quarkus.logging.Log;
-import jakarta.enterprise.context.ApplicationScoped;
-import jakarta.inject.Inject;
-import org.apache.commons.lang3.tuple.Pair;
-import org.pcollections.HashPMap;
-import org.pcollections.HashTreePMap;
-import org.pcollections.PMap;
-
-import javax.annotation.Nullable;
-import java.util.List;
-import java.util.Objects;
-
-@ApplicationScoped
-public class FileSyncHandler implements ObjSyncHandler<File, FileDto> {
-    @Inject
-    Transaction curTx;
-    @Inject
-    PersistentPeerDataService persistentPeerDataService;
-    @Inject
-    JMapHelper jMapHelper;
-    @Inject
-    RemoteTransaction remoteTx;
-    @Inject
-    FileHelper fileHelper;
-
-    @Inject
-    JKleppmannTreeManager jKleppmannTreeManager;
-    @Inject
-    DhfsFileService fileService;
-
-    private JKleppmannTreeManager.JKleppmannTree getTreeW() {
-        return jKleppmannTreeManager.getTree(JObjectKey.of("fs"));
-    }
-
-    private JKleppmannTreeManager.JKleppmannTree getTreeR() {
-        return jKleppmannTreeManager.getTree(JObjectKey.of("fs"), LockingStrategy.OPTIMISTIC);
-    }
-
-    private void resolveConflict(PeerId from, JObjectKey key, PMap<PeerId, Long> receivedChangelog,
-                                 @Nullable FileDto receivedData) {
-        var oursCurMeta = curTx.get(RemoteObjectMeta.class, key).orElse(null);
-
-        if (!oursCurMeta.knownType().isAssignableFrom(File.class))
-            throw new IllegalStateException("Object type mismatch: " + oursCurMeta.knownType() + " vs " + File.class);
-
-        if (!oursCurMeta.knownType().equals(File.class))
-            oursCurMeta = oursCurMeta.withKnownType(File.class);
-
-        curTx.put(oursCurMeta);
-
-        var oursCurFile = remoteTx.getDataLocal(File.class, key).orElse(null);
-        if (oursCurFile == null)
-            throw new StatusRuntimeException(Status.ABORTED.withDescription("Conflict but we don't have local copy"));
-
-        var theirsFile = receivedData.file();
-
-        var oursChunks = fileHelper.getChunks(oursCurFile);
-
-        File first;
-        File second;
-        List<Pair<Long, JObjectKey>> firstChunks;
-        List<Pair<Long, JObjectKey>> secondChunks;
-        PeerId otherHostname;
-
-        if (oursCurFile.mTime() >= theirsFile.mTime()) {
-            first = oursCurFile;
-            firstChunks = oursChunks;
-            second = theirsFile;
-            secondChunks = receivedData.chunks();
-            otherHostname = from;
-        } else {
-            second = oursCurFile;
-            secondChunks = oursChunks;
-            first = theirsFile;
-            firstChunks = receivedData.chunks();
-            otherHostname = persistentPeerDataService.getSelfUuid();
-        }
-
-        Log.tracev("Conflict resolution: ours: {0}, theirs: {1}, chunks: {2}, {3}", oursCurFile, theirsFile, oursChunks, receivedData.chunks());
-        Log.tracev("Conflict resolution: first: {0}, second: {1}, chunks: {2}, {3}", first, second, firstChunks, secondChunks);
-
-        HashPMap<PeerId, Long> newChangelog = HashTreePMap.from(oursCurMeta.changelog());
-
-        for (var entry : receivedChangelog.entrySet()) {
-            newChangelog = newChangelog.plus(entry.getKey(),
-                    Long.max(newChangelog.getOrDefault(entry.getKey(), 0L), entry.getValue())
-            );
-        }
-
-        oursCurMeta = oursCurMeta.withChangelog(newChangelog);
-        curTx.put(oursCurMeta);
-
-        boolean chunksDiff = !Objects.equals(firstChunks, secondChunks);
-
-        boolean wasChanged = first.mTime() != second.mTime()
-                || first.cTime() != second.cTime()
-                || first.mode() != second.mode()
-                || first.symlink() != second.symlink()
-                || chunksDiff;
-
-        if (wasChanged) {
-            oursCurMeta = oursCurMeta.withChangelog(
-                    newChangelog.plus(persistentPeerDataService.getSelfUuid(), newChangelog.getOrDefault(persistentPeerDataService.getSelfUuid(), 0L) + 1)
-            );
-            curTx.put(oursCurMeta);
-
-            remoteTx.putDataRaw(oursCurFile.withCTime(first.cTime()).withMTime(first.mTime()).withMode(first.mode()).withSymlink(first.symlink()));
-            fileHelper.replaceChunks(oursCurFile, firstChunks);
-
-            var newFile = new File(JObjectKey.random(), second.mode(), second.cTime(), second.mTime(), second.symlink());
-            remoteTx.putData(newFile);
-            fileHelper.replaceChunks(newFile, secondChunks);
-
-            var parent = fileService.inoToParent(oursCurFile.key());
-
-            int i = 0;
-
-            do {
-                try {
-                    getTreeW().move(parent.getRight(),
-                            new JKleppmannTreeNodeMetaFile(
-                                    parent.getLeft() + ".fconflict." + persistentPeerDataService.getSelfUuid() + "." + otherHostname.toString() + "." + i,
-                                    newFile.key()
-                            ),
-                            getTreeW().getNewNodeId()
-                    );
-                } catch (AlreadyExistsException aex) {
-                    i++;
-                    continue;
-                }
-                break;
-            } while (true);
-        }
-
-        var curKnownRemoteVersion = oursCurMeta.knownRemoteVersions().get(from);
-        var receivedTotalVer = receivedChangelog.values().stream().mapToLong(Long::longValue).sum();
-
-        if (curKnownRemoteVersion == null || curKnownRemoteVersion < receivedTotalVer) {
-            oursCurMeta = oursCurMeta.withKnownRemoteVersions(oursCurMeta.knownRemoteVersions().plus(from, receivedTotalVer));
-            curTx.put(oursCurMeta);
-        }
-    }
-
-    @Override
-    public void handleRemoteUpdate(PeerId from, JObjectKey key, PMap<PeerId, Long> receivedChangelog,
-                                   @Nullable FileDto receivedData) {
-        var current = curTx.get(RemoteObjectMeta.class, key).orElse(null);
-        if (current == null) {
-            current = new RemoteObjectMeta(key, HashTreePMap.empty());
-            curTx.put(current);
-        }
-
-        var changelogCompare = SyncHelper.compareChangelogs(current.changelog(), receivedChangelog);
-
-        switch (changelogCompare) {
-            case EQUAL -> {
-                Log.debug("No action on update: " + key + " from " + from);
-                if (!current.hasLocalData() && receivedData != null) {
-                    current = current.withHaveLocal(true);
-                    curTx.put(current);
-                    curTx.put(curTx.get(RemoteObjectDataWrapper.class, RemoteObjectMeta.ofDataKey(current.key()))
-                            .map(w -> w.withData(receivedData.file())).orElse(new RemoteObjectDataWrapper<>(receivedData.file())));
-
-                    if (!current.knownType().isAssignableFrom(File.class))
-                        throw new IllegalStateException("Object type mismatch: " + current.knownType() + " vs " + File.class);
-
-                    if (!current.knownType().equals(File.class))
-                        current = current.withKnownType(File.class);
-
-                    curTx.put(current);
-
-                    fileHelper.replaceChunks(receivedData.file(), receivedData.chunks());
-                }
-            }
-            case NEWER -> {
-                Log.debug("Received newer index update than known: " + key + " from " + from);
-                var newChangelog = receivedChangelog.containsKey(persistentPeerDataService.getSelfUuid()) ?
-                        receivedChangelog : receivedChangelog.plus(persistentPeerDataService.getSelfUuid(), 0L);
-                current = current.withChangelog(newChangelog);
-
-                if (receivedData != null) {
-                    current = current.withHaveLocal(true);
-                    curTx.put(current);
-                    curTx.put(curTx.get(RemoteObjectDataWrapper.class, RemoteObjectMeta.ofDataKey(current.key()))
-                            .map(w -> w.withData(receivedData.file())).orElse(new RemoteObjectDataWrapper<>(receivedData.file())));
-
-                    if (!current.knownType().isAssignableFrom(File.class))
-                        throw new IllegalStateException("Object type mismatch: " + current.knownType() + " vs " + File.class);
-
-                    if (!current.knownType().equals(File.class))
-                        current = current.withKnownType(File.class);
-
-                    curTx.put(current);
-
-                    fileHelper.replaceChunks(receivedData.file(), receivedData.chunks());
-                } else {
-                    current = current.withHaveLocal(false);
-                    curTx.put(current);
-                }
-            }
-            case OLDER -> {
-                Log.debug("Received older index update than known: " + key + " from " + from);
-                return;
-            }
-            case CONFLICT -> {
-                Log.debug("Conflict on update (inconsistent version): " + key + " from " + from);
-                assert receivedData != null;
-                resolveConflict(from, key, receivedChangelog, receivedData);
-                // TODO:
-                return;
-            }
-        }
-        var curKnownRemoteVersion = current.knownRemoteVersions().get(from);
-        var receivedTotalVer = receivedChangelog.values().stream().mapToLong(Long::longValue).sum();
-
-        if (curKnownRemoteVersion == null || curKnownRemoteVersion < receivedTotalVer) {
-            current = current.withKnownRemoteVersions(current.knownRemoteVersions().plus(from, receivedTotalVer));
-            curTx.put(current);
-        }
-
-    }
-}

dhfs-parent/dhfs-fs/src/test/java/com/usatiuk/dhfs/TempDataProfile.java

@@ -1,29 +0,0 @@
-package com.usatiuk.dhfs;
-
-import io.quarkus.test.junit.QuarkusTestProfile;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.HashMap;
-import java.util.Map;
-
-abstract public class TempDataProfile implements QuarkusTestProfile {
-    protected void getConfigOverrides(Map<String, String> toPut) {
-    }
-
-    @Override
-    final public Map<String, String> getConfigOverrides() {
-        Path tempDirWithPrefix;
-        try {
-            tempDirWithPrefix = Files.createTempDirectory("dhfs-test");
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-        var ret = new HashMap<String, String>();
-        ret.put("dhfs.objects.persistence.files.root", tempDirWithPrefix.resolve("dhfs_root_test").toString());
-        ret.put("dhfs.fuse.root", tempDirWithPrefix.resolve("dhfs_fuse_root_test").toString());
-        getConfigOverrides(ret);
-        return ret;
-    }
-}

dhfs-parent/dhfs-fs/src/test/resources/application.properties

@@ -1,12 +0,0 @@
-dhfs.objects.persistence.files.root=${HOME}/dhfs_data/dhfs_root_test
-dhfs.objects.root=${HOME}/dhfs_data/dhfs_root_d_test
-dhfs.fuse.root=${HOME}/dhfs_data/dhfs_fuse_root_test
-dhfs.objects.ref_verification=true
-dhfs.objects.deletion.delay=0
-quarkus.log.category."com.usatiuk.dhfs".level=TRACE
-quarkus.log.category."com.usatiuk.dhfs".min-level=TRACE
-quarkus.class-loading.parent-first-artifacts=com.usatiuk.dhfs:supportlib
-quarkus.http.test-port=0
-quarkus.http.test-ssl-port=0
-dhfs.local-discovery=false
-dhfs.objects.persistence.snapshot-extra-checks=true

dhfs-parent/dhfs-fuse/.dockerignore

@@ -1,5 +0,0 @@
-*
-!target/*-runner
-!target/*-runner.jar
-!target/lib/*
-!target/quarkus-app/*

dhfs-parent/dhfs-fuse/.gitignore

@@ -1,43 +0,0 @@
-#Maven
-target/
-pom.xml.tag
-pom.xml.releaseBackup
-pom.xml.versionsBackup
-release.properties
-.flattened-pom.xml
-
-# Eclipse
-.project
-.classpath
-.settings/
-bin/
-
-# IntelliJ
-.idea
-*.ipr
-*.iml
-*.iws
-
-# NetBeans
-nb-configuration.xml
-
-# Visual Studio Code
-.vscode
-.factorypath
-
-# OSX
-.DS_Store
-
-# Vim
-*.swp
-*.swo
-
-# patch
-*.orig
-*.rej
-
-# Local environment
-.env
-
-# Plugin directory
-/.quarkus/cli/plugins/

dhfs-parent/dhfs-fuse/Dockerfile

@@ -1,2 +0,0 @@
-FROM azul/zulu-openjdk-debian:21-jre-latest
-RUN apt update && apt install -y libfuse2 curl

dhfs-parent/dhfs-fuse/src/main/docker/Dockerfile.jvm

@@ -1,97 +0,0 @@
-####
-# This Dockerfile is used in order to build a container that runs the Quarkus application in JVM mode
-#
-# Before building the container image run:
-#
-# ./mvnw package
-#
-# Then, build the image with:
-#
-# docker build -f src/main/docker/Dockerfile.jvm -t quarkus/server-jvm .
-#
-# Then run the container using:
-#
-# docker run -i --rm -p 8080:8080 quarkus/server-jvm
-#
-# If you want to include the debug port into your docker image
-# you will have to expose the debug port (default 5005 being the default) like this :  EXPOSE 8080 5005.
-# Additionally you will have to set -e JAVA_DEBUG=true and -e JAVA_DEBUG_PORT=*:5005
-# when running the container
-#
-# Then run the container using :
-#
-# docker run -i --rm -p 8080:8080 quarkus/server-jvm
-#
-# This image uses the `run-java.sh` script to run the application.
-# This scripts computes the command line to execute your Java application, and
-# includes memory/GC tuning.
-# You can configure the behavior using the following environment properties:
-# - JAVA_OPTS: JVM options passed to the `java` command (example: "-verbose:class")
-# - JAVA_OPTS_APPEND: User specified Java options to be appended to generated options
-#   in JAVA_OPTS (example: "-Dsome.property=foo")
-# - JAVA_MAX_MEM_RATIO: Is used when no `-Xmx` option is given in JAVA_OPTS. This is
-#   used to calculate a default maximal heap memory based on a containers restriction.
-#   If used in a container without any memory constraints for the container then this
-#   option has no effect. If there is a memory constraint then `-Xmx` is set to a ratio
-#   of the container available memory as set here. The default is `50` which means 50%
-#   of the available memory is used as an upper boundary. You can skip this mechanism by
-#   setting this value to `0` in which case no `-Xmx` option is added.
-# - JAVA_INITIAL_MEM_RATIO: Is used when no `-Xms` option is given in JAVA_OPTS. This
-#   is used to calculate a default initial heap memory based on the maximum heap memory.
-#   If used in a container without any memory constraints for the container then this
-#   option has no effect. If there is a memory constraint then `-Xms` is set to a ratio
-#   of the `-Xmx` memory as set here. The default is `25` which means 25% of the `-Xmx`
-#   is used as the initial heap size. You can skip this mechanism by setting this value
-#   to `0` in which case no `-Xms` option is added (example: "25")
-# - JAVA_MAX_INITIAL_MEM: Is used when no `-Xms` option is given in JAVA_OPTS.
-#   This is used to calculate the maximum value of the initial heap memory. If used in
-#   a container without any memory constraints for the container then this option has
-#   no effect. If there is a memory constraint then `-Xms` is limited to the value set
-#   here. The default is 4096MB which means the calculated value of `-Xms` never will
-#   be greater than 4096MB. The value of this variable is expressed in MB (example: "4096")
-# - JAVA_DIAGNOSTICS: Set this to get some diagnostics information to standard output
-#   when things are happening. This option, if set to true, will set
-#  `-XX:+UnlockDiagnosticVMOptions`. Disabled by default (example: "true").
-# - JAVA_DEBUG: If set remote debugging will be switched on. Disabled by default (example:
-#    true").
-# - JAVA_DEBUG_PORT: Port used for remote debugging. Defaults to 5005 (example: "8787").
-# - CONTAINER_CORE_LIMIT: A calculated core limit as described in
-#   https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt. (example: "2")
-# - CONTAINER_MAX_MEMORY: Memory limit given to the container (example: "1024").
-# - GC_MIN_HEAP_FREE_RATIO: Minimum percentage of heap free after GC to avoid expansion.
-#   (example: "20")
-# - GC_MAX_HEAP_FREE_RATIO: Maximum percentage of heap free after GC to avoid shrinking.
-#   (example: "40")
-# - GC_TIME_RATIO: Specifies the ratio of the time spent outside the garbage collection.
-#   (example: "4")
-# - GC_ADAPTIVE_SIZE_POLICY_WEIGHT: The weighting given to the current GC time versus
-#   previous GC times. (example: "90")
-# - GC_METASPACE_SIZE: The initial metaspace size. (example: "20")
-# - GC_MAX_METASPACE_SIZE: The maximum metaspace size. (example: "100")
-# - GC_CONTAINER_OPTIONS: Specify Java GC to use. The value of this variable should
-#   contain the necessary JRE command-line options to specify the required GC, which
-#   will override the default of `-XX:+UseParallelGC` (example: -XX:+UseG1GC).
-# - HTTPS_PROXY: The location of the https proxy. (example: "myuser@127.0.0.1:8080")
-# - HTTP_PROXY: The location of the http proxy. (example: "myuser@127.0.0.1:8080")
-# - NO_PROXY: A comma separated lists of hosts, IP addresses or domains that can be
-#   accessed directly. (example: "foo.example.com,bar.example.com")
-#
-###
-FROM registry.access.redhat.com/ubi8/openjdk-21:1.18
-
-ENV LANGUAGE='en_US:en'
-
-
-# We make four distinct layers so if there are application changes the library layers can be re-used
-COPY --chown=185 target/quarkus-app/lib/ /deployments/lib/
-COPY --chown=185 target/quarkus-app/*.jar /deployments/
-COPY --chown=185 target/quarkus-app/app/ /deployments/app/
-COPY --chown=185 target/quarkus-app/quarkus/ /deployments/quarkus/
-
-EXPOSE 8080
-USER 185
-ENV JAVA_OPTS_APPEND="-Dquarkus.http.host=0.0.0.0 -Djava.util.logging.manager=org.jboss.logmanager.LogManager"
-ENV JAVA_APP_JAR="/deployments/quarkus-run.jar"
-
-ENTRYPOINT [ "/opt/jboss/container/java/run/run-java.sh" ]
-

dhfs-parent/dhfs-fuse/src/main/docker/Dockerfile.legacy-jar

@@ -1,93 +0,0 @@
-####
-# This Dockerfile is used in order to build a container that runs the Quarkus application in JVM mode
-#
-# Before building the container image run:
-#
-# ./mvnw package -Dquarkus.package.jar.type=legacy-jar
-#
-# Then, build the image with:
-#
-# docker build -f src/main/docker/Dockerfile.legacy-jar -t quarkus/server-legacy-jar .
-#
-# Then run the container using:
-#
-# docker run -i --rm -p 8080:8080 quarkus/server-legacy-jar
-#
-# If you want to include the debug port into your docker image
-# you will have to expose the debug port (default 5005 being the default) like this :  EXPOSE 8080 5005.
-# Additionally you will have to set -e JAVA_DEBUG=true and -e JAVA_DEBUG_PORT=*:5005
-# when running the container
-#
-# Then run the container using :
-#
-# docker run -i --rm -p 8080:8080 quarkus/server-legacy-jar
-#
-# This image uses the `run-java.sh` script to run the application.
-# This scripts computes the command line to execute your Java application, and
-# includes memory/GC tuning.
-# You can configure the behavior using the following environment properties:
-# - JAVA_OPTS: JVM options passed to the `java` command (example: "-verbose:class")
-# - JAVA_OPTS_APPEND: User specified Java options to be appended to generated options
-#   in JAVA_OPTS (example: "-Dsome.property=foo")
-# - JAVA_MAX_MEM_RATIO: Is used when no `-Xmx` option is given in JAVA_OPTS. This is
-#   used to calculate a default maximal heap memory based on a containers restriction.
-#   If used in a container without any memory constraints for the container then this
-#   option has no effect. If there is a memory constraint then `-Xmx` is set to a ratio
-#   of the container available memory as set here. The default is `50` which means 50%
-#   of the available memory is used as an upper boundary. You can skip this mechanism by
-#   setting this value to `0` in which case no `-Xmx` option is added.
-# - JAVA_INITIAL_MEM_RATIO: Is used when no `-Xms` option is given in JAVA_OPTS. This
-#   is used to calculate a default initial heap memory based on the maximum heap memory.
-#   If used in a container without any memory constraints for the container then this
-#   option has no effect. If there is a memory constraint then `-Xms` is set to a ratio
-#   of the `-Xmx` memory as set here. The default is `25` which means 25% of the `-Xmx`
-#   is used as the initial heap size. You can skip this mechanism by setting this value
-#   to `0` in which case no `-Xms` option is added (example: "25")
-# - JAVA_MAX_INITIAL_MEM: Is used when no `-Xms` option is given in JAVA_OPTS.
-#   This is used to calculate the maximum value of the initial heap memory. If used in
-#   a container without any memory constraints for the container then this option has
-#   no effect. If there is a memory constraint then `-Xms` is limited to the value set
-#   here. The default is 4096MB which means the calculated value of `-Xms` never will
-#   be greater than 4096MB. The value of this variable is expressed in MB (example: "4096")
-# - JAVA_DIAGNOSTICS: Set this to get some diagnostics information to standard output
-#   when things are happening. This option, if set to true, will set
-#  `-XX:+UnlockDiagnosticVMOptions`. Disabled by default (example: "true").
-# - JAVA_DEBUG: If set remote debugging will be switched on. Disabled by default (example:
-#    true").
-# - JAVA_DEBUG_PORT: Port used for remote debugging. Defaults to 5005 (example: "8787").
-# - CONTAINER_CORE_LIMIT: A calculated core limit as described in
-#   https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt. (example: "2")
-# - CONTAINER_MAX_MEMORY: Memory limit given to the container (example: "1024").
-# - GC_MIN_HEAP_FREE_RATIO: Minimum percentage of heap free after GC to avoid expansion.
-#   (example: "20")
-# - GC_MAX_HEAP_FREE_RATIO: Maximum percentage of heap free after GC to avoid shrinking.
-#   (example: "40")
-# - GC_TIME_RATIO: Specifies the ratio of the time spent outside the garbage collection.
-#   (example: "4")
-# - GC_ADAPTIVE_SIZE_POLICY_WEIGHT: The weighting given to the current GC time versus
-#   previous GC times. (example: "90")
-# - GC_METASPACE_SIZE: The initial metaspace size. (example: "20")
-# - GC_MAX_METASPACE_SIZE: The maximum metaspace size. (example: "100")
-# - GC_CONTAINER_OPTIONS: Specify Java GC to use. The value of this variable should
-#   contain the necessary JRE command-line options to specify the required GC, which
-#   will override the default of `-XX:+UseParallelGC` (example: -XX:+UseG1GC).
-# - HTTPS_PROXY: The location of the https proxy. (example: "myuser@127.0.0.1:8080")
-# - HTTP_PROXY: The location of the http proxy. (example: "myuser@127.0.0.1:8080")
-# - NO_PROXY: A comma separated lists of hosts, IP addresses or domains that can be
-#   accessed directly. (example: "foo.example.com,bar.example.com")
-#
-###
-FROM registry.access.redhat.com/ubi8/openjdk-21:1.18
-
-ENV LANGUAGE='en_US:en'
-
-
-COPY target/lib/* /deployments/lib/
-COPY target/*-runner.jar /deployments/quarkus-run.jar
-
-EXPOSE 8080
-USER 185
-ENV JAVA_OPTS_APPEND="-Dquarkus.http.host=0.0.0.0 -Djava.util.logging.manager=org.jboss.logmanager.LogManager"
-ENV JAVA_APP_JAR="/deployments/quarkus-run.jar"
-
-ENTRYPOINT [ "/opt/jboss/container/java/run/run-java.sh" ]

dhfs-parent/dhfs-fuse/src/main/docker/Dockerfile.native

@@ -1,27 +0,0 @@
-####
-# This Dockerfile is used in order to build a container that runs the Quarkus application in native (no JVM) mode.
-#
-# Before building the container image run:
-#
-# ./mvnw package -Dnative
-#
-# Then, build the image with:
-#
-# docker build -f src/main/docker/Dockerfile.native -t quarkus/server .
-#
-# Then run the container using:
-#
-# docker run -i --rm -p 8080:8080 quarkus/server
-#
-###
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.9
-WORKDIR /work/
-RUN chown 1001 /work \
-    && chmod "g+rwX" /work \
-    && chown 1001:root /work
-COPY --chown=1001:root target/*-runner /work/application
-
-EXPOSE 8080
-USER 1001
-
-ENTRYPOINT ["./application", "-Dquarkus.http.host=0.0.0.0"]

dhfs-parent/dhfs-fuse/src/main/docker/Dockerfile.native-micro

@@ -1,30 +0,0 @@
-####
-# This Dockerfile is used in order to build a container that runs the Quarkus application in native (no JVM) mode.
-# It uses a micro base image, tuned for Quarkus native executables.
-# It reduces the size of the resulting container image.
-# Check https://quarkus.io/guides/quarkus-runtime-base-image for further information about this image.
-#
-# Before building the container image run:
-#
-# ./mvnw package -Dnative
-#
-# Then, build the image with:
-#
-# docker build -f src/main/docker/Dockerfile.native-micro -t quarkus/server .
-#
-# Then run the container using:
-#
-# docker run -i --rm -p 8080:8080 quarkus/server
-#
-###
-FROM quay.io/quarkus/quarkus-micro-image:2.0
-WORKDIR /work/
-RUN chown 1001 /work \
-    && chmod "g+rwX" /work \
-    && chown 1001:root /work
-COPY --chown=1001:root target/*-runner /work/application
-
-EXPOSE 8080
-USER 1001
-
-ENTRYPOINT ["./application", "-Dquarkus.http.host=0.0.0.0"]

dhfs-parent/dhfs-fuse/src/main/resources/application.properties

@@ -1,34 +0,0 @@
-quarkus.grpc.server.use-separate-server=false
-dhfs.objects.peerdiscovery.port=42069
-dhfs.objects.peerdiscovery.interval=4s
-dhfs.objects.peerdiscovery.broadcast=true
-dhfs.objects.sync.timeout=30
-dhfs.objects.sync.ping.timeout=5
-dhfs.objects.invalidation.threads=16
-dhfs.objects.invalidation.delay=1000
-dhfs.objects.reconnect_interval=5s
-dhfs.objects.write_log=false
-dhfs.objects.periodic-push-op-interval=5m
-dhfs.fuse.root=${HOME}/dhfs_default/fuse
-dhfs.objects.persistence.stuff.root=${HOME}/dhfs_default/data/stuff
-dhfs.fuse.debug=false
-dhfs.fuse.enabled=true
-dhfs.files.allow_recursive_delete=false
-dhfs.files.target_chunk_size=2097152
-dhfs.files.target_chunk_alignment=19
-dhfs.objects.deletion.delay=1000
-dhfs.objects.deletion.can-delete-retry-delay=10000
-dhfs.objects.ref_verification=true
-dhfs.files.use_hash_for_chunks=false
-dhfs.objects.autosync.threads=16
-dhfs.objects.autosync.download-all=false
-dhfs.objects.move-processor.threads=16
-dhfs.objects.ref-processor.threads=16
-dhfs.objects.opsender.batch-size=100
-dhfs.objects.lock_timeout_secs=2
-dhfs.local-discovery=true
-dhfs.peerdiscovery.timeout=10000
-quarkus.log.category."com.usatiuk".min-level=TRACE
-quarkus.log.category."com.usatiuk".level=TRACE
-quarkus.http.insecure-requests=enabled
-quarkus.http.ssl.client-auth=required

dhfs-parent/kleppmanntree/pom.xml

@@ -18,11 +18,6 @@
             <artifactId>junit-jupiter-engine</artifactId>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>org.junit.jupiter</groupId>
-            <artifactId>junit-jupiter</artifactId>
-            <scope>test</scope>
-        </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-collections4</artifactId>
@@ -35,9 +30,5 @@
             <groupId>org.pcollections</groupId>
             <artifactId>pcollections</artifactId>
         </dependency>
-        <dependency>
-            <groupId>jakarta.annotation</groupId>
-            <artifactId>jakarta.annotation-api</artifactId>
-        </dependency>
     </dependencies>
 </project>

dhfs-parent/kleppmanntree/src/main/java/com/usatiuk/kleppmanntree/KleppmannTree.java

@@ -1,7 +1,5 @@
 package com.usatiuk.kleppmanntree;
 
-import jakarta.annotation.Nonnull;
-import jakarta.annotation.Nullable;
 import org.apache.commons.lang3.tuple.Pair;
 
 import java.util.*;
@@ -55,20 +53,18 @@ public class KleppmannTree<TimestampT extends Comparable<TimestampT>, PeerIdT ex
             var node = _storage.getById(effect.childId());
             var curParent = _storage.getById(effect.newParentId());
             {
-                var newCurParentChildren = curParent.children().minus(node.name());
+                var newCurParentChildren = curParent.children().minus(node.meta().getName());
                 curParent = curParent.withChildren(newCurParentChildren);
                 _storage.putNode(curParent);
             }
 
-            if (effect.oldInfo().oldMeta() != null
-                    && node.meta() != null
-                    && !node.meta().getClass().equals(effect.oldInfo().oldMeta().getClass()))
+            if (!node.meta().getClass().equals(effect.oldInfo().oldMeta().getClass()))
                 throw new IllegalArgumentException("Class mismatch for meta for node " + node.key());
 
             // Needs to be read after changing curParent, as it might be the same node
             var oldParent = _storage.getById(effect.oldInfo().oldParent());
             {
-                var newOldParentChildren = oldParent.children().plus(effect.oldName(), node.key());
+                var newOldParentChildren = oldParent.children().plus(node.meta().getName(), node.key());
                 oldParent = oldParent.withChildren(newOldParentChildren);
                 _storage.putNode(oldParent);
             }
@@ -81,7 +77,7 @@ public class KleppmannTree<TimestampT extends Comparable<TimestampT>, PeerIdT ex
             var node = _storage.getById(effect.childId());
             var curParent = _storage.getById(effect.newParentId());
             {
-                var newCurParentChildren = curParent.children().minus(node.name());
+                var newCurParentChildren = curParent.children().minus(node.meta().getName());
                 curParent = curParent.withChildren(newCurParentChildren);
                 _storage.putNode(curParent);
             }
@@ -94,7 +90,6 @@ public class KleppmannTree<TimestampT extends Comparable<TimestampT>, PeerIdT ex
     }
 
     private void undoOp(LogRecord<TimestampT, PeerIdT, MetaT, NodeIdT> op) {
-        LOGGER.finer(() -> "Will undo op: " + op);
         if (op.effects() != null)
             for (var e : op.effects().reversed())
                 undoEffect(e);
@@ -145,8 +140,8 @@ public class KleppmannTree<TimestampT extends Comparable<TimestampT>, PeerIdT ex
                             }
                         }
                 }
-            }
 
+            }
             if (!inTrash.isEmpty()) {
                 var trash = _storage.getById(_storage.getTrashId());
                 for (var n : inTrash) {
@@ -171,8 +166,8 @@ public class KleppmannTree<TimestampT extends Comparable<TimestampT>, PeerIdT ex
 
     public void move(NodeIdT newParent, MetaT newMeta, NodeIdT child, boolean failCreatingIfExists) {
         var createdMove = createMove(newParent, newMeta, child);
-        applyOp(_peers.getSelfId(), createdMove, failCreatingIfExists);
         _opRecorder.recordOp(createdMove);
+        applyOp(_peers.getSelfId(), createdMove, failCreatingIfExists);
     }
 
     public void applyExternalOp(PeerIdT from, OpMove<TimestampT, PeerIdT, MetaT, NodeIdT> op) {
@@ -183,7 +178,7 @@ public class KleppmannTree<TimestampT extends Comparable<TimestampT>, PeerIdT ex
     // Returns true if the timestamp is newer than what's seen, false otherwise
     private boolean updateTimestampImpl(PeerIdT from, TimestampT newTimestamp) {
         TimestampT oldRef = _storage.getPeerTimestampLog().getForPeer(from);
-        if (oldRef != null && oldRef.compareTo(newTimestamp) >= 0) { // FIXME?
+        if (oldRef != null && oldRef.compareTo(newTimestamp) > 0) { // FIXME?
             LOGGER.warning("Wrong op order: received older than known from " + from.toString());
             return false;
         }
@@ -191,20 +186,20 @@ public class KleppmannTree<TimestampT extends Comparable<TimestampT>, PeerIdT ex
         return true;
     }
 
-    public void updateExternalTimestamp(PeerIdT from, TimestampT timestamp) {
+    public boolean updateExternalTimestamp(PeerIdT from, TimestampT timestamp) {
+        // TODO: Ideally no point in this separate locking?
         var gotExt = _storage.getPeerTimestampLog().getForPeer(from);
         var gotSelf = _storage.getPeerTimestampLog().getForPeer(_peers.getSelfId());
-        if (!(gotExt != null && gotExt.compareTo(timestamp) >= 0))
-            updateTimestampImpl(from, timestamp);
-        if (!(gotSelf != null && gotSelf.compareTo(_clock.peekTimestamp()) >= 0))
-            updateTimestampImpl(_peers.getSelfId(), _clock.peekTimestamp()); // FIXME:? Kind of a hack?
+        if ((gotExt != null && gotExt.compareTo(timestamp) >= 0)
+                && (gotSelf != null && gotSelf.compareTo(_clock.peekTimestamp()) >= 0)) return false;
+        updateTimestampImpl(_peers.getSelfId(), _clock.peekTimestamp()); // FIXME:? Kind of a hack?
+        updateTimestampImpl(from, timestamp);
         tryTrimLog();
+        return true;
     }
 
     private void applyOp(PeerIdT from, OpMove<TimestampT, PeerIdT, MetaT, NodeIdT> op, boolean failCreatingIfExists) {
-        if (!updateTimestampImpl(op.timestamp().nodeId(), op.timestamp().timestamp())) return;
-
-        LOGGER.finer(() -> "Will apply op: " + op + " from " + from);
+        if (!updateTimestampImpl(from, op.timestamp().timestamp())) return;
 
         var log = _storage.getLog();
 
@@ -257,7 +252,6 @@ public class KleppmannTree<TimestampT extends Comparable<TimestampT>, PeerIdT ex
     }
 
     private LogRecord<TimestampT, PeerIdT, MetaT, NodeIdT> doOp(OpMove<TimestampT, PeerIdT, MetaT, NodeIdT> op, boolean failCreatingIfExists) {
-        LOGGER.finer(() -> "Doing op: " + op);
         LogRecord<TimestampT, PeerIdT, MetaT, NodeIdT> computed;
         try {
             computed = computeEffects(op, failCreatingIfExists);
@@ -297,7 +291,6 @@ public class KleppmannTree<TimestampT extends Comparable<TimestampT>, PeerIdT ex
 
     private void applyEffects(OpMove<TimestampT, PeerIdT, MetaT, NodeIdT> sourceOp, List<LogEffect<TimestampT, PeerIdT, MetaT, NodeIdT>> effects) {
         for (var effect : effects) {
-            LOGGER.finer(() -> "Applying effect: " + effect + " from op " + sourceOp);
             TreeNode<TimestampT, PeerIdT, MetaT, NodeIdT> oldParentNode = null;
             TreeNode<TimestampT, PeerIdT, MetaT, NodeIdT> newParentNode;
             TreeNode<TimestampT, PeerIdT, MetaT, NodeIdT> node;
@@ -311,7 +304,7 @@ public class KleppmannTree<TimestampT extends Comparable<TimestampT>, PeerIdT ex
                 node = _storage.getById(effect.childId());
             }
             if (oldParentNode != null) {
-                var newOldParentChildren = oldParentNode.children().minus(effect.oldName());
+                var newOldParentChildren = oldParentNode.children().minus(effect.oldInfo().oldMeta().getName());
                 oldParentNode = oldParentNode.withChildren(newOldParentChildren);
                 _storage.putNode(oldParentNode);
             }
@@ -320,12 +313,12 @@ public class KleppmannTree<TimestampT extends Comparable<TimestampT>, PeerIdT ex
             newParentNode = _storage.getById(effect.newParentId());
 
             {
-                var newNewParentChildren = newParentNode.children().plus(effect.newName(), effect.childId());
+                var newNewParentChildren = newParentNode.children().plus(effect.newMeta().getName(), effect.childId());
                 newParentNode = newParentNode.withChildren(newNewParentChildren);
                 _storage.putNode(newParentNode);
             }
             if (effect.newParentId().equals(_storage.getTrashId()) &&
-                    !Objects.equals(effect.newName(), effect.childId().toString()))
+                    !Objects.equals(effect.newMeta().getName(), effect.childId().toString()))
                 throw new IllegalArgumentException("Move to trash should have id of node as name");
             _storage.putNode(
                     node.withParent(effect.newParentId())
@@ -342,32 +335,17 @@ public class KleppmannTree<TimestampT extends Comparable<TimestampT>, PeerIdT ex
         NodeIdT newParentId = op.newParentId();
         TreeNode<TimestampT, PeerIdT, MetaT, NodeIdT> newParent = _storage.getById(newParentId);
 
-
         if (newParent == null) {
-            LOGGER.log(Level.SEVERE, "New parent not found " + op.newName() + " " + op.childId());
-
-            // Creation
-            if (oldParentId == null) {
-                LOGGER.severe(() -> "Creating both dummy parent and child node");
-                return new LogRecord<>(op, List.of(
-                        new LogEffect<>(null, op, _storage.getLostFoundId(), null, newParentId),
-                        new LogEffect<>(null, op, newParentId, op.newMeta(), op.childId())
-                ));
-            } else {
-                LOGGER.severe(() -> "Moving child node to dummy parent");
-                return new LogRecord<>(op, List.of(
-                        new LogEffect<>(null, op, _storage.getLostFoundId(), null, newParentId),
-                        new LogEffect<>(new LogEffectOld<>(node.lastEffectiveOp(), oldParentId, node.meta()), op, op.newParentId(), op.newMeta(), op.childId())
-                ));
-            }
+            LOGGER.log(Level.SEVERE, "New parent not found " + op.newMeta().getName() + " " + op.childId());
+            return new LogRecord<>(op, null);
         }
 
         if (oldParentId == null) {
-            var conflictNodeId = newParent.children().get(op.newName());
+            var conflictNodeId = newParent.children().get(op.newMeta().getName());
 
             if (conflictNodeId != null) {
                 if (failCreatingIfExists)
-                    throw new AlreadyExistsException("Already exists: " + op.newName() + ": " + conflictNodeId);
+                    throw new AlreadyExistsException("Already exists: " + op.newMeta().getName() + ": " + conflictNodeId);
 
                 var conflictNode = _storage.getById(conflictNodeId);
                 MetaT conflictNodeMeta = conflictNode.meta();
@@ -376,16 +354,13 @@ public class KleppmannTree<TimestampT extends Comparable<TimestampT>, PeerIdT ex
                     return new LogRecord<>(op, null);
                 }
 
-                LOGGER.finer(() -> "Node creation conflict: " + conflictNode);
-
-                String newConflictNodeName = op.newName() + ".conflict." + conflictNode.key();
-                String newOursName = op.newName() + ".conflict." + op.childId();
+                String newConflictNodeName = conflictNodeMeta.getName() + ".conflict." + conflictNode.key();
+                String newOursName = op.newMeta().getName() + ".conflict." + op.childId();
                 return new LogRecord<>(op, List.of(
                         new LogEffect<>(new LogEffectOld<>(conflictNode.lastEffectiveOp(), newParentId, conflictNodeMeta), conflictNode.lastEffectiveOp(), newParentId, (MetaT) conflictNodeMeta.withName(newConflictNodeName), conflictNodeId),
                         new LogEffect<>(null, op, op.newParentId(), (MetaT) op.newMeta().withName(newOursName), op.childId())
                 ));
             } else {
-                LOGGER.finer(() -> "Simple node creation");
                 return new LogRecord<>(op, List.of(
                         new LogEffect<>(null, op, newParentId, op.newMeta(), op.childId())
                 ));
@@ -397,13 +372,11 @@ public class KleppmannTree<TimestampT extends Comparable<TimestampT>, PeerIdT ex
         }
 
         MetaT oldMeta = node.meta();
-        if (oldMeta != null
-                && op.newMeta() != null
-                && !oldMeta.getClass().equals(op.newMeta().getClass())) {
+        if (!oldMeta.getClass().equals(op.newMeta().getClass())) {
             LOGGER.log(Level.SEVERE, "Class mismatch for meta for node " + node.key());
             return new LogRecord<>(op, null);
         }
-        var replaceNodeId = newParent.children().get(op.newName());
+        var replaceNodeId = newParent.children().get(op.newMeta().getName());
         if (replaceNodeId != null) {
             var replaceNode = _storage.getById(replaceNodeId);
             var replaceNodeMeta = replaceNode.meta();
@@ -412,15 +385,11 @@ public class KleppmannTree<TimestampT extends Comparable<TimestampT>, PeerIdT ex
                 return new LogRecord<>(op, null);
             }
 
-            LOGGER.finer(() -> "Node replacement: " + replaceNode);
-
             return new LogRecord<>(op, List.of(
                     new LogEffect<>(new LogEffectOld<>(replaceNode.lastEffectiveOp(), newParentId, replaceNodeMeta), replaceNode.lastEffectiveOp(), _storage.getTrashId(), (MetaT) replaceNodeMeta.withName(replaceNodeId.toString()), replaceNodeId),
                     new LogEffect<>(new LogEffectOld<>(node.lastEffectiveOp(), oldParentId, oldMeta), op, op.newParentId(), op.newMeta(), op.childId())
             ));
         }
-
-        LOGGER.finer(() -> "Simple node move");
         return new LogRecord<>(op, List.of(
                 new LogEffect<>(new LogEffectOld<>(node.lastEffectiveOp(), oldParentId, oldMeta), op, op.newParentId(), op.newMeta(), op.childId())
         ));
@@ -475,18 +444,18 @@ public class KleppmannTree<TimestampT extends Comparable<TimestampT>, PeerIdT ex
         walkTree(node -> {
             var op = node.lastEffectiveOp();
             if (node.lastEffectiveOp() == null) return;
-            LOGGER.info("visited bootstrap op for " + host + ": " + op.timestamp().toString() + " " + op.newName() + " " + op.childId() + "->" + op.newParentId());
+            LOGGER.info("visited bootstrap op for " + host + ": " + op.timestamp().toString() + " " + op.newMeta().getName() + " " + op.childId() + "->" + op.newParentId());
             result.put(node.lastEffectiveOp().timestamp(), node.lastEffectiveOp());
         });
 
         for (var le : _storage.getLog().getAll()) {
             var op = le.getValue().op();
-            LOGGER.info("bootstrap op from log for " + host + ": " + op.timestamp().toString() + " " + op.newName() + " " + op.childId() + "->" + op.newParentId());
+            LOGGER.info("bootstrap op from log for " + host + ": " + op.timestamp().toString() + " " + op.newMeta().getName() + " " + op.childId() + "->" + op.newParentId());
             result.put(le.getKey(), le.getValue().op());
         }
 
         for (var op : result.values()) {
-            LOGGER.info("Recording bootstrap op for " + host + ": " + op.timestamp().toString() + " " + op.newName() + " " + op.childId() + "->" + op.newParentId());
+            LOGGER.info("Recording bootstrap op for " + host + ": " + op.timestamp().toString() + " " + op.newMeta().getName() + " " + op.childId() + "->" + op.newParentId());
             _opRecorder.recordOpForPeer(host, op);
         }
     }

dhfs-parent/kleppmanntree/src/main/java/com/usatiuk/kleppmanntree/LogEffect.java

@@ -8,17 +8,4 @@ public record LogEffect<TimestampT extends Comparable<TimestampT>, PeerIdT exten
         NodeIdT newParentId,
         MetaT newMeta,
         NodeIdT childId) implements Serializable {
-    public String oldName() {
-        if (oldInfo.oldMeta() != null) {
-            return oldInfo.oldMeta().getName();
-        }
-        return childId.toString();
-    }
-
-    public String newName() {
-        if (newMeta != null) {
-            return newMeta.getName();
-        }
-        return childId.toString();
-    }
 }

dhfs-parent/kleppmanntree/src/main/java/com/usatiuk/kleppmanntree/OpMove.java

@@ -5,9 +5,4 @@ import java.io.Serializable;
 public record OpMove<TimestampT extends Comparable<TimestampT>, PeerIdT extends Comparable<PeerIdT>, MetaT extends NodeMeta, NodeIdT>
         (CombinedTimestamp<TimestampT, PeerIdT> timestamp, NodeIdT newParentId, MetaT newMeta,
          NodeIdT childId) implements Serializable {
-    public String newName() {
-        if (newMeta != null)
-            return newMeta.getName();
-        return childId.toString();
-    }
 }

dhfs-parent/kleppmanntree/src/main/java/com/usatiuk/kleppmanntree/StorageInterface.java

@@ -9,8 +9,6 @@ public interface StorageInterface<
 
     NodeIdT getTrashId();
 
-    NodeIdT getLostFoundId();
-
     NodeIdT getNewNodeId();
 
     TreeNode<TimestampT, PeerIdT, MetaT, NodeIdT> getById(NodeIdT id);

dhfs-parent/kleppmanntree/src/main/java/com/usatiuk/kleppmanntree/TreeNode.java

@@ -1,9 +1,9 @@
 package com.usatiuk.kleppmanntree;
 
-import jakarta.annotation.Nullable;
 import org.pcollections.PMap;
 
 import java.io.Serializable;
+import java.util.Map;
 
 public interface TreeNode<TimestampT extends Comparable<TimestampT>, PeerIdT extends Comparable<PeerIdT>, MetaT extends NodeMeta, NodeIdT> extends Serializable {
     NodeIdT key();
@@ -12,15 +12,8 @@ public interface TreeNode<TimestampT extends Comparable<TimestampT>, PeerIdT ext
 
     OpMove<TimestampT, PeerIdT, MetaT, NodeIdT> lastEffectiveOp();
 
-    @Nullable
     MetaT meta();
 
-    default String name() {
-        var meta = meta();
-        if (meta != null) return meta.getName();
-        return key().toString();
-    }
-
     PMap<String, NodeIdT> children();
 
     TreeNode<TimestampT, PeerIdT, MetaT, NodeIdT> withParent(NodeIdT parent);

dhfs-parent/kleppmanntree/src/test/java/com/usatiuk/kleppmanntree/KleppmanTreeSimpleTest.java

@@ -2,15 +2,13 @@ package com.usatiuk.kleppmanntree;
 
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.ValueSource;
 
 import java.util.List;
 
 public class KleppmanTreeSimpleTest {
     private final TestNode testNode1 = new TestNode(1);
     private final TestNode testNode2 = new TestNode(2);
-    private final TestNode testNode3 = new TestNode(3);
+
 
     @Test
     void circularTest() {
@@ -91,75 +89,4 @@ public class KleppmanTreeSimpleTest {
         Assertions.assertTrue(testNode2._storageInterface.getLog().size() <= 1);
     }
 
-    @ParameterizedTest
-    @ValueSource(booleans = {true, false})
-    void undoWithRenameTest(boolean opOrder) {
-        var d1id = testNode1._storageInterface.getNewNodeId();
-        var d2id = testNode2._storageInterface.getNewNodeId();
-        var d3id = testNode3._storageInterface.getNewNodeId();
-        testNode1._tree.move(testNode1._storageInterface.getRootId(), new TestNodeMetaDir("Test1"), d1id);
-        testNode2._tree.move(testNode1._storageInterface.getRootId(), new TestNodeMetaDir("Test1"), d2id);
-        testNode3._tree.move(testNode1._storageInterface.getRootId(), new TestNodeMetaDir("Test1"), d3id);
-        var r1 = testNode1.getRecorded();
-        var r2 = testNode2.getRecorded();
-        var r3 = testNode3.getRecorded();
-        Assertions.assertEquals(1, r1.size());
-        Assertions.assertEquals(1, r2.size());
-        Assertions.assertEquals(1, r3.size());
-
-        if (opOrder) {
-            testNode2._tree.applyExternalOp(3L, r3.getFirst());
-            testNode2._tree.applyExternalOp(1L, r1.getFirst());
-        } else {
-            testNode2._tree.applyExternalOp(1L, r1.getFirst());
-            testNode2._tree.applyExternalOp(3L, r3.getFirst());
-        }
-
-        Assertions.assertIterableEquals(List.of("Test1", "Test1.conflict." + d1id, "Test1.conflict." + d2id), testNode2._storageInterface.getById(testNode2._storageInterface.getRootId()).children().keySet());
-
-        if (opOrder) {
-            testNode1._tree.applyExternalOp(3L, r3.getFirst());
-            testNode1._tree.applyExternalOp(2L, r2.getFirst());
-        } else {
-            testNode1._tree.applyExternalOp(2L, r2.getFirst());
-            testNode1._tree.applyExternalOp(3L, r3.getFirst());
-        }
-
-        Assertions.assertIterableEquals(List.of("Test1", "Test1.conflict." + d1id, "Test1.conflict." + d2id), testNode1._storageInterface.getById(testNode1._storageInterface.getRootId()).children().keySet());
-
-        if (opOrder) {
-            testNode3._tree.applyExternalOp(2L, r2.getFirst());
-            testNode3._tree.applyExternalOp(1L, r1.getFirst());
-        } else {
-            testNode3._tree.applyExternalOp(1L, r1.getFirst());
-            testNode3._tree.applyExternalOp(2L, r2.getFirst());
-        }
-
-        Assertions.assertIterableEquals(List.of("Test1", "Test1.conflict." + d1id, "Test1.conflict." + d2id), testNode3._storageInterface.getById(testNode3._storageInterface.getRootId()).children().keySet());
-    }
-
-    @Test
-    void noFailedOpRecordTest() {
-        var d1id = testNode1._storageInterface.getNewNodeId();
-        var d2id = testNode1._storageInterface.getNewNodeId();
-        testNode1._tree.move(testNode1._storageInterface.getRootId(), new TestNodeMetaDir("Test1"), d1id);
-        Assertions.assertThrows(AlreadyExistsException.class, () -> testNode1._tree.move(testNode1._storageInterface.getRootId(), new TestNodeMetaDir("Test1"), d2id));
-        var r1 = testNode1.getRecorded();
-        Assertions.assertEquals(1, r1.size());
-    }
-
-    @Test
-    void externalOpWithDummy() {
-        Long d1id = testNode1._storageInterface.getNewNodeId();
-        Long f1id = testNode1._storageInterface.getNewNodeId();
-
-        testNode1._tree.applyExternalOp(2L, new OpMove<>(
-                new CombinedTimestamp<>(2L, 2L), d1id, new TestNodeMetaFile("Hi", 123), f1id
-        ));
-        testNode1._tree.applyExternalOp(2L, new OpMove<>(
-                new CombinedTimestamp<>(3L, 2L), testNode1._storageInterface.getRootId(), new TestNodeMetaDir("HiDir"), d1id
-        ));
-
-        Assertions.assertEquals(f1id, testNode1._tree.traverse(List.of("HiDir", "Hi")));
-    }
 }

dhfs-parent/kleppmanntree/src/test/java/com/usatiuk/kleppmanntree/TestStorageInterface.java

@@ -14,7 +14,6 @@ public class TestStorageInterface implements StorageInterface<Long, Long, TestNo
         _peerId = peerId;
         _nodes.put(getRootId(), new TestTreeNode(getRootId(), null, null));
         _nodes.put(getTrashId(), new TestTreeNode(getTrashId(), null, null));
-        _nodes.put(getLostFoundId(), new TestTreeNode(getLostFoundId(), null, null));
     }
 
     @Override
@@ -27,11 +26,6 @@ public class TestStorageInterface implements StorageInterface<Long, Long, TestNo
         return -1L;
     }
 
-    @Override
-    public Long getLostFoundId() {
-        return -2L;
-    }
-
     @Override
     public Long getNewNodeId() {
         return _curId++ | _peerId << 32;

dhfs-parent/objects/pom.xml

@@ -88,11 +88,6 @@
                     <forkCount>1C</forkCount>
                     <reuseForks>false</reuseForks>
                     <parallel>classes</parallel>
-                    <systemPropertyVariables>
-                        <junit.jupiter.execution.parallel.enabled>
-                            false
-                        </junit.jupiter.execution.parallel.enabled>
-                    </systemPropertyVariables>
                 </configuration>
             </plugin>
             <plugin>
@@ -104,6 +99,7 @@
                     <execution>
                         <id>quarkus-plugin</id>
                         <goals>
+                            <goal>build</goal>
                             <goal>generate-code</goal>
                             <goal>generate-code-tests</goal>
                         </goals>

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/CloseableKvIterator.java

@@ -1,17 +1,21 @@
-package com.usatiuk.objects.iterators;
+package com.usatiuk.dhfs.objects;
 
 import com.usatiuk.dhfs.utils.AutoCloseableNoThrow;
 import org.apache.commons.lang3.tuple.Pair;
 
 import java.util.Iterator;
 
-public interface CloseableKvIterator<K extends Comparable<? super K>, V> extends Iterator<Pair<K, V>>, AutoCloseableNoThrow {
+public interface CloseableKvIterator<K extends Comparable<K>, V> extends Iterator<Pair<K, V>>, AutoCloseableNoThrow {
     K peekNextKey();
 
+    Class<?> peekNextType();
+
     void skip();
 
     K peekPrevKey();
 
+    Class<?> peekPrevType();
+
     Pair<K, V> prev();
 
     boolean hasPrev();
@@ -19,6 +23,6 @@ public interface CloseableKvIterator<K extends Comparable<? super K>, V> extends
     void skipPrev();
 
     default CloseableKvIterator<K, V> reversed() {
-        return new ReversedKvIterator<K, V>(this);
+        return new ReversedKvIterator<>(this);
     }
 }

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/CurrentTransaction.java

@@ -1,16 +1,18 @@
-package com.usatiuk.objects.transaction;
+package com.usatiuk.dhfs.objects;
 
-import com.usatiuk.objects.JData;
-import com.usatiuk.objects.JObjectKey;
-import com.usatiuk.objects.iterators.CloseableKvIterator;
-import com.usatiuk.objects.iterators.IteratorStart;
+import com.usatiuk.dhfs.objects.persistence.IteratorStart;
+import com.usatiuk.dhfs.objects.transaction.LockingStrategy;
+import com.usatiuk.dhfs.objects.transaction.Transaction;
 import jakarta.enterprise.context.ApplicationScoped;
 import jakarta.inject.Inject;
-import jakarta.inject.Singleton;
+import org.apache.commons.lang3.tuple.Pair;
 
+import javax.annotation.Nonnull;
+import java.util.Collection;
+import java.util.Iterator;
 import java.util.Optional;
 
-@Singleton
+@ApplicationScoped
 public class CurrentTransaction implements Transaction {
     @Inject
     TransactionManager transactionManager;
@@ -35,6 +37,12 @@ public class CurrentTransaction implements Transaction {
         transactionManager.current().delete(key);
     }
 
+    @Nonnull
+    @Override
+    public Collection<JObjectKey> findAllObjects() {
+        return transactionManager.current().findAllObjects();
+    }
+
     @Override
     public CloseableKvIterator<JObjectKey, JData> getIterator(IteratorStart start, JObjectKey key) {
         return transactionManager.current().getIterator(start, key);

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/Data.java

@@ -1,4 +1,4 @@
-package com.usatiuk.objects.iterators;
+package com.usatiuk.dhfs.objects;
 
 import java.util.Optional;
 

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/IterProdFn.java

@@ -1,4 +1,6 @@
-package com.usatiuk.objects.iterators;
+package com.usatiuk.dhfs.objects;
+
+import com.usatiuk.dhfs.objects.persistence.IteratorStart;
 
 @FunctionalInterface
 public interface IterProdFn<K extends Comparable<K>, V> {

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/JData.java

@@ -0,0 +1,16 @@
+package com.usatiuk.dhfs.objects;
+
+import java.io.Serializable;
+
+// TODO: This could be maybe moved to a separate module?
+// The base class for JObject data
+// Only one instance of this "exists" per key, the instance in the manager is canonical
+// When committing a transaction, the instance is checked against it, if it isn't the same, a race occurred.
+// It is immutable, its version is filled in by the allocator from the AllocVersionProvider
+public interface JData extends Serializable {
+    JObjectKey key();
+
+    default int estimateSize() {
+        return 100;
+    }
+}

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/JDataVersionedWrapper.java

@@ -1,4 +1,4 @@
-package com.usatiuk.objects;
+package com.usatiuk.dhfs.objects;
 
 public interface JDataVersionedWrapper {
     JData data();

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/JDataVersionedWrapperImpl.java

@@ -1,4 +1,4 @@
-package com.usatiuk.objects;
+package com.usatiuk.dhfs.objects;
 
 import jakarta.annotation.Nonnull;
 

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/JDataVersionedWrapperLazy.java

@@ -0,0 +1,44 @@
+package com.usatiuk.dhfs.objects;
+
+import com.google.protobuf.ByteString;
+import com.usatiuk.dhfs.utils.SerializationHelper;
+
+public class JDataVersionedWrapperLazy implements JDataVersionedWrapper {
+    private final long _version;
+    private ByteString _rawData;
+    private JData _data;
+
+    public JDataVersionedWrapperLazy(long version, ByteString rawData) {
+        _version = version;
+        _rawData = rawData;
+    }
+
+    public JData data() {
+        if (_data != null)
+            return _data;
+
+        synchronized (this) {
+            if (_data != null)
+                return _data;
+
+            try (var is = _rawData.newInput()) {
+                _data = SerializationHelper.deserialize(is);
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+            _rawData = null;
+            return _data;
+        }
+    }
+
+    public long version() {
+        return _version;
+    }
+
+    @Override
+    public int estimateSize() {
+        if (_data != null)
+            return _data.estimateSize();
+        return _rawData.size();
+    }
+}

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/JObjectKey.java

@@ -0,0 +1,44 @@
+package com.usatiuk.dhfs.objects;
+
+import com.usatiuk.dhfs.supportlib.UninitializedByteBuffer;
+
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+
+public record JObjectKey(String name) implements Serializable, Comparable<JObjectKey> {
+    public static JObjectKey of(String name) {
+        return new JObjectKey(name);
+    }
+
+    @Override
+    public int compareTo(JObjectKey o) {
+        return name.compareTo(o.name);
+    }
+
+    @Override
+    public String toString() {
+        return name;
+    }
+
+    public byte[] bytes() {
+        return name.getBytes(StandardCharsets.UTF_8);
+    }
+
+    public ByteBuffer toByteBuffer() {
+        var heapBb = StandardCharsets.UTF_8.encode(name);
+        if (heapBb.isDirect()) return heapBb;
+        var directBb = UninitializedByteBuffer.allocateUninitialized(heapBb.remaining());
+        directBb.put(heapBb);
+        directBb.flip();
+        return directBb;
+    }
+
+    public static JObjectKey fromBytes(byte[] bytes) {
+        return new JObjectKey(new String(bytes, StandardCharsets.UTF_8));
+    }
+
+    public static JObjectKey fromByteBuffer(ByteBuffer buff) {
+        return new JObjectKey(StandardCharsets.UTF_8.decode(buff).toString());
+    }
+}

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/JObjectManager.java

@@ -0,0 +1,239 @@
+package com.usatiuk.dhfs.objects;
+
+import com.usatiuk.dhfs.objects.snapshot.SnapshotManager;
+import com.usatiuk.dhfs.objects.transaction.*;
+import com.usatiuk.dhfs.utils.AutoCloseableNoThrow;
+import io.quarkus.logging.Log;
+import io.quarkus.runtime.StartupEvent;
+import jakarta.annotation.Priority;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.enterprise.event.Observes;
+import jakarta.enterprise.inject.Instance;
+import jakarta.inject.Inject;
+
+import java.util.*;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.stream.Stream;
+
+// Manages all access to com.usatiuk.dhfs.objects.JData objects.
+// In particular, it serves as a source of truth for what is committed to the backing storage.
+// All data goes through it, it is responsible for transaction atomicity
+// TODO: persistent tx id
+@ApplicationScoped
+public class JObjectManager {
+    private final List<PreCommitTxHook> _preCommitTxHooks;
+    private boolean _ready = false;
+    @Inject
+    SnapshotManager snapshotManager;
+    @Inject
+    TransactionFactory transactionFactory;
+    @Inject
+    LockManager lockManager;
+
+    private void verifyReady() {
+        if (!_ready) throw new IllegalStateException("Wrong service order!");
+    }
+
+    void init(@Observes @Priority(200) StartupEvent event) {
+        _ready = true;
+    }
+
+    JObjectManager(Instance<PreCommitTxHook> preCommitTxHooks) {
+        _preCommitTxHooks = preCommitTxHooks.stream().sorted(Comparator.comparingInt(PreCommitTxHook::getPriority)).toList();
+    }
+
+    public TransactionPrivate createTransaction() {
+        verifyReady();
+        var tx = transactionFactory.createTransaction();
+        Log.tracev("Created transaction with snapshotId={0}", tx.snapshot().id());
+        return tx;
+    }
+
+    public TransactionHandle commit(TransactionPrivate tx) {
+        verifyReady();
+        var writes = new LinkedHashMap<JObjectKey, TxRecord.TxObjectRecord<?>>();
+        var dependenciesLocked = new LinkedHashMap<JObjectKey, Optional<JDataVersionedWrapper>>();
+        Map<JObjectKey, TransactionObject<?>> readSet;
+        var toUnlock = new ArrayList<AutoCloseableNoThrow>();
+
+        Consumer<JObjectKey> addDependency =
+                key -> {
+                    dependenciesLocked.computeIfAbsent(key, k -> {
+                        var lock = lockManager.lockObject(k);
+                        toUnlock.add(lock);
+                        return snapshotManager.readObjectDirect(k);
+                    });
+                };
+
+        // For existing objects:
+        // Check that their version is not higher than the version of transaction being committed
+        // TODO: check deletions, inserts
+        try {
+            try {
+                Function<JObjectKey, JData> getPrev =
+                        key -> switch (writes.get(key)) {
+                            case TxRecord.TxObjectRecordWrite<?> write -> write.data();
+                            case TxRecord.TxObjectRecordDeleted deleted -> null;
+                            case null -> tx.getFromSource(JData.class, key).orElse(null);
+                            default -> {
+                                throw new TxCommitException("Unexpected value: " + writes.get(key));
+                            }
+                        };
+
+                boolean somethingChanged;
+                do {
+                    somethingChanged = false;
+                    Map<JObjectKey, TxRecord.TxObjectRecord<?>> currentIteration = new HashMap();
+                    for (var hook : _preCommitTxHooks) {
+                        for (var n : tx.drainNewWrites())
+                            currentIteration.put(n.key(), n);
+                        Log.trace("Commit iteration with " + currentIteration.size() + " records for hook " + hook.getClass());
+
+                        for (var entry : currentIteration.entrySet()) {
+                            somethingChanged = true;
+                            Log.trace("Running pre-commit hook " + hook.getClass() + " for" + entry.getKey());
+                            var oldObj = getPrev.apply(entry.getKey());
+                            switch (entry.getValue()) {
+                                case TxRecord.TxObjectRecordWrite<?> write -> {
+                                    if (oldObj == null) {
+                                        hook.onCreate(write.key(), write.data());
+                                    } else {
+                                        hook.onChange(write.key(), oldObj, write.data());
+                                    }
+                                }
+                                case TxRecord.TxObjectRecordDeleted deleted -> {
+                                    hook.onDelete(deleted.key(), oldObj);
+                                }
+                                default -> throw new TxCommitException("Unexpected value: " + entry);
+                            }
+                        }
+                    }
+                    writes.putAll(currentIteration);
+                } while (somethingChanged);
+
+                if (writes.isEmpty()) {
+                    Log.trace("Committing transaction - no changes");
+                    return new TransactionHandle() {
+                        @Override
+                        public void onFlush(Runnable runnable) {
+                            runnable.run();
+                        }
+                    };
+                }
+
+            } finally {
+                readSet = tx.reads();
+
+                Stream.concat(readSet.keySet().stream(), writes.keySet().stream())
+                        .sorted(Comparator.comparing(JObjectKey::toString))
+                        .forEach(addDependency);
+
+                for (var read : readSet.entrySet()) {
+                    if (read.getValue() instanceof TransactionObjectLocked<?> locked) {
+                        toUnlock.add(locked.lock());
+                    }
+                }
+            }
+
+            Log.trace("Committing transaction start");
+            var snapshotId = tx.snapshot().id();
+
+            for (var read : readSet.entrySet()) {
+                var dep = dependenciesLocked.get(read.getKey());
+
+                if (dep.isEmpty() != read.getValue().data().isEmpty()) {
+                    Log.trace("Checking read dependency " + read.getKey() + " - not found");
+                    throw new TxCommitException("Serialization hazard: " + dep.isEmpty() + " vs " + read.getValue().data().isEmpty());
+                }
+
+                if (dep.isEmpty()) {
+                    // TODO: Every write gets a dependency due to hooks
+                    continue;
+//                    assert false;
+//                    throw new TxCommitException("Serialization hazard: " + dep.isEmpty() + " vs " + read.getValue().data().isEmpty());
+                }
+
+                if (dep.get().version() > snapshotId) {
+                    Log.trace("Checking dependency " + read.getKey() + " - newer than");
+                    throw new TxCommitException("Serialization hazard: " + dep.get().data().key() + " " + dep.get().version() + " vs " + snapshotId);
+                }
+
+                Log.trace("Checking dependency " + read.getKey() + " - ok with read");
+            }
+
+            var addFlushCallback = snapshotManager.commitTx(
+                    writes.values().stream()
+                            .filter(r -> {
+                                if (r instanceof TxRecord.TxObjectRecordWrite<?>(JData data)) {
+                                    var dep = dependenciesLocked.get(data.key());
+                                    if (dep.isPresent() && dep.get().version() > snapshotId) {
+                                        Log.trace("Skipping write " + data.key() + " - dependency " + dep.get().version() + " vs " + snapshotId);
+                                        return false;
+                                    }
+                                }
+                                return true;
+                            }).toList());
+
+            for (var callback : tx.getOnCommit()) {
+                callback.run();
+            }
+
+            for (var callback : tx.getOnFlush()) {
+                addFlushCallback.accept(callback);
+            }
+
+            return new TransactionHandle() {
+                @Override
+                public void onFlush(Runnable runnable) {
+                    addFlushCallback.accept(runnable);
+                }
+            };
+        } catch (Throwable t) {
+            Log.trace("Error when committing transaction", t);
+            throw new TxCommitException(t.getMessage(), t);
+        } finally {
+            for (var unlock : toUnlock) {
+                unlock.close();
+            }
+            tx.close();
+        }
+    }
+
+    public void rollback(TransactionPrivate tx) {
+        verifyReady();
+        tx.reads().forEach((key, value) -> {
+            if (value instanceof TransactionObjectLocked<?> locked) {
+                locked.lock().close();
+            }
+        });
+        tx.close();
+    }
+
+    //    private class TransactionObjectSourceImpl implements TransactionObjectSource {
+//        private final long _txId;
+//
+//        private TransactionObjectSourceImpl(long txId) {
+//            _txId = txId;
+//        }
+//
+//        @Override
+//        public <T extends JData> TransactionObject<T> get(Class<T> type, JObjectKey key) {
+//            var got = getObj(type, key);
+//            if (got.data().isPresent() && got.data().get().version() > _txId) {
+//                throw new TxCommitException("Serialization race for " + key + ": " + got.data().get().version() + " vs " + _txId);
+//            }
+//            return got;
+//        }
+//
+//        @Override
+//        public <T extends JData> TransactionObject<T> getWriteLocked(Class<T> type, JObjectKey key) {
+//            var got = getObjLock(type, key);
+//            if (got.data().isPresent() && got.data().get().version() > _txId) {
+//                got.lock().close();
+//                throw new TxCommitException("Serialization race for " + key + ": " + got.data().get().version() + " vs " + _txId);
+//            }
+//            return got;
+//        }
+//    }
+}

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/JavaDataSerializer.java

@@ -1,29 +1,26 @@
-package com.usatiuk.objects;
+package com.usatiuk.dhfs.objects;
 
 
 import com.google.protobuf.ByteString;
+import com.usatiuk.dhfs.utils.SerializationHelper;
 import jakarta.enterprise.context.ApplicationScoped;
-import jakarta.inject.Inject;
 
 import java.nio.ByteBuffer;
 
 @ApplicationScoped
-public class JDataVersionedWrapperSerializer implements ObjectSerializer<JDataVersionedWrapper> {
-    @Inject
-    ObjectSerializer<JData> dataSerializer;
-
+public class JavaDataSerializer implements ObjectSerializer<JDataVersionedWrapper> {
     @Override
     public ByteString serialize(JDataVersionedWrapper obj) {
         ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES);
         buffer.putLong(obj.version());
         buffer.flip();
-        return ByteString.copyFrom(buffer).concat(dataSerializer.serialize(obj.data()));
+        return ByteString.copyFrom(buffer).concat(SerializationHelper.serialize(obj.data()));
     }
 
     @Override
     public JDataVersionedWrapper deserialize(ByteString data) {
         var version = data.substring(0, Long.BYTES).asReadOnlyByteBuffer().getLong();
         var rawData = data.substring(Long.BYTES);
-        return new JDataVersionedWrapperLazy(version, rawData.size(), () -> dataSerializer.deserialize(rawData));
+        return new JDataVersionedWrapperLazy(version, rawData);
     }
 }

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/KeyPredicateKvIterator.java

@@ -1,5 +1,6 @@
-package com.usatiuk.objects.iterators;
+package com.usatiuk.dhfs.objects;
 
+import com.usatiuk.dhfs.objects.persistence.IteratorStart;
 import org.apache.commons.lang3.tuple.Pair;
 
 import java.util.NoSuchElementException;
@@ -113,6 +114,11 @@ public class KeyPredicateKvIterator<K extends Comparable<K>, V> extends Reversib
         return got;
     }
 
+    @Override
+    protected Class<? > peekTypeImpl() {
+        return _goingForward ? _backing.peekNextType() : _backing.peekPrevType();
+    }
+
     @Override
     public void close() {
         _backing.close();

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/LockManager.java

@@ -1,23 +1,14 @@
-package com.usatiuk.objects.transaction;
+package com.usatiuk.dhfs.objects;
 
-import com.usatiuk.objects.JObjectKey;
 import com.usatiuk.dhfs.utils.AutoCloseableNoThrow;
 import com.usatiuk.dhfs.utils.DataLocker;
-import jakarta.annotation.Nonnull;
-import jakarta.annotation.Nullable;
 import jakarta.enterprise.context.ApplicationScoped;
 
 @ApplicationScoped
 public class LockManager {
     private final DataLocker _objLocker = new DataLocker();
 
-    @Nonnull
     public AutoCloseableNoThrow lockObject(JObjectKey key) {
         return _objLocker.lock(key);
     }
-
-    @Nullable
-    public AutoCloseableNoThrow tryLockObject(JObjectKey key) {
-        return _objLocker.tryLock(key);
-    }
 }

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/MappingKvIterator.java

@@ -1,16 +1,19 @@
-package com.usatiuk.objects.iterators;
+package com.usatiuk.dhfs.objects;
 
 import org.apache.commons.lang3.tuple.Pair;
 
+import java.util.NoSuchElementException;
 import java.util.function.Function;
 
 public class MappingKvIterator<K extends Comparable<K>, V, V_T> implements CloseableKvIterator<K, V_T> {
     private final CloseableKvIterator<K, V> _backing;
     private final Function<V, V_T> _transformer;
+    private final Function<Class<?>, Class<?>> _classMapper;
 
-    public MappingKvIterator(CloseableKvIterator<K, V> backing, Function<V, V_T> transformer) {
+    public MappingKvIterator(CloseableKvIterator<K, V> backing, Function<V, V_T> transformer, Function<Class<?>, Class<?>> classMapper) {
         _backing = backing;
         _transformer = transformer;
+        _classMapper = classMapper;
     }
 
     @Override
@@ -18,6 +21,13 @@ public class MappingKvIterator<K extends Comparable<K>, V, V_T> implements Close
         return _backing.peekNextKey();
     }
 
+    @Override
+    public Class<?> peekNextType() {
+        if (!hasNext())
+            throw new NoSuchElementException();
+        return _classMapper.apply(_backing.peekNextType());
+    }
+
     @Override
     public void skip() {
         _backing.skip();
@@ -38,6 +48,13 @@ public class MappingKvIterator<K extends Comparable<K>, V, V_T> implements Close
         return _backing.peekPrevKey();
     }
 
+    @Override
+    public Class<?> peekPrevType() {
+        if (!hasPrev())
+            throw new NoSuchElementException();
+        return _classMapper.apply(_backing.peekPrevType());
+    }
+
     @Override
     public Pair<K, V_T> prev() {
         var got = _backing.prev();

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/MaybeTombstone.java

@@ -1,4 +1,4 @@
-package com.usatiuk.objects.iterators;
+package com.usatiuk.dhfs.objects;
 
 import java.util.Optional;
 

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/MergingKvIterator.java

@@ -0,0 +1,333 @@
+package com.usatiuk.dhfs.objects;
+
+import com.usatiuk.dhfs.objects.persistence.IteratorStart;
+import io.quarkus.logging.Log;
+import org.apache.commons.lang3.tuple.Pair;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+public class MergingKvIterator<K extends Comparable<K>, V> extends ReversibleKvIterator<K, V> {
+    private final NavigableMap<K, CloseableKvIterator<K, V>> _sortedIterators = new TreeMap<>();
+    private final String _name;
+    private Map<CloseableKvIterator<K, V>, Integer> _iterators;
+
+    private final IteratorStart _initialStartType;
+    private final K _initialStartKey;
+
+    private interface FirstMatchState<K extends Comparable<K>, V> {
+    }
+
+    private record FirstMatchNone<K extends Comparable<K>, V>() implements FirstMatchState<K, V> {
+    }
+
+    private record FirstMatchFound<K extends Comparable<K>, V>(
+            CloseableKvIterator<K, V> iterator) implements FirstMatchState<K, V> {
+    }
+
+    private record FirstMatchConsumed<K extends Comparable<K>, V>() implements FirstMatchState<K, V> {
+    }
+
+    // Fast path for the first element
+    private FirstMatchState<K, V> _firstMatchState;
+    private final List<IterProdFn<K, V>> _pendingIterators;
+
+    public MergingKvIterator(String name, IteratorStart startType, K startKey, List<IterProdFn<K, V>> iterators) {
+        _goingForward = true;
+        _name = name;
+        _initialStartType = startType;
+        _initialStartKey = startKey;
+
+        {
+            int counter = 0;
+            var iteratorsTmp = new HashMap<CloseableKvIterator<K, V>, Integer>();
+            for (var iteratorFn : iterators) {
+                var iterator = iteratorFn.get(startType, startKey);
+                if ((counter == 0) // Not really a requirement but simplifies some things for now
+                        && (startType == IteratorStart.GE || startType == IteratorStart.LE)
+                        && iterator.hasNext()
+                        && iterator.peekNextKey().equals(startKey)) {
+                    _firstMatchState = new FirstMatchFound<>(iterator);
+                    _pendingIterators = iterators;
+                    Log.tracev("{0} Created fast match: {1}", _name, _firstMatchState);
+                    return;
+                }
+                iteratorsTmp.put(iterator, counter++);
+            }
+            _iterators = Map.copyOf(iteratorsTmp);
+            _pendingIterators = null;
+        }
+
+        _firstMatchState = new FirstMatchNone<>();
+        doInitialAdvance();
+    }
+
+    private void doInitialAdvance() {
+        if (_initialStartType == IteratorStart.LT || _initialStartType == IteratorStart.LE) {
+            // Starting at a greatest key less than/less or equal than:
+            // We have a bunch of iterators that have given us theirs "greatest LT/LE key"
+            // now we need to pick the greatest of those to start with
+            // But if some of them don't have a lesser key, we need to pick the smallest of those
+            var found = _iterators.keySet().stream()
+                    .filter(CloseableKvIterator::hasNext)
+                    .map((i) -> {
+                        var peeked = i.peekNextKey();
+//                            Log.warnv("peeked: {0}, from {1}", peeked, i.getClass());
+                        return peeked;
+                    }).distinct().collect(Collectors.partitioningBy(e -> _initialStartType == IteratorStart.LE ? e.compareTo(_initialStartKey) <= 0 : e.compareTo(_initialStartKey) < 0));
+            K initialMaxValue;
+            if (!found.get(true).isEmpty())
+                initialMaxValue = found.get(true).stream().max(Comparator.naturalOrder()).orElse(null);
+            else
+                initialMaxValue = found.get(false).stream().min(Comparator.naturalOrder()).orElse(null);
+
+            for (var iterator : _iterators.keySet()) {
+                while (iterator.hasNext() && iterator.peekNextKey().compareTo(initialMaxValue) < 0) {
+                    iterator.skip();
+                }
+            }
+        }
+
+        for (CloseableKvIterator<K, V> iterator : _iterators.keySet()) {
+            advanceIterator(iterator);
+        }
+
+        Log.tracev("{0} Initialized: {1}", _name, _sortedIterators);
+        switch (_initialStartType) {
+//            case LT -> {
+//                assert _sortedIterators.isEmpty() || _sortedIterators.firstKey().compareTo(initialStartKey) < 0;
+//            }
+//            case LE -> {
+//                assert _sortedIterators.isEmpty() || _sortedIterators.firstKey().compareTo(initialStartKey) <= 0;
+//            }
+            case GT -> {
+                assert _sortedIterators.isEmpty() || _sortedIterators.firstKey().compareTo(_initialStartKey) > 0;
+            }
+            case GE -> {
+                assert _sortedIterators.isEmpty() || _sortedIterators.firstKey().compareTo(_initialStartKey) >= 0;
+            }
+        }
+    }
+
+    private void doHydrate() {
+        if (_firstMatchState instanceof FirstMatchNone) {
+            return;
+        }
+
+        boolean consumed = _firstMatchState instanceof FirstMatchConsumed;
+        if (_firstMatchState instanceof FirstMatchFound(CloseableKvIterator iterator)) {
+            iterator.close();
+        }
+
+        _firstMatchState = new FirstMatchNone<>();
+
+        {
+            int counter = 0;
+            var iteratorsTmp = new HashMap<CloseableKvIterator<K, V>, Integer>();
+            for (var iteratorFn : _pendingIterators) {
+                var iterator = iteratorFn.get(consumed ? IteratorStart.GT : IteratorStart.GE, _initialStartKey);
+                iteratorsTmp.put(iterator, counter++);
+            }
+            _iterators = Map.copyOf(iteratorsTmp);
+        }
+
+        doInitialAdvance();
+    }
+
+    @SafeVarargs
+    public MergingKvIterator(String name, IteratorStart startType, K startKey, IterProdFn<K, V>... iterators) {
+        this(name, startType, startKey, List.of(iterators));
+    }
+
+    private void advanceIterator(CloseableKvIterator<K, V> iterator) {
+        if (!iterator.hasNext()) {
+            return;
+        }
+
+        K key = iterator.peekNextKey();
+        Log.tracev("{0} Advance peeked: {1}-{2}", _name, iterator, key);
+        if (!_sortedIterators.containsKey(key)) {
+            _sortedIterators.put(key, iterator);
+            return;
+        }
+
+        // Expects that reversed iterator returns itself when reversed again
+        var oursPrio = _iterators.get(_goingForward ? iterator : iterator.reversed());
+        var them = _sortedIterators.get(key);
+        var theirsPrio = _iterators.get(_goingForward ? them : them.reversed());
+        if (oursPrio < theirsPrio) {
+            _sortedIterators.put(key, iterator);
+            advanceIterator(them);
+        } else {
+            Log.tracev("{0} Skipped: {1}", _name, iterator.peekNextKey());
+            iterator.skip();
+            advanceIterator(iterator);
+        }
+    }
+
+    @Override
+    protected void reverse() {
+        switch (_firstMatchState) {
+            case FirstMatchFound<K, V> firstMatchFound -> {
+                doHydrate();
+            }
+            case FirstMatchConsumed<K, V> firstMatchConsumed -> {
+                doHydrate();
+            }
+            default -> {
+            }
+        }
+
+        var cur = _goingForward ? _sortedIterators.pollFirstEntry() : _sortedIterators.pollLastEntry();
+        Log.tracev("{0} Reversing from {1}", _name, cur);
+        _goingForward = !_goingForward;
+        _sortedIterators.clear();
+        for (CloseableKvIterator<K, V> iterator : _iterators.keySet()) {
+            // _goingForward inverted already
+            advanceIterator(!_goingForward ? iterator.reversed() : iterator);
+        }
+        if (_sortedIterators.isEmpty() || cur == null) {
+            return;
+        }
+        // Advance to the expected key, as we might have brought back some iterators
+        // that were at their ends
+        while (!_sortedIterators.isEmpty()
+                && ((_goingForward && peekImpl().compareTo(cur.getKey()) <= 0)
+                || (!_goingForward && peekImpl().compareTo(cur.getKey()) >= 0))) {
+            skipImpl();
+        }
+        Log.tracev("{0} Reversed to {1}", _name, _sortedIterators);
+    }
+
+    @Override
+    protected K peekImpl() {
+        switch (_firstMatchState) {
+            case FirstMatchFound<K, V> firstMatchFound -> {
+                return firstMatchFound.iterator.peekNextKey();
+            }
+            case FirstMatchConsumed<K, V> firstMatchConsumed -> {
+                doHydrate();
+                break;
+            }
+            default -> {
+            }
+        }
+
+        if (_sortedIterators.isEmpty())
+            throw new NoSuchElementException();
+        return _goingForward ? _sortedIterators.firstKey() : _sortedIterators.lastKey();
+    }
+
+    @Override
+    protected void skipImpl() {
+        switch (_firstMatchState) {
+            case FirstMatchFound<K, V> firstMatchFound -> {
+                var curVal = firstMatchFound.iterator.next();
+                firstMatchFound.iterator.close();
+                _firstMatchState = new FirstMatchConsumed<>();
+//                Log.tracev("{0} Read from {1}: {2}, next: {3}", _name, firstMatchFound.iterator, curVal, _sortedIterators.keySet());
+                return;
+            }
+            case FirstMatchConsumed<K, V> firstMatchConsumed -> {
+                doHydrate();
+                break;
+            }
+            default -> {
+            }
+        }
+
+        var cur = _goingForward ? _sortedIterators.pollFirstEntry() : _sortedIterators.pollLastEntry();
+        if (cur == null) {
+            throw new NoSuchElementException();
+        }
+        cur.getValue().skip();
+        advanceIterator(cur.getValue());
+        Log.tracev("{0} Skip: {1}, next: {2}", _name, cur, _sortedIterators);
+    }
+
+    @Override
+    protected boolean hasImpl() {
+        switch (_firstMatchState) {
+            case FirstMatchFound<K, V> firstMatchFound -> {
+                return true;
+            }
+            case FirstMatchConsumed<K, V> firstMatchConsumed -> {
+                doHydrate();
+                break;
+            }
+            default -> {
+            }
+        }
+        return !_sortedIterators.isEmpty();
+    }
+
+    @Override
+    protected Pair<K, V> nextImpl() {
+        switch (_firstMatchState) {
+            case FirstMatchFound<K, V> firstMatchFound -> {
+                var curVal = firstMatchFound.iterator.next();
+                firstMatchFound.iterator.close();
+                _firstMatchState = new FirstMatchConsumed<>();
+//                Log.tracev("{0} Read from {1}: {2}, next: {3}", _name, firstMatchFound.iterator, curVal, _sortedIterators.keySet());
+                return curVal;
+            }
+            case FirstMatchConsumed<K, V> firstMatchConsumed -> {
+                doHydrate();
+                break;
+            }
+            default -> {
+            }
+        }
+
+        var cur = _goingForward ? _sortedIterators.pollFirstEntry() : _sortedIterators.pollLastEntry();
+        if (cur == null) {
+            throw new NoSuchElementException();
+        }
+        var curVal = cur.getValue().next();
+        advanceIterator(cur.getValue());
+//        Log.tracev("{0} Read from {1}: {2}, next: {3}", _name, cur.getValue(), curVal, _sortedIterators.keySet());
+        return curVal;
+    }
+
+    @Override
+    protected Class<? > peekTypeImpl() {
+        switch (_firstMatchState) {
+            case FirstMatchFound<K, V> firstMatchFound -> {
+                return firstMatchFound.iterator().peekNextType();
+            }
+            case FirstMatchConsumed<K, V> firstMatchConsumed -> {
+                doHydrate();
+                break;
+            }
+            default -> {
+            }
+        }
+
+        if (_sortedIterators.isEmpty())
+            throw new NoSuchElementException();
+
+        return _goingForward
+                ? _sortedIterators.firstEntry().getValue().peekNextType()
+                : _sortedIterators.lastEntry().getValue().peekNextType();
+    }
+
+
+    @Override
+    public void close() {
+        if (_firstMatchState instanceof FirstMatchFound(CloseableKvIterator iterator)) {
+            iterator.close();
+        }
+        for (CloseableKvIterator<K, V> iterator : _iterators.keySet()) {
+            iterator.close();
+        }
+    }
+
+    @Override
+    public String toString() {
+        return "MergingKvIterator{" +
+                "_name='" + _name + '\'' +
+                ", _sortedIterators=" + _sortedIterators.keySet() +
+                ", _iterators=" + _iterators +
+                '}';
+    }
+}

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/NavigableMapKvIterator.java

@@ -1,5 +1,6 @@
-package com.usatiuk.objects.iterators;
+package com.usatiuk.dhfs.objects;
 
+import com.usatiuk.dhfs.objects.persistence.IteratorStart;
 import org.apache.commons.lang3.tuple.Pair;
 
 import java.util.*;
@@ -90,6 +91,13 @@ public class NavigableMapKvIterator<K extends Comparable<K>, V> extends Reversib
         return Pair.of(ret);
     }
 
+    @Override
+    protected Class<? extends V> peekTypeImpl() {
+        if (_next == null)
+            throw new NoSuchElementException("No more elements");
+        return (Class<? extends V>) _next.getValue().getClass();
+    }
+
     @Override
     public void close() {
     }

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/ObjectSerializer.java

@@ -1,4 +1,4 @@
-package com.usatiuk.objects;
+package com.usatiuk.dhfs.objects;
 
 import com.google.protobuf.ByteString;
 

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/PendingDelete.java

@@ -1,6 +1,4 @@
-package com.usatiuk.objects.stores;
-
-import com.usatiuk.objects.JObjectKey;
+package com.usatiuk.dhfs.objects;
 
 public record PendingDelete(JObjectKey key, long bundleId) implements PendingWriteEntry {
 }

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/PendingWrite.java

@@ -1,6 +1,4 @@
-package com.usatiuk.objects.stores;
-
-import com.usatiuk.objects.JDataVersionedWrapper;
+package com.usatiuk.dhfs.objects;
 
 public record PendingWrite(JDataVersionedWrapper data, long bundleId) implements PendingWriteEntry {
 }

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/PendingWriteEntry.java

@@ -1,4 +1,4 @@
-package com.usatiuk.objects.stores;
+package com.usatiuk.dhfs.objects;
 
 public interface PendingWriteEntry {
     long bundleId();

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/PreCommitTxHook.java

@@ -1,7 +1,4 @@
-package com.usatiuk.objects.transaction;
-
-import com.usatiuk.objects.JData;
-import com.usatiuk.objects.JObjectKey;
+package com.usatiuk.dhfs.objects;
 
 public interface PreCommitTxHook {
     default void onChange(JObjectKey key, JData old, JData cur) {

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/PredicateKvIterator.java

@@ -1,5 +1,6 @@
-package com.usatiuk.objects.iterators;
+package com.usatiuk.dhfs.objects;
 
+import com.usatiuk.dhfs.objects.persistence.IteratorStart;
 import io.quarkus.logging.Log;
 import org.apache.commons.lang3.tuple.Pair;
 
@@ -130,6 +131,17 @@ public class PredicateKvIterator<K extends Comparable<K>, V, V_T> extends Revers
         return ret;
     }
 
+    @Override
+    protected Class<?> peekTypeImpl() {
+        if (!_checkedNext)
+            fillNext();
+
+        if (_next == null)
+            throw new NoSuchElementException("No more elements");
+
+        return _next.getValue().getClass();
+    }
+
     @Override
     public void close() {
         _backing.close();

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/ReversedKvIterator.java

@@ -1,8 +1,8 @@
-package com.usatiuk.objects.iterators;
+package com.usatiuk.dhfs.objects;
 
 import org.apache.commons.lang3.tuple.Pair;
 
-public class ReversedKvIterator<K extends Comparable<? super K>, V> implements CloseableKvIterator<K, V> {
+public class ReversedKvIterator<K extends Comparable<K>, V> implements CloseableKvIterator<K, V> {
     private final CloseableKvIterator<K, V> _backing;
 
     public ReversedKvIterator(CloseableKvIterator<K, V> backing) {
@@ -29,6 +29,11 @@ public class ReversedKvIterator<K extends Comparable<? super K>, V> implements C
         return _backing.peekPrevKey();
     }
 
+    @Override
+    public Class<?> peekNextType() {
+        return _backing.peekPrevType();
+    }
+
     @Override
     public void skip() {
         _backing.skipPrev();
@@ -39,6 +44,11 @@ public class ReversedKvIterator<K extends Comparable<? super K>, V> implements C
         return _backing.peekNextKey();
     }
 
+    @Override
+    public Class<?> peekPrevType() {
+        return _backing.peekNextType();
+    }
+
     @Override
     public Pair<K, V> prev() {
         return _backing.next();

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/ReversibleKvIterator.java

@@ -1,4 +1,4 @@
-package com.usatiuk.objects.iterators;
+package com.usatiuk.dhfs.objects;
 
 import org.apache.commons.lang3.tuple.Pair;
 
@@ -27,6 +27,8 @@ public abstract class ReversibleKvIterator<K extends Comparable<K>, V> implement
 
     abstract protected Pair<K, V> nextImpl();
 
+    abstract protected Class<?> peekTypeImpl();
+
     @Override
     public K peekNextKey() {
         ensureForward();
@@ -76,4 +78,15 @@ public abstract class ReversibleKvIterator<K extends Comparable<K>, V> implement
         skipImpl();
     }
 
+    @Override
+    public Class<?> peekNextType() {
+        ensureForward();
+        return peekTypeImpl();
+    }
+
+    @Override
+    public Class<?> peekPrevType() {
+        ensureBackward();
+        return peekTypeImpl();
+    }
 }

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/Tombstone.java

@@ -1,4 +1,4 @@
-package com.usatiuk.objects.iterators;
+package com.usatiuk.dhfs.objects;
 
 import java.util.Optional;
 

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/TombstoneMergingKvIterator.java

@@ -1,5 +1,6 @@
-package com.usatiuk.objects.iterators;
+package com.usatiuk.dhfs.objects;
 
+import com.usatiuk.dhfs.objects.persistence.IteratorStart;
 import io.quarkus.logging.Log;
 import org.apache.commons.lang3.tuple.Pair;
 
@@ -8,24 +9,25 @@ import java.util.List;
 public class TombstoneMergingKvIterator<K extends Comparable<K>, V> implements CloseableKvIterator<K, V> {
     private final CloseableKvIterator<K, V> _backing;
     private final String _name;
+    private final Class<?> _returnType;
 
-    public TombstoneMergingKvIterator(String name, IteratorStart startType, K startKey, List<IterProdFn<K, MaybeTombstone<V>>> iterators) {
+    public TombstoneMergingKvIterator(String name, IteratorStart startType, K startKey, List<IterProdFn<K, MaybeTombstone<V>>> iterators,
+                                      Class<?> returnType) {
         _name = name;
-        _backing = new PredicateKvIterator<>(
+        _returnType = returnType;
+        _backing = new MappingKvIterator<>(new TypePredicateKvIterator<>(
                 new MergingKvIterator<>(name + "-merging", startType, startKey, iterators),
                 startType, startKey,
-                pair -> {
-                    Log.tracev("{0} - Processing pair {1}", _name, pair);
-                    if (pair instanceof Tombstone) {
-                        return null;
-                    }
-                    return ((Data<V>) pair).value();
-                });
+                k -> {
+                    assert !k.equals(MaybeTombstone.class);
+                    assert Tombstone.class.isAssignableFrom(k) || Data.class.isAssignableFrom(k);
+                    return Data.class.isAssignableFrom(k);
+                }), t -> (V) returnType.cast(Data.class.cast(t).value()), (t) -> returnType);
     }
 
     @SafeVarargs
-    public TombstoneMergingKvIterator(String name, IteratorStart startType, K startKey, IterProdFn<K, MaybeTombstone<V>>... iterators) {
-        this(name, startType, startKey, List.of(iterators));
+    public TombstoneMergingKvIterator(String name, IteratorStart startType, K startKey, Class<?> returnType, IterProdFn<K, MaybeTombstone<V>>... iterators) {
+        this(name, startType, startKey, List.of(iterators), returnType);
     }
 
     @Override
@@ -33,6 +35,11 @@ public class TombstoneMergingKvIterator<K extends Comparable<K>, V> implements C
         return _backing.peekNextKey();
     }
 
+    @Override
+    public Class<?> peekNextType() {
+        return _returnType;
+    }
+
     @Override
     public void skip() {
         _backing.skip();
@@ -43,6 +50,11 @@ public class TombstoneMergingKvIterator<K extends Comparable<K>, V> implements C
         return _backing.peekPrevKey();
     }
 
+    @Override
+    public Class<?> peekPrevType() {
+        return _returnType;
+    }
+
     @Override
     public Pair<K, V> prev() {
         return _backing.prev();

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/TransactionManager.java

@@ -1,5 +1,7 @@
-package com.usatiuk.objects.transaction;
+package com.usatiuk.dhfs.objects;
 
+import com.usatiuk.dhfs.objects.transaction.Transaction;
+import com.usatiuk.dhfs.objects.transaction.TransactionHandle;
 import com.usatiuk.dhfs.utils.VoidFn;
 import io.quarkus.logging.Log;
 

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/TransactionManagerImpl.java

@@ -1,14 +1,13 @@
-package com.usatiuk.objects.transaction;
+package com.usatiuk.dhfs.objects;
 
+import com.usatiuk.dhfs.objects.transaction.Transaction;
+import com.usatiuk.dhfs.objects.transaction.TransactionHandle;
+import com.usatiuk.dhfs.objects.transaction.TransactionPrivate;
 import io.quarkus.logging.Log;
 import jakarta.enterprise.context.ApplicationScoped;
 import jakarta.inject.Inject;
-import jakarta.inject.Singleton;
-import org.apache.commons.lang3.tuple.Pair;
 
-import java.util.Collection;
-
-@Singleton
+@ApplicationScoped
 public class TransactionManagerImpl implements TransactionManager {
     private static final ThreadLocal<TransactionPrivate> _currentTransaction = new ThreadLocal<>();
     @Inject
@@ -32,10 +31,8 @@ public class TransactionManagerImpl implements TransactionManager {
         }
 
         Log.trace("Committing transaction");
-
-        Pair<Collection<Runnable>, TransactionHandle> ret;
         try {
-            ret = jObjectManager.commit(_currentTransaction.get());
+            return jObjectManager.commit(_currentTransaction.get());
         } catch (Throwable e) {
             Log.trace("Transaction commit failed", e);
             throw e;
@@ -43,15 +40,6 @@ public class TransactionManagerImpl implements TransactionManager {
             _currentTransaction.get().close();
             _currentTransaction.remove();
         }
-
-        for (var r : ret.getLeft()) {
-            try {
-                r.run();
-            } catch (Throwable e) {
-                Log.error("Transaction commit hook error: ", e);
-            }
-        }
-        return ret.getRight();
     }
 
     @Override

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/TransactionObjectLocked.java

@@ -1,7 +1,6 @@
-package com.usatiuk.objects.transaction;
+package com.usatiuk.dhfs.objects;
 
-import com.usatiuk.objects.JData;
-import com.usatiuk.objects.JDataVersionedWrapper;
+import com.usatiuk.dhfs.objects.transaction.TransactionObject;
 import com.usatiuk.dhfs.utils.AutoCloseableNoThrow;
 
 import java.util.Optional;

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/TransactionObjectNoLock.java

@@ -1,7 +1,6 @@
-package com.usatiuk.objects.transaction;
+package com.usatiuk.dhfs.objects;
 
-import com.usatiuk.objects.JData;
-import com.usatiuk.objects.JDataVersionedWrapper;
+import com.usatiuk.dhfs.objects.transaction.TransactionObject;
 
 import java.util.Optional;
 

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/TxCommitException.java

@@ -1,4 +1,4 @@
-package com.usatiuk.objects.transaction;
+package com.usatiuk.dhfs.objects;
 
 public class TxCommitException extends RuntimeException {
     public TxCommitException(String message) {

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/TypePredicateKvIterator.java

@@ -0,0 +1,141 @@
+package com.usatiuk.dhfs.objects;
+
+import com.usatiuk.dhfs.objects.persistence.IteratorStart;
+import org.apache.commons.lang3.tuple.Pair;
+
+import java.util.NoSuchElementException;
+import java.util.function.Function;
+
+public class TypePredicateKvIterator<K extends Comparable<K>, V> extends ReversibleKvIterator<K, V> {
+    private final CloseableKvIterator<K, V> _backing;
+    private final Function<Class<?>, Boolean> _filter;
+    private K _next;
+
+    public TypePredicateKvIterator(CloseableKvIterator<K, V> backing, IteratorStart start, K startKey, Function<Class<?>, Boolean> filter) {
+        _goingForward = true;
+        _backing = backing;
+        _filter = filter;
+        fillNext();
+
+        boolean shouldGoBack = false;
+        if (start == IteratorStart.LE) {
+            if (_next == null || _next.compareTo(startKey) > 0) {
+                shouldGoBack = true;
+            }
+        } else if (start == IteratorStart.LT) {
+            if (_next == null || _next.compareTo(startKey) >= 0) {
+                shouldGoBack = true;
+            }
+        }
+
+        if (shouldGoBack && _backing.hasPrev()) {
+            _goingForward = false;
+            _next = null;
+            fillNext();
+            if (_next != null)
+                _backing.skipPrev();
+            _goingForward = true;
+//            _backing.skip();
+            fillNext();
+        }
+
+
+        switch (start) {
+            case LT -> {
+//                assert _next == null || _next.getKey().compareTo(startKey) < 0;
+            }
+            case LE -> {
+//                assert _next == null || _next.getKey().compareTo(startKey) <= 0;
+            }
+            case GT -> {
+                assert _next == null || _next.compareTo(startKey) > 0;
+            }
+            case GE -> {
+                assert _next == null || _next.compareTo(startKey) >= 0;
+            }
+        }
+    }
+
+    private void fillNext() {
+        while ((_goingForward ? _backing.hasNext() : _backing.hasPrev()) && _next == null) {
+            var next = _goingForward ? _backing.peekNextType() : _backing.peekPrevType();
+            if (!_filter.apply(next)) {
+                if (_goingForward)
+                    _backing.skip();
+                else
+                    _backing.skipPrev();
+                continue;
+            } else {
+                _next = _goingForward ? _backing.peekNextKey() : _backing.peekPrevKey();
+            }
+        }
+    }
+
+    @Override
+    protected void reverse() {
+        _goingForward = !_goingForward;
+        _next = null;
+
+        fillNext();
+    }
+
+    @Override
+    protected K peekImpl() {
+        if (_next == null)
+            throw new NoSuchElementException();
+        return _next;
+    }
+
+    @Override
+    protected void skipImpl() {
+        if (_next == null)
+            throw new NoSuchElementException();
+        _next = null;
+        if (_goingForward)
+            _backing.skip();
+        else
+            _backing.skipPrev();
+        fillNext();
+    }
+
+    @Override
+    protected boolean hasImpl() {
+        return _next != null;
+    }
+
+    @Override
+    protected Pair<K, V> nextImpl() {
+        if (_next == null)
+            throw new NoSuchElementException("No more elements");
+        var retKey = _next;
+        _next = null;
+        var nextType = _goingForward ? _backing.peekNextType() : _backing.peekPrevType();
+        var got = _goingForward ? _backing.next() : _backing.prev();
+        assert got.getKey().equals(retKey);
+        assert nextType.equals(got.getValue().getClass());
+        assert _filter.apply(got.getValue().getClass());
+        fillNext();
+        return got;
+    }
+
+    @Override
+    protected Class<?> peekTypeImpl() {
+        if (_next == null)
+            throw new NoSuchElementException("No more elements");
+
+        return _goingForward ? _backing.peekNextType() : _backing.peekPrevType();
+    }
+
+    @Override
+    public void close() {
+        _backing.close();
+    }
+
+    @Override
+    public String toString() {
+        return "KeyPredicateKvIterator{" +
+                "_backing=" + _backing +
+                ", _next=" + _next +
+                '}';
+    }
+}

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/WritebackObjectPersistentStore.java

@@ -1,12 +1,9 @@
-package com.usatiuk.objects.stores;
+package com.usatiuk.dhfs.objects;
 
-import com.usatiuk.objects.JDataVersionedWrapper;
-import com.usatiuk.objects.JDataVersionedWrapperImpl;
-import com.usatiuk.objects.JObjectKey;
-import com.usatiuk.objects.iterators.*;
-import com.usatiuk.objects.snapshot.Snapshot;
-import com.usatiuk.objects.transaction.TxCommitException;
-import com.usatiuk.objects.transaction.TxRecord;
+import com.usatiuk.dhfs.objects.persistence.CachingObjectPersistentStore;
+import com.usatiuk.dhfs.objects.persistence.IteratorStart;
+import com.usatiuk.dhfs.objects.persistence.TxManifestObj;
+import com.usatiuk.dhfs.objects.transaction.TxRecord;
 import io.quarkus.logging.Log;
 import io.quarkus.runtime.ShutdownEvent;
 import io.quarkus.runtime.StartupEvent;
@@ -22,29 +19,27 @@ import org.pcollections.TreePMap;
 
 import javax.annotation.Nonnull;
 import java.util.*;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 
 @ApplicationScoped
 public class WritebackObjectPersistentStore {
     private final LinkedList<TxBundle> _pendingBundles = new LinkedList<>();
+
+    private final AtomicReference<PSortedMap<JObjectKey, PendingWriteEntry>> _pendingWrites = new AtomicReference<>(TreePMap.empty());
+    private final ReentrantReadWriteLock _pendingWritesVersionLock = new ReentrantReadWriteLock();
     private final LinkedHashMap<Long, TxBundle> _notFlushedBundles = new LinkedHashMap<>();
 
-    private record PendingWriteData(TreePMap<JObjectKey, PendingWriteEntry> pendingWrites,
-                                    long lastFlushedId,
-                                    long lastCommittedId) {
-    }
-
-    private final AtomicReference<PendingWriteData> _pendingWrites = new AtomicReference<>(null);
-
     private final Object _flushWaitSynchronizer = new Object();
-
-    private final AtomicLong _lastWrittenId = new AtomicLong(-1);
-    private final AtomicLong _lastCommittedId = new AtomicLong();
-
+    private final AtomicLong _lastWrittenTx = new AtomicLong(-1);
+    private final AtomicLong _counter = new AtomicLong();
+    private final AtomicLong _lastCommittedTx = new AtomicLong(-1);
     private final AtomicLong _waitedTotal = new AtomicLong(0);
     @Inject
     CachingObjectPersistentStore cachedStore;
@@ -55,7 +50,7 @@ public class WritebackObjectPersistentStore {
     private ExecutorService _statusExecutor;
     private volatile boolean _ready = false;
 
-    void init(@Observes @Priority(120) StartupEvent event) {
+    void init(@Observes @Priority(110) StartupEvent event) {
         {
             BasicThreadFactory factory = new BasicThreadFactory.Builder()
                     .namingPattern("tx-writeback-%d")
@@ -76,12 +71,8 @@ public class WritebackObjectPersistentStore {
             } catch (InterruptedException ignored) {
             }
         });
-        long lastTxId;
-        try (var s = cachedStore.getSnapshot()) {
-            lastTxId = s.id();
-        }
-        _lastCommittedId.set(lastTxId);
-        _pendingWrites.set(new PendingWriteData(TreePMap.empty(), lastTxId, lastTxId));
+        _counter.set(cachedStore.getLastTxId());
+        _lastCommittedTx.set(cachedStore.getLastTxId());
         _ready = true;
     }
 
@@ -114,10 +105,10 @@ public class WritebackObjectPersistentStore {
                     long diff = 0;
                     while (!_pendingBundles.isEmpty() && _pendingBundles.peek()._ready) {
                         var toCompress = _pendingBundles.poll();
-                        diff -= toCompress.size();
+                        diff -= toCompress.calculateTotalSize();
                         bundle.compress(toCompress);
                     }
-                    diff += bundle.size();
+                    diff += bundle.calculateTotalSize();
                     synchronized (_flushWaitSynchronizer) {
                         currentSize += diff;
                     }
@@ -129,11 +120,11 @@ public class WritebackObjectPersistentStore {
                 for (var e : bundle._entries.values()) {
                     switch (e) {
                         case TxBundle.CommittedEntry(JObjectKey key, JDataVersionedWrapper data, int size) -> {
-                            Log.tracev("Writing new {0}", key);
+                            Log.trace("Writing new " + key);
                             toWrite.add(Pair.of(key, data));
                         }
                         case TxBundle.DeletedEntry(JObjectKey key) -> {
-                            Log.tracev("Deleting from persistent storage {0}", key);
+                            Log.trace("Deleting from persistent storage " + key);
                             toDelete.add(key);
                         }
                         default -> throw new IllegalStateException("Unexpected value: " + e);
@@ -144,38 +135,35 @@ public class WritebackObjectPersistentStore {
                         new TxManifestObj<>(
                                 Collections.unmodifiableList(toWrite),
                                 Collections.unmodifiableList(toDelete)
-                        ), bundle.id());
+                        ), bundle.getId());
 
-                Log.tracev("Bundle {0} committed", bundle.id());
+                Log.trace("Bundle " + bundle.getId() + " committed");
 
-                while (true) {
+                // Remove from pending writes, after real commit
+                // As we are the only writers to _pendingWrites, no need to synchronize with iterator creation
+                // if they get the older version, as it will still contain all the new changes
+                synchronized (_pendingBundles) {
                     var curPw = _pendingWrites.get();
-                    var curPwMap = curPw.pendingWrites();
                     for (var e : bundle._entries.values()) {
-                        var cur = curPwMap.get(e.key());
-                        if (cur.bundleId() <= bundle.id())
-                            curPwMap = curPwMap.minus(e.key());
+                        var cur = curPw.get(e.key());
+                        if (cur.bundleId() <= bundle.getId())
+                            curPw = curPw.minus(e.key());
                     }
-                    var newCurPw = new PendingWriteData(
-                            curPwMap,
-                            bundle.id(),
-                            curPw.lastCommittedId()
-                    );
-                    if (_pendingWrites.compareAndSet(curPw, newCurPw))
-                        break;
+                    _pendingWrites.set(curPw);
+                    // No need to increment version
                 }
 
                 List<List<Runnable>> callbacks = new ArrayList<>();
                 synchronized (_notFlushedBundles) {
-                    _lastWrittenId.set(bundle.id());
-                    while (!_notFlushedBundles.isEmpty() && _notFlushedBundles.firstEntry().getKey() <= bundle.id()) {
+                    _lastWrittenTx.set(bundle.getId());
+                    while (!_notFlushedBundles.isEmpty() && _notFlushedBundles.firstEntry().getKey() <= bundle.getId()) {
                         callbacks.add(_notFlushedBundles.pollFirstEntry().getValue().setCommitted());
                     }
                 }
                 callbacks.forEach(l -> l.forEach(Runnable::run));
 
                 synchronized (_flushWaitSynchronizer) {
-                    currentSize -= bundle.size();
+                    currentSize -= bundle.calculateTotalSize();
                     // FIXME:
                     if (currentSize <= sizeLimit || !_ready)
                         _flushWaitSynchronizer.notifyAll();
@@ -190,7 +178,8 @@ public class WritebackObjectPersistentStore {
         Log.info("Writeback thread exiting");
     }
 
-    public long commitBundle(Collection<TxRecord.TxObjectRecord<?>> writes) {
+
+    public TxBundle createBundle() {
         verifyReady();
         boolean wait = false;
         while (true) {
@@ -207,24 +196,23 @@ public class WritebackObjectPersistentStore {
                     long waited = System.currentTimeMillis() - started;
                     _waitedTotal.addAndGet(waited);
                     if (Log.isTraceEnabled())
-                        Log.tracev("Thread {0} waited for tx bundle for {1} ms", Thread.currentThread().getName(), waited);
+                        Log.trace("Thread " + Thread.currentThread().getName() + " waited for tx bundle for " + waited + " ms");
                     wait = false;
                 }
             }
-
             synchronized (_pendingBundles) {
                 synchronized (_flushWaitSynchronizer) {
                     if (currentSize > sizeLimit) {
                         if (!_pendingBundles.isEmpty() && _pendingBundles.peek()._ready) {
                             var target = _pendingBundles.poll();
 
-                            long diff = -target.size();
+                            long diff = -target.calculateTotalSize();
                             while (!_pendingBundles.isEmpty() && _pendingBundles.peek()._ready) {
                                 var toCompress = _pendingBundles.poll();
-                                diff -= toCompress.size();
+                                diff -= toCompress.calculateTotalSize();
                                 target.compress(toCompress);
                             }
-                            diff += target.size();
+                            diff += target.calculateTotalSize();
                             currentSize += diff;
                             _pendingBundles.addFirst(target);
                         }
@@ -235,76 +223,79 @@ public class WritebackObjectPersistentStore {
                         continue;
                     }
                 }
-
-                TxBundle bundle;
                 synchronized (_notFlushedBundles) {
-                    bundle = new TxBundle(_lastCommittedId.incrementAndGet());
+                    var bundle = new TxBundle(_counter.incrementAndGet());
                     _pendingBundles.addLast(bundle);
-                    _notFlushedBundles.put(bundle.id(), bundle);
-                }
-
-                for (var action : writes) {
-                    switch (action) {
-                        case TxRecord.TxObjectRecordWrite<?> write -> {
-                            Log.tracev("Flushing object {0}", write.key());
-                            bundle.commit(new JDataVersionedWrapperImpl(write.data(), bundle.id()));
-                        }
-                        case TxRecord.TxObjectRecordDeleted deleted -> {
-                            Log.tracev("Deleting object {0}", deleted.key());
-                            bundle.delete(deleted.key());
-                        }
-                        default -> {
-                            throw new TxCommitException("Unexpected value: " + action.key());
-                        }
-                    }
-                }
-
-                while (true) {
-                    var curPw = _pendingWrites.get();
-                    var curPwMap = curPw.pendingWrites();
-                    for (var e : ((TxBundle) bundle)._entries.values()) {
-                        switch (e) {
-                            case TxBundle.CommittedEntry c -> {
-                                curPwMap = curPwMap.plus(c.key(), new PendingWrite(c.data, bundle.id()));
-                            }
-                            case TxBundle.DeletedEntry d -> {
-                                curPwMap = curPwMap.plus(d.key(), new PendingDelete(d.key, bundle.id()));
-                            }
-                            default -> throw new IllegalStateException("Unexpected value: " + e);
-                        }
-                    }
-                    // Now, make the changes visible to new iterators
-                    var newCurPw = new PendingWriteData(
-                            curPwMap,
-                            curPw.lastFlushedId(),
-                            bundle.id()
-                    );
-
-                    if (!_pendingWrites.compareAndSet(curPw, newCurPw))
-                        continue;
-
-                    ((TxBundle) bundle).setReady();
-                    if (_pendingBundles.peek() == bundle)
-                        _pendingBundles.notify();
-                    synchronized (_flushWaitSynchronizer) {
-                        currentSize += ((TxBundle) bundle).size();
-                    }
-
-                    return bundle.id();
+                    _notFlushedBundles.put(bundle.getId(), bundle);
+                    return bundle;
                 }
             }
         }
     }
 
+    public void commitBundle(TxBundle bundle) {
+        verifyReady();
+        _pendingWritesVersionLock.writeLock().lock();
+        try {
+            synchronized (_pendingBundles) {
+                var curPw = _pendingWrites.get();
+                for (var e : ((TxBundle) bundle)._entries.values()) {
+                    switch (e) {
+                        case TxBundle.CommittedEntry c -> {
+                            curPw = curPw.plus(c.key(), new PendingWrite(c.data, bundle.getId()));
+                        }
+                        case TxBundle.DeletedEntry d -> {
+                            curPw = curPw.plus(d.key(), new PendingDelete(d.key, bundle.getId()));
+                        }
+                        default -> throw new IllegalStateException("Unexpected value: " + e);
+                    }
+                }
+                // Now, make the changes visible to new iterators
+                _pendingWrites.set(curPw);
+                ((TxBundle) bundle).setReady();
+                if (_pendingBundles.peek() == bundle)
+                    _pendingBundles.notify();
+                synchronized (_flushWaitSynchronizer) {
+                    currentSize += ((TxBundle) bundle).calculateTotalSize();
+                }
+            }
+            assert bundle.getId() > _lastCommittedTx.get();
+            _lastCommittedTx.set(bundle.getId());
+        } finally {
+            _pendingWritesVersionLock.writeLock().unlock();
+        }
+    }
+
+    public void dropBundle(TxBundle bundle) {
+        verifyReady();
+        synchronized (_pendingBundles) {
+            Log.warn("Dropped bundle: " + bundle);
+            _pendingBundles.remove((TxBundle) bundle);
+            synchronized (_flushWaitSynchronizer) {
+                currentSize -= ((TxBundle) bundle).calculateTotalSize();
+            }
+        }
+    }
+
+    public void fence(long bundleId) {
+        var latch = new CountDownLatch(1);
+        asyncFence(bundleId, latch::countDown);
+        try {
+            latch.await();
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     public void asyncFence(long bundleId, Runnable fn) {
         verifyReady();
         if (bundleId < 0) throw new IllegalArgumentException("txId should be >0!");
-        if (_lastWrittenId.get() >= bundleId) {
+        if (_lastWrittenTx.get() >= bundleId) {
             fn.run();
             return;
         }
         synchronized (_notFlushedBundles) {
-            if (_lastWrittenId.get() >= bundleId) {
+            if (_lastWrittenTx.get() >= bundleId) {
                 fn.run();
                 return;
             }
@@ -312,105 +303,19 @@ public class WritebackObjectPersistentStore {
         }
     }
 
-    public Consumer<Runnable> commitTx(Collection<TxRecord.TxObjectRecord<?>> writes) {
-        long bundleId = commitBundle(writes);
-
-        return r -> asyncFence(bundleId, r);
-    }
-
-    public Snapshot<JObjectKey, JDataVersionedWrapper> getSnapshot() {
-        Snapshot<JObjectKey, JDataVersionedWrapper> cache = null;
-        PendingWriteData pw = null;
-
-        try {
-            while (true) {
-                pw = _pendingWrites.get();
-                cache = cachedStore.getSnapshot();
-
-                if (cache.id() >= pw.lastCommittedId())
-                    return cache;
-
-                // TODO: Can this really happen?
-                if (cache.id() < pw.lastFlushedId()) {
-                    assert false;
-                    cache.close();
-                    cache = null;
-                    continue;
-                }
-
-                break;
-            }
-
-            PendingWriteData finalPw = pw;
-            Snapshot<JObjectKey, JDataVersionedWrapper> finalCache = cache;
-            return new Snapshot<JObjectKey, JDataVersionedWrapper>() {
-                private final PSortedMap<JObjectKey, PendingWriteEntry> _pendingWrites = finalPw.pendingWrites();
-                private final Snapshot<JObjectKey, JDataVersionedWrapper> _cache = finalCache;
-                private final long txId = finalPw.lastCommittedId();
-
-                @Override
-                public CloseableKvIterator<JObjectKey, JDataVersionedWrapper> getIterator(IteratorStart start, JObjectKey key) {
-                    return new TombstoneMergingKvIterator<>("writeback-ps", start, key,
-                            (tS, tK) -> new MappingKvIterator<>(
-                                    new NavigableMapKvIterator<>(_pendingWrites, tS, tK),
-                                    e -> switch (e) {
-                                        case PendingWrite pw -> new Data<>(pw.data());
-                                        case PendingDelete d -> new Tombstone<>();
-                                        default -> throw new IllegalStateException("Unexpected value: " + e);
-                                    }),
-                            (tS, tK) -> new MappingKvIterator<>(_cache.getIterator(tS, tK), Data::new));
-                }
-
-                @Nonnull
-                @Override
-                public Optional<JDataVersionedWrapper> readObject(JObjectKey name) {
-                    var cached = _pendingWrites.get(name);
-                    if (cached != null) {
-                        return switch (cached) {
-                            case PendingWrite c -> Optional.of(c.data());
-                            case PendingDelete d -> {
-                                yield Optional.empty();
-                            }
-                            default -> throw new IllegalStateException("Unexpected value: " + cached);
-                        };
-                    }
-                    return _cache.readObject(name);
-                }
-
-                @Override
-                public long id() {
-                    assert txId >= _cache.id();
-                    return txId;
-                }
-
-                @Override
-                public void close() {
-                    _cache.close();
-                }
-            };
-        } catch (Throwable e) {
-            if (cache != null)
-                cache.close();
-            throw e;
-        }
-    }
-
-    public interface VerboseReadResult {
-    }
-
     private static class TxBundle {
         private final LinkedHashMap<JObjectKey, BundleEntry> _entries = new LinkedHashMap<>();
         private final ArrayList<Runnable> _callbacks = new ArrayList<>();
         private long _txId;
         private volatile boolean _ready = false;
-        private long _size = 0;
+        private long _size = -1;
         private boolean _wasCommitted = false;
 
         private TxBundle(long txId) {
             _txId = txId;
         }
 
-        public long id() {
+        public long getId() {
             return _txId;
         }
 
@@ -432,23 +337,21 @@ public class WritebackObjectPersistentStore {
             }
         }
 
-        private void putEntry(BundleEntry entry) {
-            var old = _entries.put(entry.key(), entry);
-            if (old != null) {
-                _size -= old.size();
-            }
-            _size += entry.size();
-        }
-
         public void commit(JDataVersionedWrapper obj) {
-            putEntry(new CommittedEntry(obj.data().key(), obj, obj.data().estimateSize()));
+            synchronized (_entries) {
+                _entries.put(obj.data().key(), new CommittedEntry(obj.data().key(), obj, obj.data().estimateSize()));
+            }
         }
 
         public void delete(JObjectKey obj) {
-            putEntry(new DeletedEntry(obj));
+            synchronized (_entries) {
+                _entries.put(obj, new DeletedEntry(obj));
+            }
         }
 
-        public long size() {
+        public long calculateTotalSize() {
+            if (_size >= 0) return _size;
+            _size = _entries.values().stream().mapToInt(BundleEntry::size).sum();
             return _size;
         }
 
@@ -457,16 +360,9 @@ public class WritebackObjectPersistentStore {
                 throw new IllegalArgumentException("Compressing an older bundle into newer");
 
             _txId = other._txId;
+            _size = -1;
 
-            for (var entry : other._entries.values()) {
-                putEntry(entry);
-            }
-
-            synchronized (_callbacks) {
-                assert !_wasCommitted;
-                assert !other._wasCommitted;
-                _callbacks.addAll(other._callbacks);
-            }
+            _entries.putAll(other._entries);
         }
 
         private interface BundleEntry {
@@ -488,9 +384,116 @@ public class WritebackObjectPersistentStore {
         }
     }
 
+    public Optional<PendingWriteEntry> getPendingWrite(JObjectKey key) {
+        synchronized (_pendingBundles) {
+            return Optional.ofNullable(_pendingWrites.get().get(key));
+        }
+    }
+
+    @Nonnull
+    public Optional<JDataVersionedWrapper> readObject(JObjectKey name) {
+        var pending = getPendingWrite(name).orElse(null);
+        return switch (pending) {
+            case PendingWrite write -> Optional.of(write.data());
+            case PendingDelete ignored -> Optional.empty();
+            case null -> cachedStore.readObject(name);
+            default -> throw new IllegalStateException("Unexpected value: " + pending);
+        };
+    }
+
+    public interface VerboseReadResult {
+    }
+
     public record VerboseReadResultPersisted(Optional<JDataVersionedWrapper> data) implements VerboseReadResult {
     }
 
     public record VerboseReadResultPending(PendingWriteEntry pending) implements VerboseReadResult {
     }
+
+    @Nonnull
+    public VerboseReadResult readObjectVerbose(JObjectKey key) {
+        var pending = getPendingWrite(key).orElse(null);
+        if (pending != null) {
+            return new VerboseReadResultPending(pending);
+        }
+        return new VerboseReadResultPersisted(cachedStore.readObject(key));
+    }
+
+    /**
+     * @param commitLocked - a function that will be called with a Consumer of a new transaction id,
+     *                     that will commit the transaction the changes in the store will be visible to new transactions
+     *                     only after the runnable is called
+     */
+    public Consumer<Runnable> commitTx(Collection<TxRecord.TxObjectRecord<?>> writes, BiConsumer<Long, Runnable> commitLocked) {
+        var bundle = createBundle();
+        long bundleId = bundle.getId();
+        try {
+            for (var action : writes) {
+                switch (action) {
+                    case TxRecord.TxObjectRecordWrite<?> write -> {
+                        Log.trace("Flushing object " + write.key());
+                        bundle.commit(new JDataVersionedWrapperImpl(write.data(), bundleId));
+                    }
+                    case TxRecord.TxObjectRecordDeleted deleted -> {
+                        Log.trace("Deleting object " + deleted.key());
+                        bundle.delete(deleted.key());
+                    }
+                    default -> {
+                        throw new TxCommitException("Unexpected value: " + action.key());
+                    }
+                }
+            }
+        } catch (Throwable t) {
+            dropBundle(bundle);
+            throw new TxCommitException(t.getMessage(), t);
+        }
+
+
+        Log.tracef("Committing transaction %d to storage", bundleId);
+        commitLocked.accept(bundleId, () -> {
+            commitBundle(bundle);
+        });
+
+        return r -> asyncFence(bundleId, r);
+    }
+
+    // Returns an iterator with a view of all commited objects
+    // Does not have to guarantee consistent view, snapshots are handled by upper layers
+    // Invalidated by commitBundle, but might return data after it has been really committed
+    public CloseableKvIterator<JObjectKey, JDataVersionedWrapper> getIterator(IteratorStart start, JObjectKey key) {
+        Log.tracev("Getting writeback iterator: {0}, {1}", start, key);
+        _pendingWritesVersionLock.readLock().lock();
+        try {
+            var curPending = _pendingWrites.get();
+            return new TombstoneMergingKvIterator<>("writeback-ps", start, key, JDataVersionedWrapper.class,
+                    (tS, tK) -> new MappingKvIterator<>(
+                            new NavigableMapKvIterator<>(curPending, tS, tK),
+                            e -> switch (e) {
+                                case PendingWrite pw -> new Data<>(pw.data());
+                                case PendingDelete d -> new Tombstone<>();
+                                default -> throw new IllegalStateException("Unexpected value: " + e);
+                            },
+                            e -> {
+                                if (PendingWrite.class.isAssignableFrom(e)) {
+                                    return Data.class;
+                                } else if (PendingDelete.class.isAssignableFrom(e)) {
+                                    return Tombstone.class;
+                                } else {
+                                    throw new IllegalStateException("Unexpected type: " + e);
+                                }
+                            }),
+                    (tS, tK) -> cachedStore.getIterator(tS, tK));
+        } finally {
+            _pendingWritesVersionLock.readLock().unlock();
+        }
+    }
+
+    public long getLastTxId() {
+        _pendingWritesVersionLock.readLock().lock();
+        try {
+            return _lastCommittedTx.get();
+        } finally {
+            _pendingWritesVersionLock.readLock().unlock();
+        }
+    }
 }

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/persistence/CachingObjectPersistentStore.java

@@ -0,0 +1,264 @@
+package com.usatiuk.dhfs.objects.persistence;
+
+import com.usatiuk.dhfs.objects.*;
+import com.usatiuk.dhfs.utils.DataLocker;
+import io.quarkus.logging.Log;
+import io.quarkus.runtime.Startup;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+import org.apache.commons.lang3.tuple.Pair;
+import org.eclipse.microprofile.config.inject.ConfigProperty;
+import org.pcollections.TreePMap;
+
+import javax.annotation.Nonnull;
+import java.util.LinkedHashMap;
+import java.util.Optional;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+@ApplicationScoped
+public class CachingObjectPersistentStore {
+    private final LinkedHashMap<JObjectKey, CacheEntry> _cache = new LinkedHashMap<>();
+    private TreePMap<JObjectKey, CacheEntry> _sortedCache = TreePMap.empty();
+    private long _cacheVersion = 0;
+
+    private final ReentrantReadWriteLock _lock = new ReentrantReadWriteLock();
+    private final DataLocker _readerLocker = new DataLocker();
+
+    @Inject
+    SerializingObjectPersistentStore delegate;
+    @ConfigProperty(name = "dhfs.objects.lru.limit")
+    long sizeLimit;
+    @ConfigProperty(name = "dhfs.objects.lru.print-stats")
+    boolean printStats;
+
+    private long _curSize = 0;
+    private long _evict = 0;
+
+    private ExecutorService _statusExecutor = null;
+
+    @Startup
+    void init() {
+        if (printStats) {
+            _statusExecutor = Executors.newSingleThreadExecutor();
+            _statusExecutor.submit(() -> {
+                try {
+                    while (true) {
+                        Thread.sleep(10000);
+                        if (_curSize > 0)
+                            Log.info("Cache status: size=" + _curSize / 1024 / 1024 + "MB" + " evicted=" + _evict);
+                        _evict = 0;
+                    }
+                } catch (InterruptedException ignored) {
+                }
+            });
+        }
+    }
+
+    private void put(JObjectKey key, Optional<JDataVersionedWrapper> obj) {
+//        Log.tracev("Adding {0} to cache: {1}", key, obj);
+        _lock.writeLock().lock();
+        try {
+            int size = obj.map(JDataVersionedWrapper::estimateSize).orElse(16);
+
+            _curSize += size;
+            CacheEntry entry = obj.<CacheEntry>map(v -> new CacheEntryYes(v, size)).orElse(new CacheEntryDeleted());
+            var old = _cache.putLast(key, entry);
+
+            _sortedCache = _sortedCache.plus(key, entry);
+            if (old != null)
+                _curSize -= old.size();
+
+            while (_curSize >= sizeLimit) {
+                var del = _cache.pollFirstEntry();
+                _sortedCache = _sortedCache.minus(del.getKey());
+                _curSize -= del.getValue().size();
+                _evict++;
+            }
+        } finally {
+            _lock.writeLock().unlock();
+        }
+    }
+
+    @Nonnull
+    public Optional<JDataVersionedWrapper> readObject(JObjectKey name) {
+        _lock.readLock().lock();
+        try {
+            var got = _cache.get(name);
+            if (got != null) {
+                return switch (got) {
+                    case CacheEntryYes yes -> Optional.of(yes.object());
+                    case CacheEntryDeleted del -> Optional.empty();
+                    default -> throw new IllegalStateException("Unexpected value: " + got);
+                };
+            }
+        } finally {
+            _lock.readLock().unlock();
+        }
+        try (var lock = _readerLocker.lock(name)) {
+            // TODO: This is possibly racy
+//            var got = delegate.readObject(name);
+//            put(name, got);
+            return delegate.readObject(name);
+        }
+    }
+
+    public void commitTx(TxManifestObj<? extends JDataVersionedWrapper> names, long txId) {
+        var serialized = delegate.prepareManifest(names);
+        Log.tracev("Committing: {0} writes, {1} deletes", names.written().size(), names.deleted().size());
+        delegate.commitTx(serialized, txId, (commit) -> {
+            _lock.writeLock().lock();
+            try {
+                // Make the changes visible atomically both in cache and in the underlying store
+                for (var write : names.written()) {
+                    put(write.getLeft(), Optional.of(write.getRight()));
+                }
+                for (var del : names.deleted()) {
+                    put(del, Optional.empty());
+                }
+                ++_cacheVersion;
+                commit.run();
+            } finally {
+                _lock.writeLock().unlock();
+            }
+        });
+        Log.tracev("Committed: {0} writes, {1} deletes", names.written().size(), names.deleted().size());
+    }
+
+
+    private class CachingKvIterator implements CloseableKvIterator<JObjectKey, JDataVersionedWrapper> {
+        private final CloseableKvIterator<JObjectKey, JDataVersionedWrapper> _delegate;
+        // This should be created under lock
+        private final long _curCacheVersion = _cacheVersion;
+
+        private CachingKvIterator(CloseableKvIterator<JObjectKey, JDataVersionedWrapper> delegate) {
+            _delegate = delegate;
+        }
+
+        @Override
+        public JObjectKey peekNextKey() {
+            return _delegate.peekNextKey();
+        }
+
+        @Override
+        public Class<?> peekNextType() {
+            return _delegate.peekNextType();
+        }
+
+        @Override
+        public void skip() {
+            _delegate.skip();
+        }
+
+        @Override
+        public void close() {
+            _delegate.close();
+        }
+
+        @Override
+        public boolean hasNext() {
+            return _delegate.hasNext();
+        }
+
+        @Override
+        public JObjectKey peekPrevKey() {
+            return _delegate.peekPrevKey();
+        }
+
+        @Override
+        public Class<?> peekPrevType() {
+            return _delegate.peekPrevType();
+        }
+
+        private void maybeCache(Pair<JObjectKey, JDataVersionedWrapper> prev) {
+            _lock.writeLock().lock();
+            try {
+                if (_cacheVersion != _curCacheVersion) {
+                    Log.tracev("Not caching: {0}", prev);
+                } else {
+                    Log.tracev("Caching: {0}", prev);
+                    put(prev.getKey(), Optional.of(prev.getValue()));
+                }
+            } finally {
+                _lock.writeLock().unlock();
+            }
+        }
+
+        @Override
+        public Pair<JObjectKey, JDataVersionedWrapper> prev() {
+            var prev = _delegate.prev();
+            maybeCache(prev);
+            return prev;
+        }
+
+        @Override
+        public boolean hasPrev() {
+            return _delegate.hasPrev();
+        }
+
+        @Override
+        public void skipPrev() {
+            _delegate.skipPrev();
+        }
+
+        @Override
+        public Pair<JObjectKey, JDataVersionedWrapper> next() {
+            var next = _delegate.next();
+            maybeCache(next);
+            return next;
+        }
+    }
+
+    // Returns an iterator with a view of all commited objects
+    // Does not have to guarantee consistent view, snapshots are handled by upper layers
+    // Warning: it has a nasty side effect of global caching, so in this case don't even call next on it,
+    // if some objects are still in writeback
+    public CloseableKvIterator<JObjectKey, MaybeTombstone<JDataVersionedWrapper>> getIterator(IteratorStart start, JObjectKey key) {
+        _lock.readLock().lock();
+        try {
+            Log.tracev("Getting cache iterator: {0}, {1}", start, key);
+            var curSortedCache = _sortedCache;
+            return new MergingKvIterator<>("cache", start, key,
+                    (mS, mK)
+                            -> new MappingKvIterator<>(
+                            new NavigableMapKvIterator<>(curSortedCache, mS, mK),
+                            e -> switch (e) {
+                                case CacheEntryYes pw -> new Data<>(pw.object());
+                                case CacheEntryDeleted d -> new Tombstone<>();
+                                default -> throw new IllegalStateException("Unexpected value: " + e);
+                            },
+                            e -> {
+                                if (CacheEntryYes.class.isAssignableFrom(e)) {
+                                    return Data.class;
+                                } else if (CacheEntryDeleted.class.isAssignableFrom(e)) {
+                                    return Tombstone.class;
+                                } else {
+                                    throw new IllegalStateException("Unexpected type: " + e);
+                                }
+                            }),
+                    (mS, mK)
+                            -> new MappingKvIterator<>(new CachingKvIterator(delegate.getIterator(mS, mK)), Data::new, (d) -> Data.class));
+        } finally {
+            _lock.readLock().unlock();
+        }
+    }
+
+    private interface CacheEntry {
+        long size();
+    }
+
+    private record CacheEntryYes(JDataVersionedWrapper object, long size) implements CacheEntry {
+    }
+
+    private record CacheEntryDeleted() implements CacheEntry {
+        @Override
+        public long size() {
+            return 16;
+        }
+    }
+
+    public long getLastTxId() {
+        return delegate.getLastCommitId();
+    }
+}

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/persistence/IteratorStart.java

@@ -1,4 +1,4 @@
-package com.usatiuk.objects.iterators;
+package com.usatiuk.dhfs.objects.persistence;
 
 public enum IteratorStart {
     LT,

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/persistence/LmdbObjectPersistentStore.java

@@ -1,15 +1,10 @@
-package com.usatiuk.objects.stores;
+package com.usatiuk.dhfs.objects.persistence;
 
 import com.google.protobuf.ByteString;
-import com.google.protobuf.UnsafeByteOperations;
-import com.usatiuk.objects.JObjectKey;
-import com.usatiuk.objects.JObjectKeyMax;
-import com.usatiuk.objects.JObjectKeyMin;
-import com.usatiuk.objects.iterators.CloseableKvIterator;
-import com.usatiuk.objects.iterators.IteratorStart;
-import com.usatiuk.objects.iterators.KeyPredicateKvIterator;
-import com.usatiuk.objects.iterators.ReversibleKvIterator;
-import com.usatiuk.objects.snapshot.Snapshot;
+import com.usatiuk.dhfs.objects.CloseableKvIterator;
+import com.usatiuk.dhfs.objects.JObjectKey;
+import com.usatiuk.dhfs.objects.KeyPredicateKvIterator;
+import com.usatiuk.dhfs.objects.ReversibleKvIterator;
 import com.usatiuk.dhfs.supportlib.UninitializedByteBuffer;
 import com.usatiuk.dhfs.utils.RefcountedCloseable;
 import io.quarkus.arc.properties.IfBuildProperty;
@@ -30,9 +25,10 @@ import java.lang.ref.Cleaner;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
-import java.util.Arrays;
-import java.util.NoSuchElementException;
-import java.util.Optional;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.function.Consumer;
 
 import static org.lmdbjava.DbiFlags.MDB_CREATE;
 import static org.lmdbjava.Env.create;
@@ -40,21 +36,18 @@ import static org.lmdbjava.Env.create;
 @ApplicationScoped
 @IfBuildProperty(name = "dhfs.objects.persistence", stringValue = "lmdb")
 public class LmdbObjectPersistentStore implements ObjectPersistentStore {
-    private static final String DB_NAME = "objects";
-    private static final ByteBuffer DB_VER_OBJ_NAME;
-
-    static {
-        byte[] tmp = "__DB_VER_OBJ".getBytes(StandardCharsets.UTF_8);
-        var bb = ByteBuffer.allocateDirect(tmp.length);
-        bb.put(tmp);
-        bb.flip();
-        DB_VER_OBJ_NAME = bb.asReadOnlyBuffer();
-    }
-
     private final Path _root;
     private Env<ByteBuffer> _env;
     private Dbi<ByteBuffer> _db;
     private boolean _ready = false;
+    private final AtomicReference<RefcountedCloseable<Txn<ByteBuffer>>> _curReadTxn = new AtomicReference<>();
+
+    private long _lastTxId = 0;
+
+    private final ReentrantReadWriteLock _lock = new ReentrantReadWriteLock();
+
+    private static final String DB_NAME = "objects";
+    private static final byte[] DB_VER_OBJ_NAME = "__DB_VER_OBJ".getBytes(StandardCharsets.UTF_8);
 
     public LmdbObjectPersistentStore(@ConfigProperty(name = "dhfs.objects.persistence.files.root") String root) {
         _root = Path.of(root).resolve("objects");
@@ -71,27 +64,22 @@ public class LmdbObjectPersistentStore implements ObjectPersistentStore {
                 .open(_root.toFile(), EnvFlags.MDB_NOTLS);
         _db = _env.openDbi(DB_NAME, MDB_CREATE);
 
-        try (Txn<ByteBuffer> txn = _env.txnWrite()) {
-            var read = readTxId(txn);
-            if (read.isPresent()) {
-                Log.infov("Read tx id {0}", read.get());
-            } else {
-                var bbData = ByteBuffer.allocateDirect(8);
-                bbData.putLong(0);
-                bbData.flip();
-                _db.put(txn, DB_VER_OBJ_NAME.asReadOnlyBuffer(), bbData);
-                txn.commit();
+        var bb = ByteBuffer.allocateDirect(DB_VER_OBJ_NAME.length);
+        bb.put(DB_VER_OBJ_NAME);
+        bb.flip();
+
+        try (Txn<ByteBuffer> txn = _env.txnRead()) {
+            var value = _db.get(txn, bb);
+            if (value != null) {
+                var ver = value.getLong();
+                Log.infov("Read version: {0}", ver);
+                _lastTxId = ver;
             }
         }
 
         _ready = true;
     }
 
-    private Optional<Long> readTxId(Txn<ByteBuffer> txn) {
-        var value = _db.get(txn, DB_VER_OBJ_NAME.asReadOnlyBuffer());
-        return Optional.ofNullable(value).map(ByteBuffer::getLong);
-    }
-
     void shutdown(@Observes @Priority(900) ShutdownEvent event) throws IOException {
         _ready = false;
         _db.close();
@@ -102,6 +90,22 @@ public class LmdbObjectPersistentStore implements ObjectPersistentStore {
         if (!_ready) throw new IllegalStateException("Wrong service order!");
     }
 
+    @Nonnull
+    @Override
+    public Collection<JObjectKey> findAllObjects() {
+//        try (Txn<ByteBuffer> txn = env.txnRead()) {
+//            try (var cursor = db.openCursor(txn)) {
+//                var keys = List.of();
+//                while (cursor.next()) {
+//                    keys.add(JObjectKey.fromBytes(cursor.key()));
+//                }
+//                return keys;
+//            }
+//        }
+        return List.of();
+    }
+
+
     @Nonnull
     @Override
     public Optional<ByteString> readObject(JObjectKey name) {
@@ -112,113 +116,33 @@ public class LmdbObjectPersistentStore implements ObjectPersistentStore {
         }
     }
 
-    @Override
-    public Snapshot<JObjectKey, ByteString> getSnapshot() {
-        var txn = new RefcountedCloseable<>(_env.txnRead());
-        long commitId = readTxId(txn.get()).orElseThrow();
-        return new Snapshot<JObjectKey, ByteString>() {
-            private final RefcountedCloseable<Txn<ByteBuffer>> _txn = txn;
-            private final long _id = commitId;
-            private boolean _closed = false;
-
-            @Override
-            public CloseableKvIterator<JObjectKey, ByteString> getIterator(IteratorStart start, JObjectKey key) {
-                assert !_closed;
-                return new KeyPredicateKvIterator<>(new LmdbKvIterator(_txn.ref(), start, key), start, key, (k) -> !StandardCharsets.UTF_8.encode(k.value()).equals(DB_VER_OBJ_NAME.asReadOnlyBuffer()));
-            }
-
-            @Nonnull
-            @Override
-            public Optional<ByteString> readObject(JObjectKey name) {
-                assert !_closed;
-                var got = _db.get(_txn.get(), name.toByteBuffer());
-                var ret = Optional.ofNullable(got).map(read -> {
-                    var uninitBb = UninitializedByteBuffer.allocateUninitialized(got.remaining());
-                    uninitBb.put(got);
-                    uninitBb.flip();
-                    return UnsafeByteOperations.unsafeWrap(uninitBb);
-                });
-                return ret;
-            }
-
-            @Override
-            public long id() {
-                assert !_closed;
-                return _id;
-            }
-
-            @Override
-            public void close() {
-                assert !_closed;
-                _closed = true;
-                _txn.unref();
-            }
-        };
-    }
-
-    @Override
-    public Runnable prepareTx(TxManifestRaw names, long txId) {
-        verifyReady();
-        var txn = _env.txnWrite();
-        try {
-            for (var written : names.written()) {
-                var putBb = _db.reserve(txn, written.getKey().toByteBuffer(), written.getValue().size());
-                written.getValue().copyTo(putBb);
-            }
-            for (JObjectKey key : names.deleted()) {
-                _db.delete(txn, key.toByteBuffer());
-            }
-
-            assert txId > readTxId(txn).orElseThrow();
-
-            var bbData = ByteBuffer.allocateDirect(8);
-            bbData.putLong(txId);
-            bbData.flip();
-            _db.put(txn, DB_VER_OBJ_NAME.asReadOnlyBuffer(), bbData);
-        } catch (Throwable t) {
-            txn.close();
-            throw t;
-        }
-        return () -> {
-            try {
-                txn.commit();
-            } finally {
-                txn.close();
-            }
-        };
-    }
-
-    @Override
-    public long getTotalSpace() {
-        verifyReady();
-        return _root.toFile().getTotalSpace();
-    }
-
-    @Override
-    public long getFreeSpace() {
-        verifyReady();
-        return _root.toFile().getFreeSpace();
-    }
-
-    @Override
-    public long getUsableSpace() {
-        verifyReady();
-        return _root.toFile().getUsableSpace();
-    }
-
     private class LmdbKvIterator extends ReversibleKvIterator<JObjectKey, ByteString> {
-        private static final Cleaner CLEANER = Cleaner.create();
         private final RefcountedCloseable<Txn<ByteBuffer>> _txn;
         private final Cursor<ByteBuffer> _cursor;
+        private boolean _hasNext = false;
+
+        private static final Cleaner CLEANER = Cleaner.create();
         private final MutableObject<Boolean> _closed = new MutableObject<>(false);
         //        private final Exception _allocationStacktrace = new Exception();
         private final Exception _allocationStacktrace = null;
-        private boolean _hasNext = false;
 
-        LmdbKvIterator(RefcountedCloseable<Txn<ByteBuffer>> txn, IteratorStart start, JObjectKey key) {
-            _txn = txn;
+        LmdbKvIterator(IteratorStart start, JObjectKey key) {
             _goingForward = true;
 
+            _lock.readLock().lock();
+            try {
+                var got = _curReadTxn.get();
+                var refInc = Optional.ofNullable(got).map(RefcountedCloseable::ref).orElse(null);
+                if (refInc != null) {
+                    _txn = got;
+                } else {
+                    var newTxn = new RefcountedCloseable<>(_env.txnRead());
+                    _curReadTxn.compareAndSet(got, newTxn);
+                    _txn = newTxn;
+                }
+            } finally {
+                _lock.readLock().unlock();
+            }
             _cursor = _db.openCursor(_txn.get());
 
             var closedRef = _closed;
@@ -231,20 +155,7 @@ public class LmdbObjectPersistentStore implements ObjectPersistentStore {
             });
 
             verifyReady();
-
-            if (key instanceof JObjectKeyMin) {
-                _hasNext = _cursor.first();
-                return;
-            } else if (key instanceof JObjectKeyMax) {
-                _hasNext = _cursor.last();
-                return;
-            }
-
-
-            if (key.toByteBuffer().remaining() == 0) {
-                if (!_cursor.first())
-                    return;
-            } else if (!_cursor.get(key.toByteBuffer(), GetOp.MDB_SET_RANGE)) {
+            if (!_cursor.get(key.toByteBuffer(), GetOp.MDB_SET_RANGE)) {
                 return;
             }
 
@@ -374,6 +285,91 @@ public class LmdbObjectPersistentStore implements ObjectPersistentStore {
             Log.tracev("Read: {0}, hasNext: {1}", ret, _hasNext);
             return ret;
         }
+
+        @Override
+        protected Class<? extends ByteString> peekTypeImpl() {
+            if (!_hasNext)
+                throw new NoSuchElementException();
+
+            return ByteString.class;
+        }
+    }
+
+    @Override
+    public CloseableKvIterator<JObjectKey, ByteString> getIterator(IteratorStart start, JObjectKey key) {
+        return new KeyPredicateKvIterator<>(new LmdbKvIterator(start, key), start, key, (k) -> !Arrays.equals(k.name().getBytes(StandardCharsets.UTF_8), DB_VER_OBJ_NAME));
+    }
+
+    @Override
+    public void commitTx(TxManifestRaw names, long txId, Consumer<Runnable> commitLocked) {
+        verifyReady();
+        try (Txn<ByteBuffer> txn = _env.txnWrite()) {
+            for (var written : names.written()) {
+                // TODO:
+                var bb = UninitializedByteBuffer.allocateUninitialized(written.getValue().size());
+                bb.put(written.getValue().asReadOnlyByteBuffer());
+                bb.flip();
+                _db.put(txn, written.getKey().toByteBuffer(), bb);
+            }
+            for (JObjectKey key : names.deleted()) {
+                _db.delete(txn, key.toByteBuffer());
+            }
+
+            var bb = ByteBuffer.allocateDirect(DB_VER_OBJ_NAME.length);
+            bb.put(DB_VER_OBJ_NAME);
+            bb.flip();
+            var bbData = ByteBuffer.allocateDirect(8);
+
+            commitLocked.accept(() -> {
+                _lock.writeLock().lock();
+                try {
+                    var realTxId = txId;
+                    if (realTxId == -1)
+                        realTxId = _lastTxId + 1;
+
+                    assert realTxId > _lastTxId;
+                    _lastTxId = realTxId;
+
+                    bbData.putLong(realTxId);
+                    bbData.flip();
+                    _db.put(txn, bb, bbData);
+
+                    _curReadTxn.set(null);
+
+                    txn.commit();
+                } finally {
+                    _lock.writeLock().unlock();
+                }
+            });
+        }
+    }
+
+    @Override
+    public long getTotalSpace() {
+        verifyReady();
+        return _root.toFile().getTotalSpace();
+    }
+
+    @Override
+    public long getFreeSpace() {
+        verifyReady();
+        return _root.toFile().getFreeSpace();
+    }
+
+    @Override
+    public long getUsableSpace() {
+        verifyReady();
+        return _root.toFile().getUsableSpace();
+    }
+
+    @Override
+    public long getLastCommitId() {
+        _lock.readLock().lock();
+        try {
+            return _lastTxId;
+        } finally {
+            _lock.readLock().unlock();
+        }
     }
 
 }

dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/persistence/MemoryObjectPersistentStore.java

@@ -0,0 +1,90 @@
+package com.usatiuk.dhfs.objects.persistence;
+
+import com.google.protobuf.ByteString;
+import com.usatiuk.dhfs.objects.CloseableKvIterator;
+import com.usatiuk.dhfs.objects.JObjectKey;
+import com.usatiuk.dhfs.objects.NavigableMapKvIterator;
+import io.quarkus.arc.properties.IfBuildProperty;
+import jakarta.enterprise.context.ApplicationScoped;
+
+import javax.annotation.Nonnull;
+import java.util.Collection;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.function.Consumer;
+
+@ApplicationScoped
+@IfBuildProperty(name = "dhfs.objects.persistence", stringValue = "memory")
+public class MemoryObjectPersistentStore implements ObjectPersistentStore {
+    private final ConcurrentSkipListMap<JObjectKey, ByteString> _objects = new ConcurrentSkipListMap<>();
+    private long _lastCommitId = 0;
+    private final ReentrantReadWriteLock _lock = new ReentrantReadWriteLock();
+
+    @Nonnull
+    @Override
+    public Collection<JObjectKey> findAllObjects() {
+        synchronized (this) {
+            return _objects.keySet();
+        }
+    }
+
+    @Nonnull
+    @Override
+    public Optional<ByteString> readObject(JObjectKey name) {
+        synchronized (this) {
+            return Optional.ofNullable(_objects.get(name));
+        }
+    }
+
+    @Override
+    public CloseableKvIterator<JObjectKey, ByteString> getIterator(IteratorStart start, JObjectKey key) {
+        return new NavigableMapKvIterator<>(_objects, start, key);
+    }
+
+    @Override
+    public void commitTx(TxManifestRaw names, long txId, Consumer<Runnable> commitLocked) {
+        synchronized (this) {
+            for (var written : names.written()) {
+                _objects.put(written.getKey(), written.getValue());
+            }
+            for (JObjectKey key : names.deleted()) {
+                _objects.remove(key);
+            }
+            commitLocked.accept(() -> {
+                _lock.writeLock().lock();
+                try {
+                    assert txId > _lastCommitId;
+                    _lastCommitId = txId;
+                } finally {
+                    _lock.writeLock().unlock();
+                }
+            });
+        }
+    }
+
+    @Override
+    public long getTotalSpace() {
+        return 0;
+    }
+
+    @Override
+    public long getFreeSpace() {
+        return 0;
+    }
+
+    @Override
+    public long getUsableSpace() {
+        return 0;
+    }
+
+    @Override
+    public long getLastCommitId() {
+        _lock.readLock().lock();
+        try {
+            return _lastCommitId;
+        } finally {
+            _lock.readLock().unlock();
+        }
+    }
+}