diff --git a/dhfs-parent/kleppmanntree/pom.xml b/dhfs-parent/kleppmanntree/pom.xml index 72f83688..bf458fd3 100644 --- a/dhfs-parent/kleppmanntree/pom.xml +++ b/dhfs-parent/kleppmanntree/pom.xml @@ -35,5 +35,9 @@ org.pcollections pcollections + + jakarta.annotation + jakarta.annotation-api + \ No newline at end of file diff --git a/dhfs-parent/kleppmanntree/src/main/java/com/usatiuk/kleppmanntree/KleppmannTree.java b/dhfs-parent/kleppmanntree/src/main/java/com/usatiuk/kleppmanntree/KleppmannTree.java index f1f238b1..f39d4bb6 100644 --- a/dhfs-parent/kleppmanntree/src/main/java/com/usatiuk/kleppmanntree/KleppmannTree.java +++ b/dhfs-parent/kleppmanntree/src/main/java/com/usatiuk/kleppmanntree/KleppmannTree.java @@ -1,5 +1,7 @@ package com.usatiuk.kleppmanntree; +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; import org.apache.commons.lang3.tuple.Pair; import java.util.*; @@ -53,18 +55,20 @@ public class KleppmannTree, PeerIdT ex var node = _storage.getById(effect.childId()); var curParent = _storage.getById(effect.newParentId()); { - var newCurParentChildren = curParent.children().minus(node.meta().getName()); + var newCurParentChildren = curParent.children().minus(node.name()); curParent = curParent.withChildren(newCurParentChildren); _storage.putNode(curParent); } - if (!node.meta().getClass().equals(effect.oldInfo().oldMeta().getClass())) + if (effect.oldInfo().oldMeta() != null + && node.meta() != null + && !node.meta().getClass().equals(effect.oldInfo().oldMeta().getClass())) throw new IllegalArgumentException("Class mismatch for meta for node " + node.key()); // Needs to be read after changing curParent, as it might be the same node var oldParent = _storage.getById(effect.oldInfo().oldParent()); { - var newOldParentChildren = oldParent.children().plus(effect.oldInfo().oldMeta().getName(), node.key()); + var newOldParentChildren = oldParent.children().plus(effect.oldName(), node.key()); oldParent = oldParent.withChildren(newOldParentChildren); _storage.putNode(oldParent); } @@ -77,7 +81,7 @@ public class KleppmannTree, PeerIdT ex var node = _storage.getById(effect.childId()); var curParent = _storage.getById(effect.newParentId()); { - var newCurParentChildren = curParent.children().minus(node.meta().getName()); + var newCurParentChildren = curParent.children().minus(node.name()); curParent = curParent.withChildren(newCurParentChildren); _storage.putNode(curParent); } @@ -141,8 +145,8 @@ public class KleppmannTree, PeerIdT ex } } } - } + if (!inTrash.isEmpty()) { var trash = _storage.getById(_storage.getTrashId()); for (var n : inTrash) { @@ -307,7 +311,7 @@ public class KleppmannTree, PeerIdT ex node = _storage.getById(effect.childId()); } if (oldParentNode != null) { - var newOldParentChildren = oldParentNode.children().minus(effect.oldInfo().oldMeta().getName()); + var newOldParentChildren = oldParentNode.children().minus(effect.oldName()); oldParentNode = oldParentNode.withChildren(newOldParentChildren); _storage.putNode(oldParentNode); } @@ -316,12 +320,12 @@ public class KleppmannTree, PeerIdT ex newParentNode = _storage.getById(effect.newParentId()); { - var newNewParentChildren = newParentNode.children().plus(effect.newMeta().getName(), effect.childId()); + var newNewParentChildren = newParentNode.children().plus(effect.newName(), effect.childId()); newParentNode = newParentNode.withChildren(newNewParentChildren); _storage.putNode(newParentNode); } if (effect.newParentId().equals(_storage.getTrashId()) && - !Objects.equals(effect.newMeta().getName(), effect.childId().toString())) + !Objects.equals(effect.newName(), effect.childId().toString())) throw new IllegalArgumentException("Move to trash should have id of node as name"); _storage.putNode( node.withParent(effect.newParentId()) @@ -338,17 +342,32 @@ public class KleppmannTree, PeerIdT ex NodeIdT newParentId = op.newParentId(); TreeNode newParent = _storage.getById(newParentId); + if (newParent == null) { - LOGGER.log(Level.SEVERE, "New parent not found " + op.newMeta().getName() + " " + op.childId()); - return new LogRecord<>(op, null); + LOGGER.log(Level.SEVERE, "New parent not found " + op.newName() + " " + op.childId()); + + // Creation + if (oldParentId == null) { + LOGGER.severe(() -> "Creating both dummy parent and child node"); + return new LogRecord<>(op, List.of( + new LogEffect<>(null, op, _storage.getLostFoundId(), null, newParentId), + new LogEffect<>(null, op, newParentId, op.newMeta(), op.childId()) + )); + } else { + LOGGER.severe(() -> "Moving child node to dummy parent"); + return new LogRecord<>(op, List.of( + new LogEffect<>(null, op, _storage.getLostFoundId(), null, newParentId), + new LogEffect<>(new LogEffectOld<>(node.lastEffectiveOp(), oldParentId, node.meta()), op, op.newParentId(), op.newMeta(), op.childId()) + )); + } } if (oldParentId == null) { - var conflictNodeId = newParent.children().get(op.newMeta().getName()); + var conflictNodeId = newParent.children().get(op.newName()); if (conflictNodeId != null) { if (failCreatingIfExists) - throw new AlreadyExistsException("Already exists: " + op.newMeta().getName() + ": " + conflictNodeId); + throw new AlreadyExistsException("Already exists: " + op.newName() + ": " + conflictNodeId); var conflictNode = _storage.getById(conflictNodeId); MetaT conflictNodeMeta = conflictNode.meta(); @@ -359,8 +378,8 @@ public class KleppmannTree, PeerIdT ex LOGGER.finer(() -> "Node creation conflict: " + conflictNode); - String newConflictNodeName = conflictNodeMeta.getName() + ".conflict." + conflictNode.key(); - String newOursName = op.newMeta().getName() + ".conflict." + op.childId(); + String newConflictNodeName = op.newName() + ".conflict." + conflictNode.key(); + String newOursName = op.newName() + ".conflict." + op.childId(); return new LogRecord<>(op, List.of( new LogEffect<>(new LogEffectOld<>(conflictNode.lastEffectiveOp(), newParentId, conflictNodeMeta), conflictNode.lastEffectiveOp(), newParentId, (MetaT) conflictNodeMeta.withName(newConflictNodeName), conflictNodeId), new LogEffect<>(null, op, op.newParentId(), (MetaT) op.newMeta().withName(newOursName), op.childId()) @@ -378,11 +397,13 @@ public class KleppmannTree, PeerIdT ex } MetaT oldMeta = node.meta(); - if (!oldMeta.getClass().equals(op.newMeta().getClass())) { + if (oldMeta != null + && op.newMeta() != null + && !oldMeta.getClass().equals(op.newMeta().getClass())) { LOGGER.log(Level.SEVERE, "Class mismatch for meta for node " + node.key()); return new LogRecord<>(op, null); } - var replaceNodeId = newParent.children().get(op.newMeta().getName()); + var replaceNodeId = newParent.children().get(op.newName()); if (replaceNodeId != null) { var replaceNode = _storage.getById(replaceNodeId); var replaceNodeMeta = replaceNode.meta(); @@ -454,18 +475,18 @@ public class KleppmannTree, PeerIdT ex walkTree(node -> { var op = node.lastEffectiveOp(); if (node.lastEffectiveOp() == null) return; - LOGGER.info("visited bootstrap op for " + host + ": " + op.timestamp().toString() + " " + op.newMeta().getName() + " " + op.childId() + "->" + op.newParentId()); + LOGGER.info("visited bootstrap op for " + host + ": " + op.timestamp().toString() + " " + op.newName() + " " + op.childId() + "->" + op.newParentId()); result.put(node.lastEffectiveOp().timestamp(), node.lastEffectiveOp()); }); for (var le : _storage.getLog().getAll()) { var op = le.getValue().op(); - LOGGER.info("bootstrap op from log for " + host + ": " + op.timestamp().toString() + " " + op.newMeta().getName() + " " + op.childId() + "->" + op.newParentId()); + LOGGER.info("bootstrap op from log for " + host + ": " + op.timestamp().toString() + " " + op.newName() + " " + op.childId() + "->" + op.newParentId()); result.put(le.getKey(), le.getValue().op()); } for (var op : result.values()) { - LOGGER.info("Recording bootstrap op for " + host + ": " + op.timestamp().toString() + " " + op.newMeta().getName() + " " + op.childId() + "->" + op.newParentId()); + LOGGER.info("Recording bootstrap op for " + host + ": " + op.timestamp().toString() + " " + op.newName() + " " + op.childId() + "->" + op.newParentId()); _opRecorder.recordOpForPeer(host, op); } } diff --git a/dhfs-parent/kleppmanntree/src/main/java/com/usatiuk/kleppmanntree/LogEffect.java b/dhfs-parent/kleppmanntree/src/main/java/com/usatiuk/kleppmanntree/LogEffect.java index 0fe9a95f..6af98a98 100644 --- a/dhfs-parent/kleppmanntree/src/main/java/com/usatiuk/kleppmanntree/LogEffect.java +++ b/dhfs-parent/kleppmanntree/src/main/java/com/usatiuk/kleppmanntree/LogEffect.java @@ -8,4 +8,17 @@ public record LogEffect, PeerIdT exten NodeIdT newParentId, MetaT newMeta, NodeIdT childId) implements Serializable { + public String oldName() { + if (oldInfo.oldMeta() != null) { + return oldInfo.oldMeta().getName(); + } + return childId.toString(); + } + + public String newName() { + if (newMeta != null) { + return newMeta.getName(); + } + return childId.toString(); + } } diff --git a/dhfs-parent/kleppmanntree/src/main/java/com/usatiuk/kleppmanntree/OpMove.java b/dhfs-parent/kleppmanntree/src/main/java/com/usatiuk/kleppmanntree/OpMove.java index 85b7f383..b04bf8ca 100644 --- a/dhfs-parent/kleppmanntree/src/main/java/com/usatiuk/kleppmanntree/OpMove.java +++ b/dhfs-parent/kleppmanntree/src/main/java/com/usatiuk/kleppmanntree/OpMove.java @@ -5,4 +5,9 @@ import java.io.Serializable; public record OpMove, PeerIdT extends Comparable, MetaT extends NodeMeta, NodeIdT> (CombinedTimestamp timestamp, NodeIdT newParentId, MetaT newMeta, NodeIdT childId) implements Serializable { + public String newName() { + if (newMeta != null) + return newMeta.getName(); + return childId.toString(); + } } diff --git a/dhfs-parent/kleppmanntree/src/main/java/com/usatiuk/kleppmanntree/StorageInterface.java b/dhfs-parent/kleppmanntree/src/main/java/com/usatiuk/kleppmanntree/StorageInterface.java index af55b35b..763dccb8 100644 --- a/dhfs-parent/kleppmanntree/src/main/java/com/usatiuk/kleppmanntree/StorageInterface.java +++ b/dhfs-parent/kleppmanntree/src/main/java/com/usatiuk/kleppmanntree/StorageInterface.java @@ -9,6 +9,8 @@ public interface StorageInterface< NodeIdT getTrashId(); + NodeIdT getLostFoundId(); + NodeIdT getNewNodeId(); TreeNode getById(NodeIdT id); diff --git a/dhfs-parent/kleppmanntree/src/main/java/com/usatiuk/kleppmanntree/TreeNode.java b/dhfs-parent/kleppmanntree/src/main/java/com/usatiuk/kleppmanntree/TreeNode.java index 22137a65..d5a4627a 100644 --- a/dhfs-parent/kleppmanntree/src/main/java/com/usatiuk/kleppmanntree/TreeNode.java +++ b/dhfs-parent/kleppmanntree/src/main/java/com/usatiuk/kleppmanntree/TreeNode.java @@ -1,5 +1,6 @@ package com.usatiuk.kleppmanntree; +import jakarta.annotation.Nullable; import org.pcollections.PMap; import java.io.Serializable; @@ -11,8 +12,15 @@ public interface TreeNode, PeerIdT ext OpMove lastEffectiveOp(); + @Nullable MetaT meta(); + default String name() { + var meta = meta(); + if (meta != null) return meta.getName(); + return key().toString(); + } + PMap children(); TreeNode withParent(NodeIdT parent); diff --git a/dhfs-parent/kleppmanntree/src/test/java/com/usatiuk/kleppmanntree/KleppmanTreeSimpleTest.java b/dhfs-parent/kleppmanntree/src/test/java/com/usatiuk/kleppmanntree/KleppmanTreeSimpleTest.java index 598f4d7d..46f71686 100644 --- a/dhfs-parent/kleppmanntree/src/test/java/com/usatiuk/kleppmanntree/KleppmanTreeSimpleTest.java +++ b/dhfs-parent/kleppmanntree/src/test/java/com/usatiuk/kleppmanntree/KleppmanTreeSimpleTest.java @@ -147,4 +147,19 @@ public class KleppmanTreeSimpleTest { var r1 = testNode1.getRecorded(); Assertions.assertEquals(1, r1.size()); } + + @Test + void externalOpWithDummy() { + Long d1id = testNode1._storageInterface.getNewNodeId(); + Long f1id = testNode1._storageInterface.getNewNodeId(); + + testNode1._tree.applyExternalOp(2L, new OpMove<>( + new CombinedTimestamp<>(2L, 2L), d1id, new TestNodeMetaFile("Hi", 123), f1id + )); + testNode1._tree.applyExternalOp(2L, new OpMove<>( + new CombinedTimestamp<>(3L, 2L), testNode1._storageInterface.getRootId(), new TestNodeMetaDir("HiDir"), d1id + )); + + Assertions.assertEquals(f1id, testNode1._tree.traverse(List.of("HiDir", "Hi"))); + } } diff --git a/dhfs-parent/kleppmanntree/src/test/java/com/usatiuk/kleppmanntree/TestStorageInterface.java b/dhfs-parent/kleppmanntree/src/test/java/com/usatiuk/kleppmanntree/TestStorageInterface.java index 415f146a..bb041897 100644 --- a/dhfs-parent/kleppmanntree/src/test/java/com/usatiuk/kleppmanntree/TestStorageInterface.java +++ b/dhfs-parent/kleppmanntree/src/test/java/com/usatiuk/kleppmanntree/TestStorageInterface.java @@ -14,6 +14,7 @@ public class TestStorageInterface implements StorageInterface refsFrom, public Collection collectRefsTo() { return Stream.concat(children().values().stream(), switch (meta()) { - case JKleppmannTreeNodeMetaDirectory dir -> Stream.of(); + case JKleppmannTreeNodeMetaDirectory dir -> Stream.empty(); case JKleppmannTreeNodeMetaFile file -> Stream.of(file.getFileIno()); case JKleppmannTreeNodeMetaPeer peer -> Stream.of(peer.getPeerId()); + case null -> Stream.empty(); default -> throw new IllegalStateException("Unexpected value: " + meta()); } ).collect(Collectors.toUnmodifiableSet()); diff --git a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jkleppmanntree/structs/JKleppmannTreePersistentData.java b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jkleppmanntree/structs/JKleppmannTreePersistentData.java index e31dc73c..a363cd46 100644 --- a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jkleppmanntree/structs/JKleppmannTreePersistentData.java +++ b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jkleppmanntree/structs/JKleppmannTreePersistentData.java @@ -49,6 +49,6 @@ public record JKleppmannTreePersistentData( @Override public Collection collectRefsTo() { - return List.of(new JObjectKey(key().name() + "_jt_trash"), new JObjectKey(key().name() + "_jt_root")); + return List.of(new JObjectKey(key().name() + "_jt_trash"), new JObjectKey(key().name() + "_jt_root"), new JObjectKey(key().name() + "_jt_lf")); } } diff --git a/dhfs-parent/server/src/main/resources/application.properties b/dhfs-parent/server/src/main/resources/application.properties index 4f499e77..85807041 100644 --- a/dhfs-parent/server/src/main/resources/application.properties +++ b/dhfs-parent/server/src/main/resources/application.properties @@ -34,7 +34,7 @@ dhfs.objects.opsender.batch-size=100 dhfs.objects.lock_timeout_secs=2 dhfs.local-discovery=true dhfs.peerdiscovery.timeout=5000 -quarkus.log.category."com.usatiuk.dhfs".min-level=TRACE -quarkus.log.category."com.usatiuk.dhfs".level=TRACE +quarkus.log.category."com.usatiuk".min-level=TRACE +quarkus.log.category."com.usatiuk".level=TRACE quarkus.http.insecure-requests=enabled quarkus.http.ssl.client-auth=required diff --git a/dhfs-parent/server/src/test/java/com/usatiuk/dhfs/integration/ResyncIT.java b/dhfs-parent/server/src/test/java/com/usatiuk/dhfs/integration/ResyncIT.java index c23d7c7c..f3cbfc08 100644 --- a/dhfs-parent/server/src/test/java/com/usatiuk/dhfs/integration/ResyncIT.java +++ b/dhfs-parent/server/src/test/java/com/usatiuk/dhfs/integration/ResyncIT.java @@ -133,4 +133,39 @@ public class ResyncIT { }); } + @Test + void folderAfterMove() throws IOException, InterruptedException, TimeoutException { + await().atMost(45, TimeUnit.SECONDS).until(() -> 0 == container1.execInContainer("/bin/sh", "-c", "mkdir /root/dhfs_default/fuse/testd1").getExitCode()); + await().atMost(45, TimeUnit.SECONDS).until(() -> 0 == container1.execInContainer("/bin/sh", "-c", "echo tesempty1 > /root/dhfs_default/fuse/testd1/testf1").getExitCode()); + await().atMost(45, TimeUnit.SECONDS).until(() -> 0 == container1.execInContainer("/bin/sh", "-c", "mv /root/dhfs_default/fuse/testd1 /root/dhfs_default/fuse/testd2").getExitCode()); + await().atMost(45, TimeUnit.SECONDS).until(() -> 0 == container1.execInContainer("/bin/sh", "-c", "echo tesempty2 > /root/dhfs_default/fuse/testd2/testf2").getExitCode()); + + c1uuid = container1.execInContainer("/bin/sh", "-c", "cat /root/dhfs_default/data/stuff/self_uuid").getStdout(); + c2uuid = container2.execInContainer("/bin/sh", "-c", "cat /root/dhfs_default/data/stuff/self_uuid").getStdout(); + + Assertions.assertDoesNotThrow(() -> UUID.fromString(c1uuid)); + Assertions.assertDoesNotThrow(() -> UUID.fromString(c2uuid)); + + waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("New address"), 60, TimeUnit.SECONDS); + waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("New address"), 60, TimeUnit.SECONDS); + + var c1curl = container1.execInContainer("/bin/sh", "-c", + "curl --header \"Content-Type: application/json\" " + + " --request PUT " + + " --data '{\"uuid\":\"" + c2uuid + "\"}' " + + " http://localhost:8080/objects-manage/known-peers"); + + var c2curl = container2.execInContainer("/bin/sh", "-c", + "curl --header \"Content-Type: application/json\" " + + " --request PUT " + + " --data '{\"uuid\":\"" + c1uuid + "\"}' " + + " http://localhost:8080/objects-manage/known-peers"); + + waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS); + waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS); + + await().atMost(45, TimeUnit.SECONDS).until(() -> "tesempty1\n".equals(container2.execInContainer("/bin/sh", "-c", "cat /root/dhfs_default/fuse/testd2/testf1").getStdout())); + await().atMost(45, TimeUnit.SECONDS).until(() -> "tesempty2\n".equals(container2.execInContainer("/bin/sh", "-c", "cat /root/dhfs_default/fuse/testd2/testf2").getStdout())); + } + }