diff --git a/dhfs-parent/autoprotomap/integration-tests/src/main/resources/application.properties b/dhfs-parent/autoprotomap/integration-tests/src/main/resources/application.properties index b1645fe9..e69de29b 100644 --- a/dhfs-parent/autoprotomap/integration-tests/src/main/resources/application.properties +++ b/dhfs-parent/autoprotomap/integration-tests/src/main/resources/application.properties @@ -1 +0,0 @@ -quarkus.package.jar.decompiler.enabled=true \ No newline at end of file diff --git a/dhfs-parent/objects/src/main/resources/application.properties b/dhfs-parent/objects/src/main/resources/application.properties index ce078310..5ec5eb9a 100644 --- a/dhfs-parent/objects/src/main/resources/application.properties +++ b/dhfs-parent/objects/src/main/resources/application.properties @@ -4,7 +4,6 @@ dhfs.objects.lru.limit=134217728 dhfs.objects.lru.print-stats=true dhfs.objects.lock_timeout_secs=15 dhfs.objects.persistence.files.root=${HOME}/dhfs_default/data/objs -quarkus.package.jar.decompiler.enabled=true dhfs.objects.persistence.snapshot-extra-checks=false dhfs.objects.transaction.never-lock=true quarkus.log.category."com.usatiuk.dhfs.objects.iterators".level=INFO diff --git a/dhfs-parent/server/pom.xml b/dhfs-parent/server/pom.xml index 9206ef27..226c9ccd 100644 --- a/dhfs-parent/server/pom.xml +++ b/dhfs-parent/server/pom.xml @@ -166,6 +166,11 @@ 1C false classes + + + false + + diff --git a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/JDataRefcounted.java b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/JDataRefcounted.java index 399ea19e..0626fda1 100644 --- a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/JDataRefcounted.java +++ b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/JDataRefcounted.java @@ -14,7 +14,7 @@ public interface JDataRefcounted extends JData { JDataRefcounted withFrozen(boolean frozen); - default Collection collectRefsTo() { + default Collection collectRefsTo() { return List.of(); } } diff --git a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/RefcounterTxHook.java b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/RefcounterTxHook.java index fb277842..157836c6 100644 --- a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/RefcounterTxHook.java +++ b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/RefcounterTxHook.java @@ -33,16 +33,16 @@ public class RefcounterTxHook implements PreCommitTxHook { for (var curRef : curRefs) { if (!oldRefs.contains(curRef)) { - var referenced = getRef(curRef.obj()); - curTx.put(referenced.withRefsFrom(referenced.refsFrom().plus(new JDataNormalRef(curRef.obj())))); + var referenced = getRef(curRef); + curTx.put(referenced.withRefsFrom(referenced.refsFrom().plus(new JDataNormalRef(key)))); Log.tracev("Added ref from {0} to {1}", key, curRef); } } for (var oldRef : oldRefs) { if (!curRefs.contains(oldRef)) { - var referenced = getRef(oldRef.obj()); - curTx.put(referenced.withRefsFrom(referenced.refsFrom().minus(new JDataNormalRef(oldRef.obj())))); + var referenced = getRef(oldRef); + curTx.put(referenced.withRefsFrom(referenced.refsFrom().minus(new JDataNormalRef(key)))); Log.tracev("Removed ref from {0} to {1}", key, oldRef); } } @@ -55,8 +55,8 @@ public class RefcounterTxHook implements PreCommitTxHook { } for (var newRef : refCur.collectRefsTo()) { - var referenced = getRef(newRef.obj()); - curTx.put(referenced.withRefsFrom(referenced.refsFrom().plus(new JDataNormalRef(newRef.obj())))); + var referenced = getRef(newRef); + curTx.put(referenced.withRefsFrom(referenced.refsFrom().plus(new JDataNormalRef(key)))); Log.tracev("Added ref from {0} to {1}", key, newRef); } } @@ -68,8 +68,8 @@ public class RefcounterTxHook implements PreCommitTxHook { } for (var removedRef : refCur.collectRefsTo()) { - var referenced = getRef(removedRef.obj()); - curTx.put(referenced.withRefsFrom(referenced.refsFrom().minus(new JDataNormalRef(removedRef.obj())))); + var referenced = getRef(removedRef); + curTx.put(referenced.withRefsFrom(referenced.refsFrom().minus(new JDataNormalRef(key)))); Log.tracev("Removed ref from {0} to {1}", key, removedRef); } } diff --git a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/RemoteObjectDataWrapper.java b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/RemoteObjectDataWrapper.java index a70b0404..955ec26c 100644 --- a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/RemoteObjectDataWrapper.java +++ b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/RemoteObjectDataWrapper.java @@ -32,8 +32,8 @@ public record RemoteObjectDataWrapper(PCollection collectRefsTo() { - return data.collectRefsTo().stream().map(JDataNormalRef::new).toList(); + public Collection collectRefsTo() { + return data.collectRefsTo(); } @Override diff --git a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/RemoteObjectMeta.java b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/RemoteObjectMeta.java index 1b97646e..91946446 100644 --- a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/RemoteObjectMeta.java +++ b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/RemoteObjectMeta.java @@ -91,8 +91,8 @@ public record RemoteObjectMeta(PCollection refsFrom, boolean frozen, } @Override - public Collection collectRefsTo() { - if (hasLocalData) return List.of(new JDataNormalRef(dataKey())); + public Collection collectRefsTo() { + if (hasLocalData) return List.of(dataKey()); return List.of(); } diff --git a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jkleppmanntree/structs/JKleppmannTreeNode.java b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jkleppmanntree/structs/JKleppmannTreeNode.java index 4636e39c..2251f30b 100644 --- a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jkleppmanntree/structs/JKleppmannTreeNode.java +++ b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jkleppmanntree/structs/JKleppmannTreeNode.java @@ -1,6 +1,9 @@ package com.usatiuk.dhfs.objects.jkleppmanntree.structs; -import com.usatiuk.dhfs.objects.*; +import com.usatiuk.dhfs.objects.JDataRef; +import com.usatiuk.dhfs.objects.JDataRefcounted; +import com.usatiuk.dhfs.objects.JObjectKey; +import com.usatiuk.dhfs.objects.PeerId; import com.usatiuk.dhfs.objects.repository.peersync.structs.JKleppmannTreeNodeMetaPeer; import com.usatiuk.kleppmanntree.OpMove; import com.usatiuk.kleppmanntree.TreeNode; @@ -55,12 +58,12 @@ public record JKleppmannTreeNode(JObjectKey key, PCollection refsFrom, } @Override - public Collection collectRefsTo() { - return Stream.concat(children().values().stream().map(JDataNormalRef::new), + public Collection collectRefsTo() { + return Stream.concat(children().values().stream(), switch (meta()) { - case JKleppmannTreeNodeMetaDirectory dir -> Stream.of(); - case JKleppmannTreeNodeMetaFile file -> Stream.of(new JDataNormalRef(file.getFileIno())); - case JKleppmannTreeNodeMetaPeer peer -> Stream.of(new JDataNormalRef(peer.getPeerId())); + case JKleppmannTreeNodeMetaDirectory dir -> Stream.of(); + case JKleppmannTreeNodeMetaFile file -> Stream.of(file.getFileIno()); + case JKleppmannTreeNodeMetaPeer peer -> Stream.of(peer.getPeerId()); default -> throw new IllegalStateException("Unexpected value: " + meta()); } ).collect(Collectors.toUnmodifiableSet()); diff --git a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jkleppmanntree/structs/JKleppmannTreePersistentData.java b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jkleppmanntree/structs/JKleppmannTreePersistentData.java index a08fe5c4..e31dc73c 100644 --- a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jkleppmanntree/structs/JKleppmannTreePersistentData.java +++ b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jkleppmanntree/structs/JKleppmannTreePersistentData.java @@ -1,6 +1,9 @@ package com.usatiuk.dhfs.objects.jkleppmanntree.structs; -import com.usatiuk.dhfs.objects.*; +import com.usatiuk.dhfs.objects.JDataRef; +import com.usatiuk.dhfs.objects.JDataRefcounted; +import com.usatiuk.dhfs.objects.JObjectKey; +import com.usatiuk.dhfs.objects.PeerId; import com.usatiuk.kleppmanntree.CombinedTimestamp; import com.usatiuk.kleppmanntree.LogRecord; import com.usatiuk.kleppmanntree.OpMove; @@ -45,8 +48,7 @@ public record JKleppmannTreePersistentData( } @Override - public Collection collectRefsTo() { - return List.of(new JObjectKey(key().name() + "_jt_trash"), new JObjectKey(key().name() + "_jt_root")) - .stream().map(JDataNormalRef::new).toList(); + public Collection collectRefsTo() { + return List.of(new JObjectKey(key().name() + "_jt_trash"), new JObjectKey(key().name() + "_jt_root")); } } diff --git a/dhfs-parent/server/src/test/java/com/usatiuk/dhfs/RefcounterTest.java b/dhfs-parent/server/src/test/java/com/usatiuk/dhfs/RefcounterTest.java new file mode 100644 index 00000000..b3683ce0 --- /dev/null +++ b/dhfs-parent/server/src/test/java/com/usatiuk/dhfs/RefcounterTest.java @@ -0,0 +1,69 @@ +package com.usatiuk.dhfs; + +import com.usatiuk.dhfs.objects.JDataRef; +import com.usatiuk.dhfs.objects.JObjectKey; +import com.usatiuk.dhfs.objects.testobjs.TestRefcount; +import com.usatiuk.dhfs.objects.transaction.Transaction; +import com.usatiuk.dhfs.objects.transaction.TransactionManager; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.TestProfile; +import jakarta.inject.Inject; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.pcollections.HashTreePSet; + +import java.util.List; +import java.util.Map; + +class Profiles { + public static class RefcounterTestProfile extends TempDataProfile { + @Override + protected void getConfigOverrides(Map ret) { + ret.put("quarkus.log.category.\"com.usatiuk.dhfs\".level", "INFO"); + ret.put("dhfs.fuse.enabled", "false"); + ret.put("dhfs.objects.ref_verification", "false"); + } + } +} + +@QuarkusTest +@TestProfile(Profiles.RefcounterTestProfile.class) +public class RefcounterTest { + + @Inject + Transaction curTx; + @Inject + TransactionManager txm; + + @Test + void refcountParentChange() { + final JObjectKey PARENT_1_KEY = JObjectKey.of("refcountParentChange_parent1"); + final JObjectKey PARENT_2_KEY = JObjectKey.of("refcountParentChange_parent2"); + final JObjectKey CHILD_KEY = JObjectKey.of("refcountParentChange_child"); + + txm.run(() -> { + curTx.put((new TestRefcount(PARENT_1_KEY)).withFrozen(true)); + curTx.put((new TestRefcount(PARENT_2_KEY)).withFrozen(true)); + }); + + txm.run(() -> { + curTx.put((new TestRefcount(CHILD_KEY)).withFrozen(false)); + curTx.put(curTx.get(TestRefcount.class, PARENT_1_KEY).get().withKids(HashTreePSet.empty().plus(CHILD_KEY))); + }); + + txm.run(() -> { + var kid = curTx.get(TestRefcount.class, CHILD_KEY).get(); + Assertions.assertIterableEquals(List.of(PARENT_1_KEY), kid.refsFrom().stream().map(JDataRef::obj).toList()); + }); + + txm.run(() -> { + curTx.put(curTx.get(TestRefcount.class, PARENT_1_KEY).get().withKids(HashTreePSet.empty().minus(CHILD_KEY))); + curTx.put(curTx.get(TestRefcount.class, PARENT_2_KEY).get().withKids(HashTreePSet.empty().plus(CHILD_KEY))); + }); + + txm.run(() -> { + Assertions.assertIterableEquals(List.of(PARENT_2_KEY), curTx.get(TestRefcount.class, CHILD_KEY).get().refsFrom().stream().map(JDataRef::obj).toList()); + }); + } + +} diff --git a/dhfs-parent/server/src/test/java/com/usatiuk/dhfs/integration/DhfsFuseIT.java b/dhfs-parent/server/src/test/java/com/usatiuk/dhfs/integration/DhfsFuseIT.java index 192084e5..d247a52c 100644 --- a/dhfs-parent/server/src/test/java/com/usatiuk/dhfs/integration/DhfsFuseIT.java +++ b/dhfs-parent/server/src/test/java/com/usatiuk/dhfs/integration/DhfsFuseIT.java @@ -355,4 +355,82 @@ public class DhfsFuseIT { } + @Test + void removeAndMove() throws IOException, InterruptedException, TimeoutException { + var client = DockerClientFactory.instance().client(); + Log.info("Creating"); + await().atMost(45, TimeUnit.SECONDS).until(() -> 0 == container1.execInContainer("/bin/sh", "-c", "echo tesempty > /root/dhfs_default/fuse/testf1").getExitCode()); + await().atMost(45, TimeUnit.SECONDS).until(() -> "tesempty\n".equals(container1.execInContainer("/bin/sh", "-c", "cat /root/dhfs_default/fuse/testf1").getStdout())); + Log.info("Listing"); + await().atMost(45, TimeUnit.SECONDS).until(() -> 0 == container2.execInContainer("/bin/sh", "-c", "ls /root/dhfs_default/fuse/").getExitCode()); + await().atMost(45, TimeUnit.SECONDS).until(() -> "tesempty\n".equals(container2.execInContainer("/bin/sh", "-c", "cat /root/dhfs_default/fuse/testf1").getStdout())); + + client.pauseContainerCmd(container1.getContainerId()).exec(); + waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Lost connection to"), 60, TimeUnit.SECONDS, 1); + + Log.info("Removing"); + await().atMost(45, TimeUnit.SECONDS).until(() -> 0 == container2.execInContainer("/bin/sh", "-c", "rm /root/dhfs_default/fuse/testf1").getExitCode()); + + client.pauseContainerCmd(container2.getContainerId()).exec(); + client.unpauseContainerCmd(container1.getContainerId()).exec(); + waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Lost connection to"), 60, TimeUnit.SECONDS, 1); + Log.info("Moving"); + await().atMost(45, TimeUnit.SECONDS).until(() -> 0 == container1.execInContainer("/bin/sh", "-c", "mv /root/dhfs_default/fuse/testf1 /root/dhfs_default/fuse/testf2").getExitCode()); + Log.info("Listing"); + await().atMost(45, TimeUnit.SECONDS).until(() -> 0 == container1.execInContainer("/bin/sh", "-c", "ls /root/dhfs_default/fuse/").getExitCode()); + Log.info("Reading"); + await().atMost(45, TimeUnit.SECONDS).until(() -> "tesempty\n".equals(container1.execInContainer("/bin/sh", "-c", "cat /root/dhfs_default/fuse/testf2").getStdout())); + client.unpauseContainerCmd(container2.getContainerId()).exec(); + + waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS, 1); + waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS, 1); + // Either removed, or moved + // TODO: it always seems to be removed? + Log.info("Reading both"); + await().atMost(45, TimeUnit.SECONDS).until(() -> { + var ls1 = container1.execInContainer("/bin/sh", "-c", "ls /root/dhfs_default/fuse/"); + var ls2 = container2.execInContainer("/bin/sh", "-c", "ls /root/dhfs_default/fuse/"); + var cat1 = container1.execInContainer("/bin/sh", "-c", "cat /root/dhfs_default/fuse/*"); + var cat2 = container2.execInContainer("/bin/sh", "-c", "cat /root/dhfs_default/fuse/*"); + Log.info("cat1: " + cat1); + Log.info("cat2: " + cat2); + Log.info("ls1: " + ls1); + Log.info("ls2: " + ls2); + + if (!ls1.getStdout().equals(ls2.getStdout())) { + Log.info("Different ls?"); + return false; + } + + if (ls1.getStdout().trim().isEmpty() && ls2.getStdout().trim().isEmpty()) { + Log.info("Both empty"); + return true; + } + + if (!cat1.getStdout().equals(cat2.getStdout())) { + Log.info("Different cat?"); + return false; + } + + if (!(cat1.getExitCode() == 0 && cat2.getExitCode() == 0 && ls1.getExitCode() == 0 && ls2.getExitCode() == 0)) { + return false; + } + + boolean hasMoved = cat1.getStdout().contains("tesempty") && cat2.getStdout().contains("tesempty") + && ls1.getStdout().contains("testf2") && !ls1.getStdout().contains("testf1") + && ls2.getStdout().contains("testf2") && !ls2.getStdout().contains("testf1"); + + boolean removed = !cat1.getStdout().contains("tesempty") && !cat2.getStdout().contains("tesempty") + && !ls1.getStdout().contains("testf2") && !ls1.getStdout().contains("testf1") + && !ls2.getStdout().contains("testf2") && !ls2.getStdout().contains("testf1"); + + if (hasMoved && removed) { + Log.info("Both removed and moved"); + return false; + } + + return hasMoved || removed; + }); + } + } diff --git a/dhfs-parent/server/src/test/java/com/usatiuk/dhfs/objects/testobjs/TestRefcount.java b/dhfs-parent/server/src/test/java/com/usatiuk/dhfs/objects/testobjs/TestRefcount.java new file mode 100644 index 00000000..dc262d39 --- /dev/null +++ b/dhfs-parent/server/src/test/java/com/usatiuk/dhfs/objects/testobjs/TestRefcount.java @@ -0,0 +1,36 @@ +package com.usatiuk.dhfs.objects.testobjs; + +import com.usatiuk.dhfs.objects.JDataRef; +import com.usatiuk.dhfs.objects.JDataRefcounted; +import com.usatiuk.dhfs.objects.JObjectKey; +import org.pcollections.HashTreePSet; +import org.pcollections.PCollection; + +import java.util.Collection; + +public record TestRefcount(JObjectKey key, PCollection refsFrom, boolean frozen, + PCollection kids) implements JDataRefcounted { + + public TestRefcount(JObjectKey key) { + this(key, HashTreePSet.empty(), false, HashTreePSet.empty()); + } + + @Override + public TestRefcount withRefsFrom(PCollection refs) { + return new TestRefcount(key, refs, frozen, kids); + } + + @Override + public TestRefcount withFrozen(boolean frozen) { + return new TestRefcount(key, refsFrom, frozen, kids); + } + + public TestRefcount withKids(PCollection kids) { + return new TestRefcount(key, refsFrom, frozen, kids); + } + + @Override + public Collection collectRefsTo() { + return kids; + } +}