Dhfs-app: less verbose docker tests

dhfs-parent/dhfs-app/pom.xml

@@ -13,6 +13,11 @@
     </parent>
 
     <dependencies>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-params</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.testcontainers</groupId>
             <artifactId>testcontainers</artifactId>

dhfs-parent/dhfs-app/src/test/java/com/usatiuk/dhfs/integration/DhfsFuseIT.java

@@ -32,9 +32,11 @@ public class DhfsFuseIT {
     String c1uuid;
     String c2uuid;
 
+    Network network;
+
     @BeforeEach
     void setup(TestInfo testInfo) throws IOException, InterruptedException, TimeoutException {
-        Network network = Network.newNetwork();
+        network = Network.newNetwork();
         container1 = new GenericContainer<>(DhfsImage.getInstance())
                 .withPrivilegedMode(true)
                 .withCreateContainerCmdModifier(cmd -> Objects.requireNonNull(cmd.getHostConfig()).withDevices(Device.parse("/dev/fuse")))
@@ -81,6 +83,7 @@ public class DhfsFuseIT {
     @AfterEach
     void stop() {
         Stream.of(container1, container2).parallel().forEach(GenericContainer::stop);
+        network.close();
     }
 
     @Test

dhfs-parent/dhfs-app/src/test/java/com/usatiuk/dhfs/integration/DhfsFusex3IT.java

@@ -133,6 +133,7 @@ public class DhfsFusex3IT {
     @AfterEach
     void stop() {
         Stream.of(container1, container2, container3).parallel().forEach(GenericContainer::stop);
+        network.close();
     }
 
     @Test

dhfs-parent/dhfs-app/src/test/java/com/usatiuk/dhfs/integration/DhfsImage.java

@@ -81,6 +81,7 @@ public class DhfsImage implements Future<String> {
                                         "-Ddhfs.objects.reconnect_interval=1s",
                                         "-Dquarkus.log.category.\"com.usatiuk\".level=TRACE",
                                         "-Dquarkus.log.category.\"com.usatiuk.dhfs\".level=TRACE",
+                                        "-Dquarkus.log.category.\"com.usatiuk.objects.transaction\".level=INFO",
                                         "-Ddhfs.objects.periodic-push-op-interval=5s",
                                         "-Ddhfs.fuse.root=/dhfs_test/fuse",
                                         "-Ddhfs.objects.persistence.files.root=/dhfs_test/data",

dhfs-parent/dhfs-app/src/test/java/com/usatiuk/dhfs/integration/KillIT.java

@@ -18,10 +18,7 @@ import java.nio.file.Files;
 import java.time.Duration;
 import java.util.Objects;
 import java.util.UUID;
-import java.util.concurrent.CyclicBarrier;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
+import java.util.concurrent.*;
 import java.util.stream.Stream;
 
 import static org.awaitility.Awaitility.await;
@@ -39,12 +36,18 @@ public class KillIT {
     File data1;
     File data2;
 
+    Network network;
+
+    ExecutorService executor;
+
     @BeforeEach
     void setup(TestInfo testInfo) throws IOException, InterruptedException, TimeoutException {
+        executor = Executors.newCachedThreadPool();
+
         data1 = Files.createTempDirectory("").toFile();
         data2 = Files.createTempDirectory("").toFile();
 
-        Network network = Network.newNetwork();
+        network = Network.newNetwork();
 
         container1 = new GenericContainer<>(DhfsImage.getInstance())
                 .withPrivilegedMode(true)
@@ -96,11 +99,28 @@ public class KillIT {
         Stream.of(container1, container2).parallel().forEach(GenericContainer::stop);
         TestDataCleaner.purgeDirectory(data1);
         TestDataCleaner.purgeDirectory(data2);
+        executor.close();
+        network.close();
+    }
+
+    private void checkConsistency() {
+        await().atMost(45, TimeUnit.SECONDS).until(() -> {
+            Log.info("Listing consistency");
+            var ls1 = container1.execInContainer("/bin/sh", "-c", "ls /dhfs_test/fuse");
+            var cat1 = container1.execInContainer("/bin/sh", "-c", "cat /dhfs_test/fuse/*");
+            var ls2 = container2.execInContainer("/bin/sh", "-c", "ls /dhfs_test/fuse");
+            var cat2 = container2.execInContainer("/bin/sh", "-c", "cat /dhfs_test/fuse/*");
+            Log.info(ls1);
+            Log.info(cat1);
+            Log.info(ls2);
+            Log.info(cat2);
+
+            return ls1.equals(ls2) && cat1.equals(cat2);
+        });
     }
 
     @Test
     void killTest(TestInfo testInfo) throws Exception {
-        var executor = Executors.newFixedThreadPool(2);
         var barrier = new CyclicBarrier(2);
         var ret1 = executor.submit(() -> {
             try {
@@ -124,24 +144,11 @@ public class KillIT {
         waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS);
         waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS);
 
-        await().atMost(45, TimeUnit.SECONDS).until(() -> {
-            Log.info("Listing consistency");
-            var ls1 = container1.execInContainer("/bin/sh", "-c", "ls /dhfs_test/fuse");
-            var cat1 = container1.execInContainer("/bin/sh", "-c", "cat /dhfs_test/fuse/*");
-            var ls2 = container2.execInContainer("/bin/sh", "-c", "ls /dhfs_test/fuse");
-            var cat2 = container2.execInContainer("/bin/sh", "-c", "cat /dhfs_test/fuse/*");
-            Log.info(ls1);
-            Log.info(cat1);
-            Log.info(ls2);
-            Log.info(cat2);
-
-            return ls1.equals(ls2) && cat1.equals(cat2);
-        });
+        checkConsistency();
     }
 
     @Test
     void killTestDirs(TestInfo testInfo) throws Exception {
-        var executor = Executors.newFixedThreadPool(2);
         var barrier = new CyclicBarrier(2);
         var ret1 = executor.submit(() -> {
             try {
@@ -165,18 +172,64 @@ public class KillIT {
         waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS);
         waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS);
 
-        await().atMost(45, TimeUnit.SECONDS).until(() -> {
-            Log.info("Listing consistency");
-            var ls1 = container1.execInContainer("/bin/sh", "-c", "ls /dhfs_test/fuse");
-            var cat1 = container1.execInContainer("/bin/sh", "-c", "cat /dhfs_test/fuse/*");
-            var ls2 = container2.execInContainer("/bin/sh", "-c", "ls /dhfs_test/fuse");
-            var cat2 = container2.execInContainer("/bin/sh", "-c", "cat /dhfs_test/fuse/*");
-            Log.info(ls1);
-            Log.info(cat1);
-            Log.info(ls2);
-            Log.info(cat2);
+        checkConsistency();
+    }
 
-            return ls1.equals(ls2) && cat1.equals(cat2);
+    @Test
+    void killTest2(TestInfo testInfo) throws Exception {
+        var barrier = new CyclicBarrier(2);
+        var ret1 = executor.submit(() -> {
+            try {
+                Log.info("Writing to container 1");
+                barrier.await();
+                container1.execInContainer("/bin/sh", "-c", "counter=0; while  [ ! -f /tmp/stopprinting ]; do counter=`expr $counter + 1`; echo $counter >> /dhfs_test/fuse/test1; done");
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
         });
+        barrier.await();
+        Thread.sleep(10000);
+        var client = DockerClientFactory.instance().client();
+        client.killContainerCmd(container2.getContainerId()).exec();
+        container2.stop();
+        waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Lost connection to"), 60, TimeUnit.SECONDS);
+        container1.execInContainer("/bin/sh", "-c", "touch /tmp/stopprinting");
+        container2.start();
+        waitingConsumer2 = new WaitingConsumer();
+        var loggingConsumer2 = new Slf4jLogConsumer(LoggerFactory.getLogger(KillIT.class)).withPrefix("2-" + testInfo.getDisplayName());
+        container2.followOutput(loggingConsumer2.andThen(waitingConsumer2));
+        waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS);
+        waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS);
+
+        checkConsistency();
+    }
+
+    @Test
+    void killTestDirs2(TestInfo testInfo) throws Exception {
+        var barrier = new CyclicBarrier(2);
+        var ret1 = executor.submit(() -> {
+            try {
+                Log.info("Writing to container 1");
+                barrier.await();
+                container1.execInContainer("/bin/sh", "-c", "counter=0; while [ ! -f /tmp/stopprinting ]; do counter=`expr $counter + 1`; echo $counter >> /dhfs_test/fuse/test$counter; done");
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        });
+        barrier.await();
+        Thread.sleep(10000);
+        var client = DockerClientFactory.instance().client();
+        client.killContainerCmd(container2.getContainerId()).exec();
+        container2.stop();
+        waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Lost connection to"), 60, TimeUnit.SECONDS);
+        container1.execInContainer("/bin/sh", "-c", "touch /tmp/stopprinting");
+        container2.start();
+        waitingConsumer2 = new WaitingConsumer();
+        var loggingConsumer2 = new Slf4jLogConsumer(LoggerFactory.getLogger(KillIT.class)).withPrefix("2-" + testInfo.getDisplayName());
+        container2.followOutput(loggingConsumer2.andThen(waitingConsumer2));
+        waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS);
+        waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS);
+
+        checkConsistency();
     }
 }

dhfs-parent/dhfs-app/src/test/java/com/usatiuk/dhfs/integration/LazyFs.java

@@ -156,14 +156,14 @@ public class LazyFs {
                 "op=\"write\"\n" +
                 "file=\"" + mdbPath() + "\"\n" +
                 "persist=[1,4]\n" +
-                "occurrence=2");
+                "occurrence=10");
     }
 
     public void startTornSeq() {
         start("[[injection]]\n" +
                 "type=\"torn-op\"\n" +
                 "file=\"" + mdbPath() + "\"\n" +
-                "occurrence=5\n" +
+                "occurrence=10\n" +
                 "parts=3 #or parts_bytes=[4096,3600,1260]\n" +
                 "persist=[1,3]");
     }

dhfs-parent/dhfs-app/src/test/java/com/usatiuk/dhfs/integration/LazyFsIT.java

@@ -4,6 +4,11 @@ import com.github.dockerjava.api.model.Device;
 import com.usatiuk.dhfs.TestDataCleaner;
 import io.quarkus.logging.Log;
 import org.junit.jupiter.api.*;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.EnumSource;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
+import org.junit.jupiter.params.ParameterizedTest;
 import org.slf4j.LoggerFactory;
 import org.testcontainers.DockerClientFactory;
 import org.testcontainers.containers.GenericContainer;
@@ -41,16 +46,18 @@ public class LazyFsIT {
     LazyFs lazyFs1;
     LazyFs lazyFs2;
 
-    private final ExecutorService executor = Executors.newCachedThreadPool();
+    ExecutorService executor;
+    Network network;
 
     @BeforeEach
     void setup(TestInfo testInfo) throws IOException, InterruptedException, TimeoutException {
+        executor = Executors.newCachedThreadPool();
         data1 = Files.createTempDirectory("dhfsdata").toFile();
         data2 = Files.createTempDirectory("dhfsdata").toFile();
         data1Lazy = Files.createTempDirectory("lazyfsroot").toFile();
         data2Lazy = Files.createTempDirectory("lazyfsroot").toFile();
 
-        Network network = Network.newNetwork();
+        network = Network.newNetwork();
 
         lazyFs1 = new LazyFs(testInfo.getDisplayName(), data1.toString(), data1Lazy.toString());
         lazyFs1.start();
@@ -114,26 +121,34 @@ public class LazyFsIT {
         TestDataCleaner.purgeDirectory(data2Lazy);
 
         executor.close();
+        network.close();
     }
 
-    private void checkConsistency() {
+    private void checkConsistency(String testName) {
         await().atMost(45, TimeUnit.SECONDS).until(() -> {
-            Log.info("Listing consistency");
             var ls1 = container1.execInContainer("/bin/sh", "-c", "ls /dhfs_test/fuse");
             var cat1 = container1.execInContainer("/bin/sh", "-c", "cat /dhfs_test/fuse/*");
             var ls2 = container2.execInContainer("/bin/sh", "-c", "ls /dhfs_test/fuse");
             var cat2 = container2.execInContainer("/bin/sh", "-c", "cat /dhfs_test/fuse/*");
-            Log.info(ls1);
-            Log.info(cat1);
-            Log.info(ls2);
-            Log.info(cat2);
+            Log.info("Listing consistency " + testName + "\n"
+                    + ls1 + "\n"
+                    + cat1 + "\n"
+                    + ls2 + "\n"
+                    + cat2 + "\n");
 
             return ls1.equals(ls2) && cat1.equals(cat2);
         });
     }
 
-    @Test
-    void killTest(TestInfo testInfo) throws Exception {
+    private static enum CrashType {
+        CRASH,
+        TORN_OP,
+        TORN_SEQ
+    }
+
+    @ParameterizedTest
+    @EnumSource(CrashType.class)
+    void killTest(CrashType crashType, TestInfo testInfo) throws Exception {
         var barrier = new CountDownLatch(1);
         executor.submit(() -> {
             try {
@@ -145,7 +160,7 @@ public class LazyFsIT {
             }
         });
         barrier.await();
-        Thread.sleep(2000);
+        Thread.sleep(3000);
         Log.info("Killing");
         lazyFs1.crash();
         waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Caused by: org.lmdbjava"), 5, TimeUnit.SECONDS);
@@ -154,19 +169,55 @@ public class LazyFsIT {
         container1.stop();
         waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Lost connection to"), 60, TimeUnit.SECONDS);
         Log.info("Restart");
-        lazyFs1.start();
+        switch (crashType) {
+            case CRASH -> lazyFs1.start();
+            case TORN_OP -> lazyFs1.startTornOp();
+            case TORN_SEQ -> lazyFs1.startTornSeq();
+        }
         container1.start();
+
         waitingConsumer1 = new WaitingConsumer();
         var loggingConsumer1 = new Slf4jLogConsumer(LoggerFactory.getLogger(LazyFsIT.class)).withPrefix("1-" + testInfo.getDisplayName());
         container1.followOutput(loggingConsumer1.andThen(waitingConsumer1));
         waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS);
         waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS);
 
-        checkConsistency();
+        executor.submit(() -> {
+            try {
+                Log.info("Writing to container 1");
+                barrier.countDown();
+                container1.execInContainer("/bin/sh", "-c", "counter=0; while true; do counter=`expr $counter + 1`; echo $counter >> /dhfs_test/fuse/test2; done");
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        });
+        Log.info("Killing");
+        if (crashType.equals(CrashType.CRASH)) {
+            Thread.sleep(3000);
+            lazyFs1.crash();
+        }
+        waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Caused by: org.lmdbjava"), 5, TimeUnit.SECONDS);
+        client.killContainerCmd(container1.getContainerId()).exec();
+        container1.stop();
+        lazyFs1.stop();
+        waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Lost connection to"), 60, TimeUnit.SECONDS);
+        Log.info("Restart");
+        lazyFs1.start();
+        container1.start();
+
+        waitingConsumer1 = new WaitingConsumer();
+        loggingConsumer1 = new Slf4jLogConsumer(LoggerFactory.getLogger(LazyFsIT.class)).withPrefix("1-" + testInfo.getDisplayName());
+        container1.followOutput(loggingConsumer1.andThen(waitingConsumer1));
+        waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS);
+        waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS);
+
+        checkConsistency(testInfo.getDisplayName());
     }
 
-    @Test
-    void killTestDirs(TestInfo testInfo) throws Exception {
+
+    @ParameterizedTest
+    @EnumSource(CrashType.class)
+    void killTestDirs(CrashType crashType, TestInfo testInfo) throws Exception {
         var barrier = new CountDownLatch(1);
         executor.submit(() -> {
             try {
@@ -178,7 +229,7 @@ public class LazyFsIT {
             }
         });
         barrier.await();
-        Thread.sleep(2000);
+        Thread.sleep(3000);
         Log.info("Killing");
         lazyFs1.crash();
         waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Caused by: org.lmdbjava"), 5, TimeUnit.SECONDS);
@@ -187,39 +238,11 @@ public class LazyFsIT {
         container1.stop();
         waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Lost connection to"), 60, TimeUnit.SECONDS);
         Log.info("Restart");
-        lazyFs1.start();
-        container1.start();
-        waitingConsumer1 = new WaitingConsumer();
-        var loggingConsumer1 = new Slf4jLogConsumer(LoggerFactory.getLogger(LazyFsIT.class)).withPrefix("1-" + testInfo.getDisplayName());
-        container1.followOutput(loggingConsumer1.andThen(waitingConsumer1));
-        waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS);
-        waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS);
-
-        checkConsistency();
-    }
-
-    @Test
-    void killTestDirs2(TestInfo testInfo) throws Exception {
-        var barrier = new CountDownLatch(1);
-        executor.submit(() -> {
-            try {
-                Log.info("Writing to container 1");
-                barrier.countDown();
-                container1.execInContainer("/bin/sh", "-c", "counter=0; while true; do counter=`expr $counter + 1`; echo $counter >> /dhfs_test/fuse/test$counter; done");
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-        });
-        barrier.await();
-        Log.info("Killing");
-        lazyFs1.crash();
-        waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Caused by: org.lmdbjava"), 5, TimeUnit.SECONDS);
-        var client = DockerClientFactory.instance().client();
-        client.killContainerCmd(container1.getContainerId()).exec();
-        container1.stop();
-        waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Lost connection to"), 60, TimeUnit.SECONDS);
-        Log.info("Restart");
-        lazyFs1.startTornOp();
+        switch (crashType) {
+            case CRASH -> lazyFs1.start();
+            case TORN_OP -> lazyFs1.startTornOp();
+            case TORN_SEQ -> lazyFs1.startTornSeq();
+        }
         container1.start();
 
         waitingConsumer1 = new WaitingConsumer();
@@ -238,6 +261,10 @@ public class LazyFsIT {
             }
         });
         Log.info("Killing");
+        if (crashType.equals(CrashType.CRASH)) {
+            Thread.sleep(3000);
+            lazyFs1.crash();
+        }
         waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Caused by: org.lmdbjava"), 5, TimeUnit.SECONDS);
         client.killContainerCmd(container1.getContainerId()).exec();
         container1.stop();
@@ -253,64 +280,152 @@ public class LazyFsIT {
         waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS);
         waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS);
 
-        checkConsistency();
+        checkConsistency(testInfo.getDisplayName());
     }
 
-    @Test
-    void killTestDirs3(TestInfo testInfo) throws Exception {
+    @ParameterizedTest
+    @EnumSource(CrashType.class)
+    void killTest2(CrashType crashType, TestInfo testInfo) throws Exception {
         var barrier = new CountDownLatch(1);
         executor.submit(() -> {
             try {
                 Log.info("Writing to container 1");
                 barrier.countDown();
-                container1.execInContainer("/bin/sh", "-c", "counter=0; while true; do counter=`expr $counter + 1`; echo $counter >> /dhfs_test/fuse/test$counter; done");
+                container1.execInContainer("/bin/sh", "-c", "counter=0; while  [ ! -f /tmp/stopprinting1 ]; do counter=`expr $counter + 1`; echo $counter >> /dhfs_test/fuse/test1; done");
             } catch (Exception e) {
                 throw new RuntimeException(e);
             }
         });
         barrier.await();
         Log.info("Killing");
-        lazyFs1.crash();
-        waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Caused by: org.lmdbjava"), 5, TimeUnit.SECONDS);
+        lazyFs2.crash();
+        waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Caused by: org.lmdbjava"), 5, TimeUnit.SECONDS);
         var client = DockerClientFactory.instance().client();
-        client.killContainerCmd(container1.getContainerId()).exec();
-        container1.stop();
-        waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Lost connection to"), 60, TimeUnit.SECONDS);
+        client.killContainerCmd(container2.getContainerId()).exec();
+        container2.stop();
+        waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Lost connection to"), 60, TimeUnit.SECONDS);
         Log.info("Restart");
-        lazyFs1.startTornSeq();
-        container1.start();
+        switch (crashType) {
+            case CRASH -> lazyFs2.start();
+            case TORN_OP -> lazyFs2.startTornOp();
+            case TORN_SEQ -> lazyFs2.startTornSeq();
+        }
+        container1.execInContainer("/bin/sh", "-c", "touch /tmp/stopprinting1");
+        container2.start();
 
-        waitingConsumer1 = new WaitingConsumer();
-        var loggingConsumer1 = new Slf4jLogConsumer(LoggerFactory.getLogger(LazyFsIT.class)).withPrefix("1-" + testInfo.getDisplayName());
-        container1.followOutput(loggingConsumer1.andThen(waitingConsumer1));
+        waitingConsumer2 = new WaitingConsumer();
+        var loggingConsumer2 = new Slf4jLogConsumer(LoggerFactory.getLogger(LazyFsIT.class)).withPrefix("2-" + testInfo.getDisplayName());
+        container2.followOutput(loggingConsumer2.andThen(waitingConsumer2));
         waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS);
         waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS);
 
+        var barrier2 = new CountDownLatch(1);
         executor.submit(() -> {
             try {
                 Log.info("Writing to container 1");
-                barrier.countDown();
-                container1.execInContainer("/bin/sh", "-c", "counter=0; while true; do counter=`expr $counter + 1`; echo $counter >> /dhfs_test/fuse/2test$counter; done");
+                barrier2.countDown();
+                container1.execInContainer("/bin/sh", "-c", "counter=0; while  [ ! -f /tmp/stopprinting2 ]; do counter=`expr $counter + 1`; echo $counter >> /dhfs_test/fuse/test2; done");
             } catch (Exception e) {
                 throw new RuntimeException(e);
             }
         });
+        barrier2.await();
         Log.info("Killing");
-        waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Caused by: org.lmdbjava"), 5, TimeUnit.SECONDS);
-        client.killContainerCmd(container1.getContainerId()).exec();
-        container1.stop();
-        lazyFs1.stop();
-        waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Lost connection to"), 60, TimeUnit.SECONDS);
+        if (crashType.equals(CrashType.CRASH)) {
+            Thread.sleep(2000);
+            lazyFs2.crash();
+        }
+        waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Caused by: org.lmdbjava"), 30, TimeUnit.SECONDS);
+        client.killContainerCmd(container2.getContainerId()).exec();
+        container2.stop();
+        lazyFs2.stop();
+        waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Lost connection to"), 60, TimeUnit.SECONDS);
         Log.info("Restart");
-        lazyFs1.start();
-        container1.start();
+        container1.execInContainer("/bin/sh", "-c", "touch /tmp/stopprinting2");
+        lazyFs2.start();
+        container2.start();
 
-        waitingConsumer1 = new WaitingConsumer();
-        loggingConsumer1 = new Slf4jLogConsumer(LoggerFactory.getLogger(LazyFsIT.class)).withPrefix("1-" + testInfo.getDisplayName());
-        container1.followOutput(loggingConsumer1.andThen(waitingConsumer1));
+        waitingConsumer2 = new WaitingConsumer();
+        loggingConsumer2 = new Slf4jLogConsumer(LoggerFactory.getLogger(LazyFsIT.class)).withPrefix("2-" + testInfo.getDisplayName());
+        container2.followOutput(loggingConsumer2.andThen(waitingConsumer2));
         waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS);
         waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS);
 
-        checkConsistency();
+        checkConsistency(testInfo.getDisplayName());
     }
+
+
+    @ParameterizedTest
+    @EnumSource(CrashType.class)
+    void killTestDirs2(CrashType crashType, TestInfo testInfo) throws Exception {
+        var barrier = new CountDownLatch(1);
+        executor.submit(() -> {
+            try {
+                Log.info("Writing to container 1");
+                barrier.countDown();
+                container1.execInContainer("/bin/sh", "-c", "counter=0; while  [ ! -f /tmp/stopprinting1 ]; do counter=`expr $counter + 1`; echo $counter >> /dhfs_test/fuse/test$counter; done");
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        });
+        barrier.await();
+        Thread.sleep(3000);
+        Log.info("Killing");
+        lazyFs2.crash();
+        waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Caused by: org.lmdbjava"), 5, TimeUnit.SECONDS);
+        var client = DockerClientFactory.instance().client();
+        client.killContainerCmd(container2.getContainerId()).exec();
+        container2.stop();
+        waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Lost connection to"), 60, TimeUnit.SECONDS);
+        Log.info("Restart");
+        switch (crashType) {
+            case CRASH -> lazyFs2.start();
+            case TORN_OP -> lazyFs2.startTornOp();
+            case TORN_SEQ -> lazyFs2.startTornSeq();
+        }
+        container1.execInContainer("/bin/sh", "-c", "touch /tmp/stopprinting1");
+        container2.start();
+
+        waitingConsumer2 = new WaitingConsumer();
+        var loggingConsumer2 = new Slf4jLogConsumer(LoggerFactory.getLogger(LazyFsIT.class)).withPrefix("2-" + testInfo.getDisplayName());
+        container2.followOutput(loggingConsumer2.andThen(waitingConsumer2));
+        waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS);
+        waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS);
+
+        var barrier2 = new CountDownLatch(1);
+        executor.submit(() -> {
+            try {
+                Log.info("Writing to container 1");
+                barrier2.countDown();
+                container1.execInContainer("/bin/sh", "-c", "counter=0; while  [ ! -f /tmp/stopprinting2 ]; do counter=`expr $counter + 1`; echo $counter >> /dhfs_test/fuse/2test$counter; done");
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        });
+        barrier2.await();
+        Log.info("Killing");
+        if (crashType.equals(CrashType.CRASH)) {
+            Thread.sleep(2000);
+            lazyFs2.crash();
+        }
+        waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Caused by: org.lmdbjava"), 30, TimeUnit.SECONDS);
+        client.killContainerCmd(container2.getContainerId()).exec();
+        container2.stop();
+        lazyFs2.stop();
+        waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Lost connection to"), 60, TimeUnit.SECONDS);
+        container1.execInContainer("/bin/sh", "-c", "touch /tmp/stopprinting2");
+        Log.info("Restart");
+        lazyFs2.start();
+        container2.start();
+
+        waitingConsumer2 = new WaitingConsumer();
+        loggingConsumer2 = new Slf4jLogConsumer(LoggerFactory.getLogger(LazyFsIT.class)).withPrefix("2-" + testInfo.getDisplayName());
+        container2.followOutput(loggingConsumer2.andThen(waitingConsumer2));
+        waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS);
+        waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS);
+
+        checkConsistency(testInfo.getDisplayName());
+    }
+
+
 }

dhfs-parent/dhfs-app/src/test/java/com/usatiuk/dhfs/integration/ResyncIT.java

@@ -29,9 +29,11 @@ public class ResyncIT {
     String c1uuid;
     String c2uuid;
 
+    Network network;
+
     @BeforeEach
     void setup(TestInfo testInfo) throws IOException, InterruptedException, TimeoutException {
-        Network network = Network.newNetwork();
+        network = Network.newNetwork();
 
         container1 = new GenericContainer<>(DhfsImage.getInstance())
                 .withPrivilegedMode(true)
@@ -55,6 +57,7 @@ public class ResyncIT {
     @AfterEach
     void stop() {
         Stream.of(container1, container2).parallel().forEach(GenericContainer::stop);
+        network.close();
     }
 
     @Test