From 2e2eb3ac9775cfc5abcbf851250720c9fcb172f9 Mon Sep 17 00:00:00 2001 From: Stepan Usatiuk Date: Wed, 23 Apr 2025 15:07:09 +0200 Subject: [PATCH] Dhfs-app: lazyfs torn op testing --- .../com/usatiuk/dhfs/integration/LazyFs.java | 33 ++++- .../usatiuk/dhfs/integration/LazyFsIT.java | 116 ++++++++++++++++++ 2 files changed, 146 insertions(+), 3 deletions(-) diff --git a/dhfs-parent/dhfs-app/src/test/java/com/usatiuk/dhfs/integration/LazyFs.java b/dhfs-parent/dhfs-app/src/test/java/com/usatiuk/dhfs/integration/LazyFs.java index 2bfeface..5727ffda 100644 --- a/dhfs-parent/dhfs-app/src/test/java/com/usatiuk/dhfs/integration/LazyFs.java +++ b/dhfs-parent/dhfs-app/src/test/java/com/usatiuk/dhfs/integration/LazyFs.java @@ -52,7 +52,7 @@ public class LazyFs { return fifoFile.getAbsolutePath(); } - public void start() { + public void start(String extraOpts) { var lfsPath = Path.of(lazyFsPath).resolve("build").resolve("lazyfs"); if (!lfsPath.toFile().isFile()) throw new IllegalStateException("LazyFs binary does not exist: " + lfsPath.toAbsolutePath()); @@ -71,7 +71,7 @@ public class LazyFs { "blocks_per_page=1\n" + "[filesystem]\n" + "log_all_operations=false\n" + - "logfile=\"\""; + "logfile=\"\"\n" + extraOpts; rwFile.write(config.getBytes()); Log.info("LazyFs config: \n" + config); } catch (Exception e) { @@ -141,6 +141,33 @@ public class LazyFs { Log.info("LazyFs started"); } + public void start() { + start(""); + } + + private String mdbPath() { + return Path.of(dataRoot).resolve("objects").resolve("data.mdb").toAbsolutePath().toString(); + } + + public void startTornOp() { + start("\n" + + "[[injection]]\n" + + "type=\"torn-seq\"\n" + + "op=\"write\"\n" + + "file=\"" + mdbPath() + "\"\n" + + "persist=[1,4]\n" + + "occurrence=2"); + } + + public void startTornSeq() { + start("[[injection]]\n" + + "type=\"torn-op\"\n" + + "file=\"" + mdbPath() + "\"\n" + + "occurrence=5\n" + + "parts=3 #or parts_bytes=[4096,3600,1260]\n" + + "persist=[1,3]"); + } + public void crash() { try { var cmd = "echo \"lazyfs::crash::timing=after::op=write::from_rgx=*\" > " + fifoPath(); @@ -158,7 +185,7 @@ public class LazyFs { if (fs == null) { return; } - Runtime.getRuntime().exec(new String[]{"/bin/sh", "-c", "fusermount3 -u " + mountRoot}); + Runtime.getRuntime().exec(new String[]{"/bin/sh", "-c", "fusermount3 -u " + mountRoot}).waitFor(); fs = null; } } catch (Exception e) { diff --git a/dhfs-parent/dhfs-app/src/test/java/com/usatiuk/dhfs/integration/LazyFsIT.java b/dhfs-parent/dhfs-app/src/test/java/com/usatiuk/dhfs/integration/LazyFsIT.java index 4b7dfb45..77d5aa61 100644 --- a/dhfs-parent/dhfs-app/src/test/java/com/usatiuk/dhfs/integration/LazyFsIT.java +++ b/dhfs-parent/dhfs-app/src/test/java/com/usatiuk/dhfs/integration/LazyFsIT.java @@ -197,4 +197,120 @@ public class LazyFsIT { checkConsistency(); } + + @Test + void killTestDirs2(TestInfo testInfo) throws Exception { + var barrier = new CountDownLatch(1); + executor.submit(() -> { + try { + Log.info("Writing to container 1"); + barrier.countDown(); + container1.execInContainer("/bin/sh", "-c", "counter=0; while true; do counter=`expr $counter + 1`; echo $counter >> /dhfs_test/fuse/test$counter; done"); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + barrier.await(); + Log.info("Killing"); + lazyFs1.crash(); + waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Caused by: org.lmdbjava"), 5, TimeUnit.SECONDS); + var client = DockerClientFactory.instance().client(); + client.killContainerCmd(container1.getContainerId()).exec(); + container1.stop(); + waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Lost connection to"), 60, TimeUnit.SECONDS); + Log.info("Restart"); + lazyFs1.startTornOp(); + container1.start(); + + waitingConsumer1 = new WaitingConsumer(); + var loggingConsumer1 = new Slf4jLogConsumer(LoggerFactory.getLogger(LazyFsIT.class)).withPrefix("1-" + testInfo.getDisplayName()); + container1.followOutput(loggingConsumer1.andThen(waitingConsumer1)); + waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS); + waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS); + + executor.submit(() -> { + try { + Log.info("Writing to container 1"); + barrier.countDown(); + container1.execInContainer("/bin/sh", "-c", "counter=0; while true; do counter=`expr $counter + 1`; echo $counter >> /dhfs_test/fuse/2test$counter; done"); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + Log.info("Killing"); + waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Caused by: org.lmdbjava"), 5, TimeUnit.SECONDS); + client.killContainerCmd(container1.getContainerId()).exec(); + container1.stop(); + lazyFs1.stop(); + waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Lost connection to"), 60, TimeUnit.SECONDS); + Log.info("Restart"); + lazyFs1.start(); + container1.start(); + + waitingConsumer1 = new WaitingConsumer(); + loggingConsumer1 = new Slf4jLogConsumer(LoggerFactory.getLogger(LazyFsIT.class)).withPrefix("1-" + testInfo.getDisplayName()); + container1.followOutput(loggingConsumer1.andThen(waitingConsumer1)); + waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS); + waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS); + + checkConsistency(); + } + + @Test + void killTestDirs3(TestInfo testInfo) throws Exception { + var barrier = new CountDownLatch(1); + executor.submit(() -> { + try { + Log.info("Writing to container 1"); + barrier.countDown(); + container1.execInContainer("/bin/sh", "-c", "counter=0; while true; do counter=`expr $counter + 1`; echo $counter >> /dhfs_test/fuse/test$counter; done"); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + barrier.await(); + Log.info("Killing"); + lazyFs1.crash(); + waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Caused by: org.lmdbjava"), 5, TimeUnit.SECONDS); + var client = DockerClientFactory.instance().client(); + client.killContainerCmd(container1.getContainerId()).exec(); + container1.stop(); + waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Lost connection to"), 60, TimeUnit.SECONDS); + Log.info("Restart"); + lazyFs1.startTornSeq(); + container1.start(); + + waitingConsumer1 = new WaitingConsumer(); + var loggingConsumer1 = new Slf4jLogConsumer(LoggerFactory.getLogger(LazyFsIT.class)).withPrefix("1-" + testInfo.getDisplayName()); + container1.followOutput(loggingConsumer1.andThen(waitingConsumer1)); + waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS); + waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS); + + executor.submit(() -> { + try { + Log.info("Writing to container 1"); + barrier.countDown(); + container1.execInContainer("/bin/sh", "-c", "counter=0; while true; do counter=`expr $counter + 1`; echo $counter >> /dhfs_test/fuse/2test$counter; done"); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + Log.info("Killing"); + waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Caused by: org.lmdbjava"), 5, TimeUnit.SECONDS); + client.killContainerCmd(container1.getContainerId()).exec(); + container1.stop(); + lazyFs1.stop(); + waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Lost connection to"), 60, TimeUnit.SECONDS); + Log.info("Restart"); + lazyFs1.start(); + container1.start(); + + waitingConsumer1 = new WaitingConsumer(); + loggingConsumer1 = new Slf4jLogConsumer(LoggerFactory.getLogger(LazyFsIT.class)).withPrefix("1-" + testInfo.getDisplayName()); + container1.followOutput(loggingConsumer1.andThen(waitingConsumer1)); + waitingConsumer2.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS); + waitingConsumer1.waitUntil(frame -> frame.getUtf8String().contains("Connected"), 60, TimeUnit.SECONDS); + + checkConsistency(); + } }