From bebe0fb5d6b875a92907e856731f5abb67834fb5 Mon Sep 17 00:00:00 2001 From: Stepan Usatiuk Date: Sun, 12 May 2024 14:29:07 +0200 Subject: [PATCH] find files --- .../dhfs/storage/HelloGrpcService.java | 16 ---- .../storage/api/DhfsObjectGrpcService.java | 32 +++++++ .../usatiuk/dhfs/storage/data/Namespace.java | 5 + .../com/usatiuk/dhfs/storage/data/Object.java | 10 ++ .../storage/repository/ObjectRepository.java | 18 ++++ .../SimpleFileObjectRepository.java | 94 +++++++++++++++++++ server/src/main/proto/dhfs_objects.proto | 55 +++++++++++ server/src/main/proto/hello.proto | 19 ---- .../src/main/resources/application.properties | 2 + .../dhfs/storage/DhfsObjectGrpcService.java | 27 ++++++ .../dhfs/storage/HelloGrpcServiceTest.java | 24 ----- .../dhfs/storage/SimpleFileRepoTest.java | 22 +++++ .../src/test/resources/application.properties | 1 + 13 files changed, 266 insertions(+), 59 deletions(-) delete mode 100644 server/src/main/java/com/usatiuk/dhfs/storage/HelloGrpcService.java create mode 100644 server/src/main/java/com/usatiuk/dhfs/storage/api/DhfsObjectGrpcService.java create mode 100644 server/src/main/java/com/usatiuk/dhfs/storage/data/Namespace.java create mode 100644 server/src/main/java/com/usatiuk/dhfs/storage/data/Object.java create mode 100644 server/src/main/java/com/usatiuk/dhfs/storage/repository/ObjectRepository.java create mode 100644 server/src/main/java/com/usatiuk/dhfs/storage/repository/SimpleFileObjectRepository.java create mode 100644 server/src/main/proto/dhfs_objects.proto delete mode 100644 server/src/main/proto/hello.proto create mode 100644 server/src/test/java/com/usatiuk/dhfs/storage/DhfsObjectGrpcService.java delete mode 100644 server/src/test/java/com/usatiuk/dhfs/storage/HelloGrpcServiceTest.java create mode 100644 server/src/test/java/com/usatiuk/dhfs/storage/SimpleFileRepoTest.java create mode 100644 server/src/test/resources/application.properties diff --git a/server/src/main/java/com/usatiuk/dhfs/storage/HelloGrpcService.java b/server/src/main/java/com/usatiuk/dhfs/storage/HelloGrpcService.java deleted file mode 100644 index 56c52777..00000000 --- a/server/src/main/java/com/usatiuk/dhfs/storage/HelloGrpcService.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.usatiuk.dhfs.storage; - -import io.quarkus.grpc.GrpcService; - -import io.smallrye.mutiny.Uni; - -@GrpcService -public class HelloGrpcService implements HelloGrpc { - - @Override - public Uni sayHello(HelloRequest request) { - return Uni.createFrom().item("Hello " + request.getName() + "!") - .map(msg -> HelloReply.newBuilder().setMessage(msg).build()); - } - -} diff --git a/server/src/main/java/com/usatiuk/dhfs/storage/api/DhfsObjectGrpcService.java b/server/src/main/java/com/usatiuk/dhfs/storage/api/DhfsObjectGrpcService.java new file mode 100644 index 00000000..49bf01df --- /dev/null +++ b/server/src/main/java/com/usatiuk/dhfs/storage/api/DhfsObjectGrpcService.java @@ -0,0 +1,32 @@ +package com.usatiuk.dhfs.storage.api; + +import com.usatiuk.dhfs.storage.repository.ObjectRepository; +import io.quarkus.grpc.GrpcService; +import io.smallrye.mutiny.Uni; +import jakarta.inject.Inject; + +@GrpcService +public class DhfsObjectGrpcService implements DhfsObjectGrpc { + @Inject + ObjectRepository objectRepository; + + @Override + public Uni findObjects(FindObjectsRequest request) { + return objectRepository.findObjects(request.getNamespace(), request.getPrefix()) + .map(m -> FindObjectsReply.FindObjectsEntry.newBuilder().setName(m).build()) + .collect().in(FindObjectsReply::newBuilder, FindObjectsReply.Builder::addFound) + .map(FindObjectsReply.Builder::build); + } + + @Override + public Uni readObject(ReadObjectRequest request) { + return Uni.createFrom().item("Hello ") + .map(msg -> ReadObjectReply.newBuilder().build()); + } + + @Override + public Uni writeObject(WriteObjectRequest request) { + return Uni.createFrom().item("Hello ") + .map(msg -> WriteObjectReply.newBuilder().build()); + } +} diff --git a/server/src/main/java/com/usatiuk/dhfs/storage/data/Namespace.java b/server/src/main/java/com/usatiuk/dhfs/storage/data/Namespace.java new file mode 100644 index 00000000..97edd3f9 --- /dev/null +++ b/server/src/main/java/com/usatiuk/dhfs/storage/data/Namespace.java @@ -0,0 +1,5 @@ +package com.usatiuk.dhfs.storage.data; + +public class Namespace { + String name; +} diff --git a/server/src/main/java/com/usatiuk/dhfs/storage/data/Object.java b/server/src/main/java/com/usatiuk/dhfs/storage/data/Object.java new file mode 100644 index 00000000..1c3a7f17 --- /dev/null +++ b/server/src/main/java/com/usatiuk/dhfs/storage/data/Object.java @@ -0,0 +1,10 @@ +package com.usatiuk.dhfs.storage.data; + +import java.nio.ByteBuffer; + +public class Object { + Namespace namespace; + + String name; + ByteBuffer data; +} diff --git a/server/src/main/java/com/usatiuk/dhfs/storage/repository/ObjectRepository.java b/server/src/main/java/com/usatiuk/dhfs/storage/repository/ObjectRepository.java new file mode 100644 index 00000000..156dcf6c --- /dev/null +++ b/server/src/main/java/com/usatiuk/dhfs/storage/repository/ObjectRepository.java @@ -0,0 +1,18 @@ +package com.usatiuk.dhfs.storage.repository; + +import com.usatiuk.dhfs.storage.data.Object; +import io.smallrye.mutiny.Multi; +import io.smallrye.mutiny.Uni; + +import javax.annotation.Nonnull; +import java.nio.ByteBuffer; + +public interface ObjectRepository { + @Nonnull + public Multi findObjects(String namespace, String prefix); + + @Nonnull + public Uni readObject(String namespace, String name); + @Nonnull + public Uni writeObject(String namespace, String name, Object data); +} diff --git a/server/src/main/java/com/usatiuk/dhfs/storage/repository/SimpleFileObjectRepository.java b/server/src/main/java/com/usatiuk/dhfs/storage/repository/SimpleFileObjectRepository.java new file mode 100644 index 00000000..ff94ad5e --- /dev/null +++ b/server/src/main/java/com/usatiuk/dhfs/storage/repository/SimpleFileObjectRepository.java @@ -0,0 +1,94 @@ +package com.usatiuk.dhfs.storage.repository; + +import com.usatiuk.dhfs.storage.data.Object; +import io.quarkus.logging.Log; +import io.quarkus.runtime.Shutdown; +import io.quarkus.runtime.Startup; +import io.smallrye.mutiny.Multi; +import io.smallrye.mutiny.Uni; +import io.vertx.mutiny.core.Vertx; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import org.eclipse.microprofile.config.inject.ConfigProperty; + +import javax.annotation.Nonnull; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +@ApplicationScoped +public class SimpleFileObjectRepository implements ObjectRepository { + @ConfigProperty(name = "dhfs.filerepo.root") + String root; + + @Inject + Vertx vertx; + + @Startup + void init() { + if (!Paths.get(root).toFile().exists()) { + Paths.get(root).toFile().mkdirs(); + Log.info("Creted root " + root); + } + + Log.info("Initializing with root " + root); + } + + @Shutdown + void shutdown() { + Log.info("Shutdown"); + } + + private Multi TraverseDir(String dir) { + return vertx.fileSystem().readDir(dir).onItem().transformToMulti( + t -> { + List> results = new ArrayList<>(); + t.forEach(entry -> { + if (Paths.get(entry).toFile().isDirectory()) { + results.add(TraverseDir(Paths.get(entry).toString())); + } else { + results.add(Multi.createFrom().item(entry)); + } + }); + return Multi.createBy().merging().streams(results); + } + ); + + } + + @Nonnull + @Override + public Multi findObjects(String namespace, String prefix) { + Path path = Paths.get(root, namespace, prefix); + Path rootDir = path.toFile().isDirectory() ? path : path.getParent(); + String prefixInRoot = path.toFile().isDirectory() ? "" : path.getFileName().toString(); + return vertx.fileSystem().readDir(rootDir.toString()).onItem().transformToMulti( + t -> { + List> results = new ArrayList<>(); + t.forEach(entry -> { + if (entry.startsWith(prefixInRoot)) { + if (Paths.get(entry).toFile().isDirectory()) { + results.add(TraverseDir(Paths.get(entry).toString())); + } else { + results.add(Multi.createFrom().item(entry)); + } + } + }); + return Multi.createBy().merging().streams(results); + } + ).map(f -> rootDir.relativize(Paths.get(f)).toString()); + } + + @Nonnull + @Override + public Uni readObject(String namespace, String name) { + return null; + } + + @Nonnull + @Override + public Uni writeObject(String namespace, String name, Object data) { + return null; + } +} diff --git a/server/src/main/proto/dhfs_objects.proto b/server/src/main/proto/dhfs_objects.proto new file mode 100644 index 00000000..fc3e4043 --- /dev/null +++ b/server/src/main/proto/dhfs_objects.proto @@ -0,0 +1,55 @@ +syntax = "proto3"; + +option java_multiple_files = true; +option java_package = "com.usatiuk.dhfs.storage.api"; +option java_outer_classname = "DhfsStorageApi"; + +package hello; + +service DhfsObjectGrpc { + rpc FindObjects (FindObjectsRequest) returns (FindObjectsReply) {} + rpc ReadObject (ReadObjectRequest) returns (ReadObjectReply) {} + rpc WriteObject (WriteObjectRequest) returns (WriteObjectReply) {} + rpc DeleteObject (DeleteObjectRequest) returns (DeleteObjectReply) {} +} + +message FindObjectsRequest { + string namespace = 1; + string prefix = 2; +} + +message FindObjectsReply { + message FindObjectsEntry { + string name = 1; + } + repeated FindObjectsEntry found = 1; +} + +message ReadObjectRequest { + string namespace = 1; + string name = 2; +} + +message ReadObjectReply { + string message = 1; + bytes data = 10; +} + +message WriteObjectRequest { + string namespace = 1; + string name = 2; + bytes data = 10; +} + +message WriteObjectReply { + bool ok = 1; +} + +message DeleteObjectRequest { + string namespace = 1; + string name = 2; +} + +message DeleteObjectReply { + bool ok = 1; +} diff --git a/server/src/main/proto/hello.proto b/server/src/main/proto/hello.proto deleted file mode 100644 index a062d292..00000000 --- a/server/src/main/proto/hello.proto +++ /dev/null @@ -1,19 +0,0 @@ -syntax = "proto3"; - -option java_multiple_files = true; -option java_package = "com.usatiuk.dhfs.storage"; -option java_outer_classname = "HelloGrpcProto"; - -package hello; - -service HelloGrpc { - rpc SayHello (HelloRequest) returns (HelloReply) {} -} - -message HelloRequest { - string name = 1; -} - -message HelloReply { - string message = 1; -} diff --git a/server/src/main/resources/application.properties b/server/src/main/resources/application.properties index e69de29b..5f4d7a6b 100644 --- a/server/src/main/resources/application.properties +++ b/server/src/main/resources/application.properties @@ -0,0 +1,2 @@ +quarkus.grpc.server.use-separate-server=false +dhfs.filerepo.root=${HOME}/dhfs_root \ No newline at end of file diff --git a/server/src/test/java/com/usatiuk/dhfs/storage/DhfsObjectGrpcService.java b/server/src/test/java/com/usatiuk/dhfs/storage/DhfsObjectGrpcService.java new file mode 100644 index 00000000..65fefd44 --- /dev/null +++ b/server/src/test/java/com/usatiuk/dhfs/storage/DhfsObjectGrpcService.java @@ -0,0 +1,27 @@ +package com.usatiuk.dhfs.storage; + +import com.usatiuk.dhfs.storage.api.DhfsObjectGrpc; +import com.usatiuk.dhfs.storage.api.FindObjectsReply; +import com.usatiuk.dhfs.storage.api.FindObjectsRequest; +import io.quarkus.grpc.GrpcClient; +import io.quarkus.test.junit.QuarkusTest; +import org.eclipse.microprofile.config.inject.ConfigProperty; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +@QuarkusTest +class DhfsObjectGrpcService { + @GrpcClient + DhfsObjectGrpc dhfsObjectGrpc; + + @ConfigProperty(name = "dhfs.filerepo.root") + String tempDirectory; + + @Test + void testFind() { + FindObjectsReply reply = dhfsObjectGrpc + .findObjects(FindObjectsRequest.newBuilder().setNamespace("TestNs").build()).await().atMost(Duration.ofSeconds(5)); + } + +} diff --git a/server/src/test/java/com/usatiuk/dhfs/storage/HelloGrpcServiceTest.java b/server/src/test/java/com/usatiuk/dhfs/storage/HelloGrpcServiceTest.java deleted file mode 100644 index 3ea5f438..00000000 --- a/server/src/test/java/com/usatiuk/dhfs/storage/HelloGrpcServiceTest.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.usatiuk.dhfs.storage; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import java.time.Duration; - -import io.quarkus.grpc.GrpcClient; -import io.quarkus.test.junit.QuarkusTest; - -import org.junit.jupiter.api.Test; - -@QuarkusTest -class HelloGrpcServiceTest { - @GrpcClient - HelloGrpc helloGrpc; - - @Test - void testHello() { - HelloReply reply = helloGrpc - .sayHello(HelloRequest.newBuilder().setName("Neo").build()).await().atMost(Duration.ofSeconds(5)); - assertEquals("Hello Neo!", reply.getMessage()); - } - -} diff --git a/server/src/test/java/com/usatiuk/dhfs/storage/SimpleFileRepoTest.java b/server/src/test/java/com/usatiuk/dhfs/storage/SimpleFileRepoTest.java new file mode 100644 index 00000000..f01bc17e --- /dev/null +++ b/server/src/test/java/com/usatiuk/dhfs/storage/SimpleFileRepoTest.java @@ -0,0 +1,22 @@ +package com.usatiuk.dhfs.storage; + +import io.quarkus.test.junit.QuarkusTest; +import org.eclipse.microprofile.config.inject.ConfigProperty; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeEach; + +@QuarkusTest +public abstract class SimpleFileRepoTest { + @ConfigProperty(name = "dhfs.filerepo.root") + String tempDirectory; + + @BeforeEach + void setup() { + + } + + @AfterAll + static void teardown() { + + } +} diff --git a/server/src/test/resources/application.properties b/server/src/test/resources/application.properties new file mode 100644 index 00000000..a0a22e30 --- /dev/null +++ b/server/src/test/resources/application.properties @@ -0,0 +1 @@ +dhfs.filerepo.root=${HOME}/dhfs_root_test \ No newline at end of file