From bbf275855c93b829636787143de7e4474a6e30bb Mon Sep 17 00:00:00 2001 From: Stepan Usatiuk Date: Sat, 30 Nov 2024 20:08:53 +0100 Subject: [PATCH] dump --- dhfs-parent/objects/pom.xml | 99 +++++ .../com/usatiuk/dhfs/objects/DataLocker.java | 40 ++ .../java/com/usatiuk/dhfs/objects/JData.java | 11 + .../com/usatiuk/dhfs/objects/JObject.java | 11 + .../dhfs/objects/JObjectInterface.java | 9 + .../com/usatiuk/dhfs/objects/JObjectKey.java | 4 + .../usatiuk/dhfs/objects/JObjectManager.java | 90 +++++ .../com/usatiuk/dhfs/objects/LockWrapper.java | 60 +++ .../persistence/ObjectPersistentStore.java | 33 ++ .../SerializingFileObjectPersistentStore.java | 351 ++++++++++++++++++ .../dhfs/objects/persistence/TxManifest.java | 13 + .../com/usatiuk/dhfs/objects/ObjectsTest.java | 97 +++++ .../persistence/FakeObjectStorage.java | 80 ++++ .../usatiuk/dhfs/objects/test/objs/Kid.java | 18 + .../dhfs/objects/test/objs/KidData.java | 19 + .../dhfs/objects/test/objs/KidDataImpl.java | 28 ++ .../dhfs/objects/test/objs/Parent.java | 25 ++ .../dhfs/objects/test/objs/ParentData.java | 14 + .../objects/test/objs/ParentDataImpl.java | 41 ++ .../dhfs/objects/test/objs/TestData.java | 34 ++ dhfs-parent/pom.xml | 7 + dhfs-parent/server/pom.xml | 11 +- .../files/service/DhfsFileServiceImpl.java | 2 +- .../jkleppmanntree/JKleppmannTreeManager.java | 2 +- .../dhfs/objects/jrepository/JObject.java | 2 +- .../objects/jrepository/JObjectManager.java | 2 +- .../jrepository/JObjectManagerImpl.java | 2 +- .../jrepository/JObjectRefProcessor.java | 2 +- .../objects/jrepository/JObjectTxManager.java | 2 +- .../dhfs/objects/jrepository/TxWriteback.java | 2 +- .../objects/jrepository/TxWritebackImpl.java | 2 +- .../repository/PersistentPeerDataService.java | 2 +- .../repository/RemoteObjectServiceServer.java | 2 +- .../dhfs/objects/repository/SyncHandler.java | 2 +- .../autosync/AutoSyncProcessor.java | 2 +- .../DeferredInvalidationQueueService.java | 2 +- .../InvalidationQueueService.java | 2 +- .../repository/opsupport/OpSender.java | 2 +- .../FileObjectPersistentStore.java | 6 +- dhfs-parent/utils/pom.xml | 59 +++ .../com/usatiuk/dhfs}/utils/ByteUtils.java | 2 +- .../utils/HashSetDelayedBlockingQueue.java | 7 +- .../dhfs/utils}/SerializationHelper.java | 6 +- .../StatusRuntimeExceptionNoStacktrace.java | 2 +- .../java/com/usatiuk/dhfs}/utils/VoidFn.java | 2 +- .../HashSetDelayedBlockingQueueTest.java | 2 +- 46 files changed, 1183 insertions(+), 30 deletions(-) create mode 100644 dhfs-parent/objects/pom.xml create mode 100644 dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/DataLocker.java create mode 100644 dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/JData.java create mode 100644 dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/JObject.java create mode 100644 dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/JObjectInterface.java create mode 100644 dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/JObjectKey.java create mode 100644 dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/JObjectManager.java create mode 100644 dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/LockWrapper.java create mode 100644 dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/persistence/ObjectPersistentStore.java create mode 100644 dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/persistence/SerializingFileObjectPersistentStore.java create mode 100644 dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/persistence/TxManifest.java create mode 100644 dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/ObjectsTest.java create mode 100644 dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/persistence/FakeObjectStorage.java create mode 100644 dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/test/objs/Kid.java create mode 100644 dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/test/objs/KidData.java create mode 100644 dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/test/objs/KidDataImpl.java create mode 100644 dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/test/objs/Parent.java create mode 100644 dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/test/objs/ParentData.java create mode 100644 dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/test/objs/ParentDataImpl.java create mode 100644 dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/test/objs/TestData.java create mode 100644 dhfs-parent/utils/pom.xml rename dhfs-parent/{server/src/main/java/com/usatiuk => utils/src/main/java/com/usatiuk/dhfs}/utils/ByteUtils.java (93%) rename dhfs-parent/{server/src/main/java/com/usatiuk => utils/src/main/java/com/usatiuk/dhfs}/utils/HashSetDelayedBlockingQueue.java (98%) rename dhfs-parent/{server/src/main/java/com/usatiuk/dhfs => utils/src/main/java/com/usatiuk/dhfs/utils}/SerializationHelper.java (90%) rename dhfs-parent/{server/src/main/java/com/usatiuk => utils/src/main/java/com/usatiuk/dhfs}/utils/StatusRuntimeExceptionNoStacktrace.java (94%) rename dhfs-parent/{server/src/main/java/com/usatiuk => utils/src/main/java/com/usatiuk/dhfs}/utils/VoidFn.java (68%) rename dhfs-parent/{server/src/test/java/com/usatiuk => utils/src/test/java/com/usatiuk/dhfs}/utils/HashSetDelayedBlockingQueueTest.java (99%) diff --git a/dhfs-parent/objects/pom.xml b/dhfs-parent/objects/pom.xml new file mode 100644 index 00000000..49970b92 --- /dev/null +++ b/dhfs-parent/objects/pom.xml @@ -0,0 +1,99 @@ + + + 4.0.0 + + com.usatiuk.dhfs + parent + 1.0-SNAPSHOT + + + objects + + + 21 + 21 + UTF-8 + + + + + io.quarkus + quarkus-junit5 + test + + + io.quarkus + quarkus-arc + + + io.quarkus + quarkus-grpc + + + net.openhft + zero-allocation-hashing + + + org.projectlombok + lombok + provided + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.apache.commons + commons-lang3 + + + org.jboss.slf4j + slf4j-jboss-logmanager + test + + + com.usatiuk.dhfs + utils + 1.0-SNAPSHOT + + + com.usatiuk.dhfs + supportlib + 1.0-SNAPSHOT + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + 1C + false + classes + + + + ${quarkus.platform.group-id} + quarkus-maven-plugin + ${quarkus.platform.version} + true + + + quarkus-plugin + + build + generate-code + generate-code-tests + + + + + + + + \ No newline at end of file diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/DataLocker.java b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/DataLocker.java new file mode 100644 index 00000000..8b45f4ab --- /dev/null +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/DataLocker.java @@ -0,0 +1,40 @@ +package com.usatiuk.dhfs.objects; + +import io.quarkus.logging.Log; +import jakarta.enterprise.context.ApplicationScoped; + +import java.lang.ref.Cleaner; +import java.lang.ref.WeakReference; +import java.util.concurrent.ConcurrentHashMap; + +@ApplicationScoped +public class DataLocker { + private final ConcurrentHashMap>> _locks = new ConcurrentHashMap<>(); + private final static Cleaner CLEANER = Cleaner.create(); + + public LockWrapper get(T data) { + while (true) { + var have = _locks.get(data.getKey()); + if (have != null) { + var ret = have.get(); + if (ret != null) { + if (ret.sameObject(data)) { + return (LockWrapper) ret; + } else { + Log.warn("Removed stale lock for " + data.getKey()); + _locks.remove(data.getKey(), have); + } + } + } + + var ret = new LockWrapper<>(data); + var ref = new WeakReference<>(ret); + + if (_locks.putIfAbsent(data.getKey(), ref) == null) { + CLEANER.register(ret, () -> _locks.remove(data.getKey(), ref)); + return ret; + } + } + } + +} diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/JData.java b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/JData.java new file mode 100644 index 00000000..608ea9f5 --- /dev/null +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/JData.java @@ -0,0 +1,11 @@ +package com.usatiuk.dhfs.objects; + +import java.util.function.Function; + +public interface JData { + JObjectKey getKey(); + + JData bindCopy(); + + Function binder(); +} diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/JObject.java b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/JObject.java new file mode 100644 index 00000000..62f32be6 --- /dev/null +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/JObject.java @@ -0,0 +1,11 @@ +package com.usatiuk.dhfs.objects; + +public abstract class JObject { + protected final JObjectInterface _jObjectInterface; + + public JObject(JObjectInterface jObjectInterface) { + _jObjectInterface = jObjectInterface; + } + + public abstract JData getData(); +} diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/JObjectInterface.java b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/JObjectInterface.java new file mode 100644 index 00000000..9860727e --- /dev/null +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/JObjectInterface.java @@ -0,0 +1,9 @@ +package com.usatiuk.dhfs.objects; + +import java.util.Optional; + +public interface JObjectInterface { + Optional getObject(JObjectKey key); + + Optional getObject(JObjectKey key, Class type); +} diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/JObjectKey.java b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/JObjectKey.java new file mode 100644 index 00000000..9927eeee --- /dev/null +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/JObjectKey.java @@ -0,0 +1,4 @@ +package com.usatiuk.dhfs.objects; + +public record JObjectKey(String name) { +} diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/JObjectManager.java b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/JObjectManager.java new file mode 100644 index 00000000..0a55315f --- /dev/null +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/JObjectManager.java @@ -0,0 +1,90 @@ +package com.usatiuk.dhfs.objects; + +import com.usatiuk.dhfs.objects.persistence.ObjectPersistentStore; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +@ApplicationScoped +public class JObjectManager { + @Inject + ObjectPersistentStore objectStorage; + + @Inject + DataLocker dataLocker; + + public class Transaction implements JObjectInterface { + private final Map _objects = new HashMap<>(); + + private JObject dataToObject(JData data) { + return data.binder().apply(this); + } + + @Override + public Optional getObject(JObjectKey key) { + if (_objects.containsKey(key)) { + return Optional.of(_objects.get(key)); + } + + var data = objectStorage.readObject(key).orElse(null); + if (data == null) { + return Optional.empty(); + } + var ret = dataToObject(data); + _objects.put(key, ret); + return Optional.of(ret); + } + + @Override + public Optional getObject(JObjectKey key, Class type) { + if (_objects.containsKey(key)) { + var got = _objects.get(key); + if (type.isInstance(got)) { + return Optional.of(type.cast(got)); + } else { + throw new IllegalArgumentException("Object type mismatch"); + } + } + + var data = objectStorage.readObject(key).orElse(null); + if (data == null) { + return Optional.empty(); + } + var got = dataToObject(data); + if (type.isInstance(got)) { + _objects.put(key, got); + return Optional.of(type.cast(got)); + } else { + throw new IllegalArgumentException("Object type mismatch"); + } + } + + public void commit() { + _objects.forEach((key, value) -> { + var data = (TestData) value.getData(); + + if (!data.isChanged()) { + return; + } + + if (_objectStorage.get(key) == null) { + _objectStorage.put(data.copy()); + return; + } + + if (_objectStorage.get(key).getVersion() <= data.getVersion()) { + _objectStorage.put(data.copy()); + } else { + throw new IllegalArgumentException("Version mismatch"); + } + }); + } + } + + public Transaction beginTransaction() { + return new Transaction(); + } +} diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/LockWrapper.java b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/LockWrapper.java new file mode 100644 index 00000000..4538a6aa --- /dev/null +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/LockWrapper.java @@ -0,0 +1,60 @@ +package com.usatiuk.dhfs.objects; + +import java.util.concurrent.locks.ReentrantReadWriteLock; + +public class LockWrapper { + private final JData _data; + private final ReentrantReadWriteLock _lock = new ReentrantReadWriteLock(); + + public LockWrapper(T data) { + _data = data; + } + + public boolean sameObject(JData data) { + return _data == data; + } + + interface DataAccessor extends AutoCloseable { + T getData(); + } + + public class ReadLocked implements DataAccessor { + public ReadLocked() { + _lock.readLock().lock(); + } + + @Override + public void close() { + _lock.readLock().unlock(); + } + + @Override + public B getData() { + return (B) _data; + } + } + + public ReadLocked read() { + return new ReadLocked<>(); + } + + public class WriteLocked implements DataAccessor { + public WriteLocked() { + _lock.writeLock().lock(); + } + + @Override + public void close() { + _lock.writeLock().unlock(); + } + + @Override + public B getData() { + return (B) _data; + } + } + + public WriteLocked write() { + return new WriteLocked<>(); + } +} diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/persistence/ObjectPersistentStore.java b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/persistence/ObjectPersistentStore.java new file mode 100644 index 00000000..e03af1a4 --- /dev/null +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/persistence/ObjectPersistentStore.java @@ -0,0 +1,33 @@ +package com.usatiuk.dhfs.objects.persistence; + +import com.usatiuk.dhfs.objects.JData; +import com.usatiuk.dhfs.objects.JObjectKey; + +import javax.annotation.Nonnull; +import java.util.Collection; +import java.util.Optional; + +public interface ObjectPersistentStore { + @Nonnull + Collection findAllObjects(); + + @Nonnull + Optional readObject(JObjectKey name); + + void writeObjectDirect(JObjectKey name, JData object); + + void writeObject(JObjectKey name, JData object); + + + + void commitTx(TxManifest names); + + // Deletes object metadata and data + void deleteObjectDirect(JObjectKey name); + + long getTotalSpace(); + + long getFreeSpace(); + + long getUsableSpace(); +} diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/persistence/SerializingFileObjectPersistentStore.java b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/persistence/SerializingFileObjectPersistentStore.java new file mode 100644 index 00000000..08b9decd --- /dev/null +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/persistence/SerializingFileObjectPersistentStore.java @@ -0,0 +1,351 @@ +package com.usatiuk.dhfs.objects.persistence; + +import com.google.protobuf.UnsafeByteOperations; +import com.usatiuk.dhfs.objects.JData; +import com.usatiuk.dhfs.objects.JObjectKey; +import com.usatiuk.dhfs.supportlib.UninitializedByteBuffer; +import com.usatiuk.dhfs.utils.ByteUtils; +import com.usatiuk.dhfs.utils.SerializationHelper; +import com.usatiuk.dhfs.utils.StatusRuntimeExceptionNoStacktrace; +import io.grpc.Status; +import io.quarkus.logging.Log; +import io.quarkus.runtime.ShutdownEvent; +import io.quarkus.runtime.StartupEvent; +import jakarta.annotation.Priority; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.event.Observes; +import net.openhft.hashing.LongHashFunction; +import org.apache.commons.lang3.concurrent.BasicThreadFactory; +import org.eclipse.microprofile.config.inject.ConfigProperty; + +import javax.annotation.Nonnull; +import java.io.*; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.util.*; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; + +// File format: +// 64-bit metadata serialized size +// 64-bit offset of "rest of" metadata (if -1 then file has no data, +// if 0 then file has data and metadata fits into META_BLOCK_SIZE) +// Until META_BLOCK_SIZE - metadata (encoded as ObjectMetadataP) +// data (encoded as JObjectDataP) +// rest of metadata + +@ApplicationScoped +public class SerializingFileObjectPersistentStore implements ObjectPersistentStore { + private final Path _root; + private final Path _txManifest; + private ExecutorService _flushExecutor; + private RandomAccessFile _txFile; + private volatile boolean _ready = false; + + public SerializingFileObjectPersistentStore(@ConfigProperty(name = "dhfs.objects.persistence.files.root") String root) { + this._root = Path.of(root).resolve("objects"); + _txManifest = Path.of(root).resolve("cur-tx-manifest"); + } + + void init(@Observes @Priority(100) StartupEvent event) throws IOException { + if (!_root.toFile().exists()) { + Log.info("Initializing with root " + _root); + _root.toFile().mkdirs(); + for (int i = 0; i < 256; i++) { + _root.resolve(String.valueOf(i)).toFile().mkdirs(); + } + } + if (!Files.exists(_txManifest)) { + Files.createFile(_txManifest); + } + _txFile = new RandomAccessFile(_txManifest.toFile(), "rw"); + { + BasicThreadFactory factory = new BasicThreadFactory.Builder() + .namingPattern("persistent-commit-%d") + .build(); + + _flushExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors(), factory); + } + + tryReplay(); + Log.info("Transaction replay done"); + _ready = true; + } + + void shutdown(@Observes @Priority(900) ShutdownEvent event) throws IOException { + _ready = false; + Log.debug("Deleting manifest file"); + _txFile.close(); + Files.delete(_txManifest); + Log.debug("Manifest file deleted"); + } + + private void verifyReady() { + if (!_ready) throw new IllegalStateException("Wrong service order!"); + } + + private void tryReplay() { + var read = readTxManifest(); + if (read != null) + commitTxImpl(read, false); + } + + private Path getObjPath(@Nonnull JObjectKey obj) { + int h = Objects.hash(obj); + int p1 = h & 0b00000000_00000000_11111111_00000000; + return _root.resolve(String.valueOf(p1 >> 8)).resolve(obj.toString()); + } + + private Path getTmpObjPath(@Nonnull JObjectKey obj) { + int h = Objects.hash(obj); + int p1 = h & 0b00000000_00000000_11111111_00000000; + return _root.resolve(String.valueOf(p1 >> 8)).resolve(obj + ".tmp"); + } + + private void findAllObjectsImpl(Collection out, Path path) { + var read = path.toFile().listFiles(); + if (read == null) return; + + for (var s : read) { + if (s.isDirectory()) { + findAllObjectsImpl(out, s.toPath()); + } else { + if (s.getName().endsWith(".tmp")) continue; // FIXME: + out.add(new JObjectKey(s.getName())); // FIXME: + } + } + } + + @Nonnull + @Override + public Collection findAllObjects() { + verifyReady(); + ArrayList out = new ArrayList<>(); + findAllObjectsImpl(out, _root); + return Collections.unmodifiableCollection(out); + } + + @Nonnull + @Override + public Optional readObject(JObjectKey name) { + verifyReady(); + var path = getObjPath(name); + try (var rf = new RandomAccessFile(path.toFile(), "r")) { + ByteBuffer buf = UninitializedByteBuffer.allocateUninitialized(Math.toIntExact(rf.getChannel().size())); + fillBuffer(buf, rf.getChannel()); + buf.flip(); + + var bs = UnsafeByteOperations.unsafeWrap(buf); + // This way, the input will be considered "immutable" which would allow avoiding copies + // when parsing byte arrays + var ch = bs.newCodedInput(); + ch.enableAliasing(true); +// return JObjectDataP.parseFrom(ch); + return null; + } catch (EOFException | FileNotFoundException | NoSuchFileException fx) { + return Optional.empty(); + } catch (IOException e) { + Log.error("Error reading file " + path, e); + throw new StatusRuntimeExceptionNoStacktrace(Status.INTERNAL); + } + } + + private void fillBuffer(ByteBuffer dst, FileChannel src) throws IOException { + int rem = dst.remaining(); + int readTotal = 0; + int readCur = 0; + while (readTotal < rem && (readCur = src.read(dst)) != -1) { + readTotal += readCur; + } + if (rem != readTotal) + throw new EOFException(); + } + + private void writeObjectImpl(Path path, JData data, boolean sync) throws IOException { + try (var fsb = new FileOutputStream(path.toFile(), false)) { +// int dataSize = data.getSerializedSize(); + int dataSize = 0; + +// if (fsb.getChannel().write(metaBb.limit(META_BLOCK_SIZE)) != META_BLOCK_SIZE) +// throw new IOException("Could not write to file"); + + if (sync) { + fsb.flush(); + fsb.getFD().sync(); + } + } + } + + @Override + public void writeObjectDirect(JObjectKey name, JData data) { + verifyReady(); + try { + var path = getObjPath(name); + writeObjectImpl(path, data, false); + } catch (IOException e) { + Log.error("Error writing file " + name, e); + throw new StatusRuntimeExceptionNoStacktrace(Status.INTERNAL); + } + } + + @Override + public void writeObject(JObjectKey name, JData obj) { + verifyReady(); + try { + var tmpPath = getTmpObjPath(name); + writeObjectImpl(tmpPath, obj, true); + } catch (IOException e) { + Log.error("Error writing new file " + name, e); + } + } + + + private TxManifest readTxManifest() { + try { + var channel = _txFile.getChannel(); + + if (channel.size() == 0) + return null; + + channel.position(0); + + var buf = ByteBuffer.allocate(Math.toIntExact(channel.size())); + + fillBuffer(buf, channel); + buf.flip(); + + long checksum = buf.getLong(); + var data = buf.slice(); + var hash = LongHashFunction.xx3().hashBytes(data); + + if (hash != checksum) + throw new StatusRuntimeExceptionNoStacktrace(Status.DATA_LOSS.withDescription("Transaction manifest checksum mismatch!")); + + return SerializationHelper.deserialize(data.array(), data.arrayOffset()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private void putTxManifest(TxManifest manifest) { + try { + var channel = _txFile.getChannel(); + var data = SerializationHelper.serializeArray(manifest); + channel.truncate(data.length + 8); + channel.position(0); + var hash = LongHashFunction.xx3().hashBytes(data); + if (channel.write(ByteUtils.longToBb(hash)) != 8) + throw new StatusRuntimeExceptionNoStacktrace(Status.INTERNAL); + if (channel.write(ByteBuffer.wrap(data)) != data.length) + throw new StatusRuntimeExceptionNoStacktrace(Status.INTERNAL); + channel.force(true); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void commitTx(TxManifest manifest) { + verifyReady(); + commitTxImpl(manifest, true); + } + + public void commitTxImpl(TxManifest manifest, boolean failIfNotFound) { + try { + if (manifest.getDeleted().isEmpty() && manifest.getWritten().isEmpty()) { + Log.debug("Empty manifest, skipping"); + return; + } + + putTxManifest(manifest); + + var latch = new CountDownLatch(manifest.getWritten().size() + manifest.getDeleted().size()); + ConcurrentLinkedQueue errors = new ConcurrentLinkedQueue<>(); + + for (var n : manifest.getWritten()) { + _flushExecutor.execute(() -> { + try { + Files.move(getTmpObjPath(n), getObjPath(n), ATOMIC_MOVE, REPLACE_EXISTING); + } catch (Throwable t) { + if (!failIfNotFound && (t instanceof NoSuchFileException)) return; + Log.error("Error writing " + n, t); + errors.add(t); + } finally { + latch.countDown(); + } + }); + } + for (var d : manifest.getDeleted()) { + _flushExecutor.execute(() -> { + try { + deleteImpl(getObjPath(d)); + } catch (Throwable t) { + Log.error("Error deleting " + d, t); + errors.add(t); + } finally { + latch.countDown(); + } + }); + } + + latch.await(); + + if (!errors.isEmpty()) { + throw new RuntimeException("Errors when commiting tx!"); + } + + // No real need to truncate here +// try (var channel = _txFile.getChannel()) { +// channel.truncate(0); +// } +// } catch (IOException e) { +// Log.error("Failed committing transaction to disk: ", e); +// throw new RuntimeException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + private void deleteImpl(Path path) { + try { + Files.delete(path); + } catch (NoSuchFileException ignored) { + } catch (IOException e) { + Log.error("Error deleting file " + path, e); + throw new StatusRuntimeExceptionNoStacktrace(Status.INTERNAL); + } + } + + @Override + public void deleteObjectDirect(JObjectKey name) { + verifyReady(); + deleteImpl(getObjPath(name)); + } + + @Override + public long getTotalSpace() { + verifyReady(); + return _root.toFile().getTotalSpace(); + } + + @Override + public long getFreeSpace() { + verifyReady(); + return _root.toFile().getFreeSpace(); + } + + @Override + public long getUsableSpace() { + verifyReady(); + return _root.toFile().getUsableSpace(); + } + +} diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/persistence/TxManifest.java b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/persistence/TxManifest.java new file mode 100644 index 00000000..3a91f71e --- /dev/null +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/persistence/TxManifest.java @@ -0,0 +1,13 @@ +package com.usatiuk.dhfs.objects.persistence; + +import com.usatiuk.dhfs.objects.JObjectKey; + +import java.io.Serializable; +import java.util.List; + +// FIXME: Serializable +public interface TxManifest extends Serializable { + List getWritten(); + + List getDeleted(); +} diff --git a/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/ObjectsTest.java b/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/ObjectsTest.java new file mode 100644 index 00000000..6a65b186 --- /dev/null +++ b/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/ObjectsTest.java @@ -0,0 +1,97 @@ +package com.usatiuk.dhfs.objects; + +import com.usatiuk.dhfs.objects.persistence.FakeObjectStorage; +import com.usatiuk.dhfs.objects.test.objs.Kid; +import com.usatiuk.dhfs.objects.test.objs.Parent; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class ObjectsTest { + private final FakeObjectStorage _storage = new FakeObjectStorage(); + private final JObjectManager _tx = new JObjectManager(_storage); + + @Test + void createObject() { + { + var tx = _tx.beginTransaction(); + var parent = tx.getObject(new JObjectKey("Parent"), Parent.class); + parent.setName("John"); + tx.commit(); + } + + { + var tx2 = _tx.beginTransaction(); + var parent = tx2.getObject(new JObjectKey("Parent")); + Assertions.assertInstanceOf(Parent.class, parent); + Assertions.assertEquals("John", ((Parent) parent).getName()); + } + } + + @Test + void createObjectConflict() { + { + var tx = _tx.beginTransaction(); + var parent = tx.getObject(new JObjectKey("Parent"), Parent.class); + parent.setName("John"); + + var tx2 = _tx.beginTransaction(); + var parent2 = tx2.getObject(new JObjectKey("Parent"), Parent.class); + parent2.setName("John"); + + tx.commit(); + Assertions.assertThrows(Exception.class, tx2::commit); + } + } + + @Test + void editConflict() { + { + var tx = _tx.beginTransaction(); + var parent = tx.getObject(new JObjectKey("Parent"), Parent.class); + parent.setName("John"); + tx.commit(); + } + + { + var tx = _tx.beginTransaction(); + var parent = tx.getObject(new JObjectKey("Parent"), Parent.class); + parent.setName("John2"); + + var tx2 = _tx.beginTransaction(); + var parent2 = tx2.getObject(new JObjectKey("Parent"), Parent.class); + parent2.setName("John3"); + + tx.commit(); + Assertions.assertThrows(Exception.class, tx2::commit); + } + + { + var tx2 = _tx.beginTransaction(); + var parent = tx2.getObject(new JObjectKey("Parent")); + Assertions.assertInstanceOf(Parent.class, parent); + Assertions.assertEquals("John2", ((Parent) parent).getName()); + } + } + + @Test + void nestedCreate() { + { + var tx = _tx.beginTransaction(); + var parent = tx.getObject(new JObjectKey("Parent"), Parent.class); + var kid = tx.getObject(new JObjectKey("Kid"), Kid.class); + parent.setName("John"); + kid.setName("KidName"); + parent.setKidKey(kid.getKey()); + tx.commit(); + } + + { + var tx2 = _tx.beginTransaction(); + var parent = tx2.getObject(new JObjectKey("Parent")); + Assertions.assertInstanceOf(Parent.class, parent); + Assertions.assertEquals("John", ((Parent) parent).getName()); + Assertions.assertEquals("KidName", ((Parent) parent).getKid().getName()); + } + } + +} diff --git a/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/persistence/FakeObjectStorage.java b/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/persistence/FakeObjectStorage.java new file mode 100644 index 00000000..5566f1f5 --- /dev/null +++ b/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/persistence/FakeObjectStorage.java @@ -0,0 +1,80 @@ +package com.usatiuk.dhfs.objects.persistence; + +import com.usatiuk.dhfs.objects.JData; +import com.usatiuk.dhfs.objects.JObjectKey; +import com.usatiuk.dhfs.objects.test.objs.TestData; + +import javax.annotation.Nonnull; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +public class FakeObjectStorage implements ObjectPersistentStore { + private final Map _objects = new HashMap<>(); + private final Map _pending = new HashMap<>(); + + @Nonnull + @Override + public Collection findAllObjects() { + synchronized (this) { + return _objects.keySet(); + } + } + + @Nonnull + @Override + public Optional readObject(JObjectKey name) { + synchronized (this) { + return Optional.ofNullable(_objects.get(name)); + } + } + + @Override + public void writeObjectDirect(JObjectKey name, JData object) { + synchronized (this) { + _objects.put(name, (TestData) object); + } + } + + @Override + public void writeObject(JObjectKey name, JData object) { + synchronized (this) { + _pending.put(name, (TestData) object); + } + } + + @Override + public void commitTx(TxManifest names) { + synchronized (this) { + for (JObjectKey key : names.getWritten()) { + _objects.put(key, _pending.get(key)); + } + for (JObjectKey key : names.getDeleted()) { + _objects.remove(key); + } + } + } + + @Override + public void deleteObjectDirect(JObjectKey name) { + synchronized (this) { + _objects.remove(name); + } + } + + @Override + public long getTotalSpace() { + return 0; + } + + @Override + public long getFreeSpace() { + return 0; + } + + @Override + public long getUsableSpace() { + return 0; + } +} diff --git a/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/test/objs/Kid.java b/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/test/objs/Kid.java new file mode 100644 index 00000000..d5fb404d --- /dev/null +++ b/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/test/objs/Kid.java @@ -0,0 +1,18 @@ +package com.usatiuk.dhfs.objects.test.objs; + +import com.usatiuk.dhfs.objects.JData; +import com.usatiuk.dhfs.objects.JObject; +import com.usatiuk.dhfs.objects.JObjectInterface; + +public class Kid extends JObject { + + public Kid(JObjectInterface jObjectInterface, KidData data) { + super(jObjectInterface, data); + } + + @Override + public JData getData() { + return _data; + } + +} diff --git a/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/test/objs/KidData.java b/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/test/objs/KidData.java new file mode 100644 index 00000000..e519803e --- /dev/null +++ b/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/test/objs/KidData.java @@ -0,0 +1,19 @@ +package com.usatiuk.dhfs.objects.test.objs; + +import com.usatiuk.dhfs.objects.JData; +import com.usatiuk.dhfs.objects.JObject; +import com.usatiuk.dhfs.objects.JObjectInterface; + +import java.util.function.Function; + +public interface KidData extends JData { + String getName(); + + void setName(String name); + + KidData bindCopy(); + + default Function binder() { + return jo -> new Kid(jo, bindCopy()); + } +} diff --git a/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/test/objs/KidDataImpl.java b/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/test/objs/KidDataImpl.java new file mode 100644 index 00000000..48b8baf6 --- /dev/null +++ b/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/test/objs/KidDataImpl.java @@ -0,0 +1,28 @@ +package com.usatiuk.dhfs.objects.test.objs; + +import com.usatiuk.dhfs.objects.JObjectKey; + +public class KidDataImpl extends TestData implements KidData { + private String _name; + + public KidDataImpl(long version, JObjectKey key, String name) { + super(version, key); + _name = name; + } + + @Override + public String getName() { + return _name; + } + + @Override + public void setName(String name) { + _name = name; + onChanged(); + } + + @Override + public KidDataImpl copy() { + return new KidDataImpl(isChanged() ? getVersion() + 1 : getVersion(), getKey(), _name); + } +} diff --git a/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/test/objs/Parent.java b/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/test/objs/Parent.java new file mode 100644 index 00000000..176c983f --- /dev/null +++ b/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/test/objs/Parent.java @@ -0,0 +1,25 @@ +package com.usatiuk.dhfs.objects.test.objs; + +import com.usatiuk.dhfs.objects.JData; +import com.usatiuk.dhfs.objects.JObject; +import com.usatiuk.dhfs.objects.JObjectInterface; +import lombok.experimental.Delegate; + +public class Parent extends JObject { + @Delegate + private final ParentData _data; + + public Parent(JObjectInterface jObjectInterface, ParentData data) { + super(jObjectInterface); + _data = data; + } + + @Override + public JData getData() { + return _data; + } + + public Kid getKid() { + return _jObjectInterface.getObject(_data.getKidKey(), Kid.class); + } +} diff --git a/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/test/objs/ParentData.java b/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/test/objs/ParentData.java new file mode 100644 index 00000000..b3f0e76f --- /dev/null +++ b/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/test/objs/ParentData.java @@ -0,0 +1,14 @@ +package com.usatiuk.dhfs.objects.test.objs; + +import com.usatiuk.dhfs.objects.JData; +import com.usatiuk.dhfs.objects.JObjectKey; + +public interface ParentData extends JData { + String getName(); + + void setName(String name); + + JObjectKey getKidKey(); + + void setKidKey(JObjectKey kid); +} diff --git a/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/test/objs/ParentDataImpl.java b/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/test/objs/ParentDataImpl.java new file mode 100644 index 00000000..c77a0020 --- /dev/null +++ b/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/test/objs/ParentDataImpl.java @@ -0,0 +1,41 @@ +package com.usatiuk.dhfs.objects.test.objs; + +import com.usatiuk.dhfs.objects.JObjectKey; + +public class ParentDataImpl extends TestData implements ParentData { + private String _name; + private JObjectKey _kidKey; + + public ParentDataImpl(long version, JObjectKey key, String name, JObjectKey kidKey) { + super(version, key); + _name = name; + _kidKey = kidKey; + } + + @Override + public String getName() { + return _name; + } + + @Override + public void setName(String name) { + _name = name; + onChanged(); + } + + @Override + public JObjectKey getKidKey() { + return _kidKey; + } + + @Override + public void setKidKey(JObjectKey kid) { + _kidKey = kid; + onChanged(); + } + + @Override + public ParentDataImpl copy() { + return new ParentDataImpl(isChanged() ? getVersion() + 1 : getVersion(), getKey(), _name, _kidKey); + } +} diff --git a/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/test/objs/TestData.java b/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/test/objs/TestData.java new file mode 100644 index 00000000..0bf25df1 --- /dev/null +++ b/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/test/objs/TestData.java @@ -0,0 +1,34 @@ +package com.usatiuk.dhfs.objects.test.objs; + +import com.usatiuk.dhfs.objects.JData; +import com.usatiuk.dhfs.objects.JObjectKey; + +public abstract class TestData implements JData { + private boolean _changed = false; + private final long _version; + private final JObjectKey _key; + + protected TestData(long version, JObjectKey key) { + _version = version; + _key = key; + } + + void onChanged() { + _changed = true; + } + + public boolean isChanged() { + return _changed; + } + + public long getVersion() { + return _version; + } + + @Override + public JObjectKey getKey() { + return _key; + } + + public abstract TestData copy(); +} diff --git a/dhfs-parent/pom.xml b/dhfs-parent/pom.xml index 8597d81c..3140d94e 100644 --- a/dhfs-parent/pom.xml +++ b/dhfs-parent/pom.xml @@ -15,6 +15,8 @@ kleppmanntree supportlib autoprotomap + objects + utils @@ -54,6 +56,11 @@ 1.18.34 provided + + net.openhft + zero-allocation-hashing + 0.16 + org.awaitility awaitility diff --git a/dhfs-parent/server/pom.xml b/dhfs-parent/server/pom.xml index d9e34d5d..bb74c72a 100644 --- a/dhfs-parent/server/pom.xml +++ b/dhfs-parent/server/pom.xml @@ -51,7 +51,6 @@ net.openhft zero-allocation-hashing - 0.16 io.quarkus @@ -147,6 +146,16 @@ supportlib 1.0-SNAPSHOT + + com.usatiuk.dhfs + objects + 1.0-SNAPSHOT + + + com.usatiuk.dhfs + utils + 1.0-SNAPSHOT + diff --git a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/files/service/DhfsFileServiceImpl.java b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/files/service/DhfsFileServiceImpl.java index 08bf639f..33b30d85 100644 --- a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/files/service/DhfsFileServiceImpl.java +++ b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/files/service/DhfsFileServiceImpl.java @@ -15,7 +15,7 @@ import com.usatiuk.dhfs.objects.jrepository.JObject; import com.usatiuk.dhfs.objects.jrepository.JObjectManager; import com.usatiuk.dhfs.objects.jrepository.JObjectTxManager; import com.usatiuk.dhfs.objects.repository.PersistentPeerDataService; -import com.usatiuk.utils.StatusRuntimeExceptionNoStacktrace; +import com.usatiuk.dhfs.utils.StatusRuntimeExceptionNoStacktrace; import io.grpc.Status; import io.grpc.StatusRuntimeException; import io.quarkus.logging.Log; diff --git a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jkleppmanntree/JKleppmannTreeManager.java b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jkleppmanntree/JKleppmannTreeManager.java index 43502d20..2743bf48 100644 --- a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jkleppmanntree/JKleppmannTreeManager.java +++ b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jkleppmanntree/JKleppmannTreeManager.java @@ -9,7 +9,7 @@ import com.usatiuk.dhfs.objects.repository.opsupport.OpObject; import com.usatiuk.dhfs.objects.repository.opsupport.OpObjectRegistry; import com.usatiuk.dhfs.objects.repository.opsupport.OpSender; import com.usatiuk.kleppmanntree.*; -import com.usatiuk.utils.VoidFn; +import com.usatiuk.dhfs.utils.VoidFn; import io.quarkus.logging.Log; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; diff --git a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jrepository/JObject.java b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jrepository/JObject.java index b21a9ece..1d0a9ca0 100644 --- a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jrepository/JObject.java +++ b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jrepository/JObject.java @@ -1,6 +1,6 @@ package com.usatiuk.dhfs.objects.jrepository; -import com.usatiuk.utils.VoidFn; +import com.usatiuk.dhfs.utils.VoidFn; public abstract class JObject { public abstract ObjectMetadata getMeta(); diff --git a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jrepository/JObjectManager.java b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jrepository/JObjectManager.java index 5c7ac28f..377c9533 100644 --- a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jrepository/JObjectManager.java +++ b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jrepository/JObjectManager.java @@ -1,6 +1,6 @@ package com.usatiuk.dhfs.objects.jrepository; -import com.usatiuk.utils.VoidFn; +import com.usatiuk.dhfs.utils.VoidFn; import jakarta.annotation.Nullable; import java.util.Collection; diff --git a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jrepository/JObjectManagerImpl.java b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jrepository/JObjectManagerImpl.java index 5a24c1e7..5cd3e2ce 100644 --- a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jrepository/JObjectManagerImpl.java +++ b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jrepository/JObjectManagerImpl.java @@ -7,7 +7,7 @@ import com.usatiuk.dhfs.objects.repository.PersistentPeerDataService; import com.usatiuk.dhfs.objects.repository.RemoteObjectServiceClient; import com.usatiuk.dhfs.objects.repository.invalidation.InvalidationQueueService; import com.usatiuk.dhfs.objects.repository.persistence.ObjectPersistentStore; -import com.usatiuk.utils.VoidFn; +import com.usatiuk.dhfs.utils.VoidFn; import io.grpc.Status; import io.grpc.StatusRuntimeException; import io.quarkus.logging.Log; diff --git a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jrepository/JObjectRefProcessor.java b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jrepository/JObjectRefProcessor.java index 66914cca..5de25357 100644 --- a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jrepository/JObjectRefProcessor.java +++ b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jrepository/JObjectRefProcessor.java @@ -3,7 +3,7 @@ package com.usatiuk.dhfs.objects.jrepository; import com.usatiuk.dhfs.objects.repository.PersistentPeerDataService; import com.usatiuk.dhfs.objects.repository.RemoteObjectServiceClient; import com.usatiuk.dhfs.objects.repository.autosync.AutoSyncProcessor; -import com.usatiuk.utils.HashSetDelayedBlockingQueue; +import com.usatiuk.dhfs.utils.HashSetDelayedBlockingQueue; import io.quarkus.logging.Log; import io.quarkus.runtime.ShutdownEvent; import io.quarkus.runtime.StartupEvent; diff --git a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jrepository/JObjectTxManager.java b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jrepository/JObjectTxManager.java index 3d8282aa..3634f3a2 100644 --- a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jrepository/JObjectTxManager.java +++ b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jrepository/JObjectTxManager.java @@ -4,7 +4,7 @@ import com.usatiuk.autoprotomap.runtime.ProtoSerializer; import com.usatiuk.dhfs.objects.persistence.JObjectDataP; import com.usatiuk.dhfs.objects.persistence.ObjectMetadataP; import com.usatiuk.dhfs.objects.repository.invalidation.InvalidationQueueService; -import com.usatiuk.utils.VoidFn; +import com.usatiuk.dhfs.utils.VoidFn; import io.quarkus.logging.Log; import jakarta.annotation.Nullable; import jakarta.enterprise.context.ApplicationScoped; diff --git a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jrepository/TxWriteback.java b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jrepository/TxWriteback.java index 14c6146f..70a4e60e 100644 --- a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jrepository/TxWriteback.java +++ b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jrepository/TxWriteback.java @@ -1,6 +1,6 @@ package com.usatiuk.dhfs.objects.jrepository; -import com.usatiuk.utils.VoidFn; +import com.usatiuk.dhfs.utils.VoidFn; public interface TxWriteback { TxBundle createBundle(); diff --git a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jrepository/TxWritebackImpl.java b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jrepository/TxWritebackImpl.java index db5e7119..ab1b1440 100644 --- a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jrepository/TxWritebackImpl.java +++ b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jrepository/TxWritebackImpl.java @@ -3,7 +3,7 @@ package com.usatiuk.dhfs.objects.jrepository; import com.usatiuk.dhfs.objects.persistence.JObjectDataP; import com.usatiuk.dhfs.objects.persistence.ObjectMetadataP; import com.usatiuk.dhfs.objects.repository.persistence.ObjectPersistentStore; -import com.usatiuk.utils.VoidFn; +import com.usatiuk.dhfs.utils.VoidFn; import io.quarkus.logging.Log; import io.quarkus.runtime.ShutdownEvent; import io.quarkus.runtime.StartupEvent; diff --git a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/PersistentPeerDataService.java b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/PersistentPeerDataService.java index 05f8f66f..0413d8b8 100644 --- a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/PersistentPeerDataService.java +++ b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/PersistentPeerDataService.java @@ -1,6 +1,6 @@ package com.usatiuk.dhfs.objects.repository; -import com.usatiuk.dhfs.SerializationHelper; +import com.usatiuk.dhfs.utils.SerializationHelper; import com.usatiuk.dhfs.ShutdownChecker; import com.usatiuk.dhfs.objects.jrepository.*; import com.usatiuk.dhfs.objects.repository.invalidation.InvalidationQueueService; diff --git a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/RemoteObjectServiceServer.java b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/RemoteObjectServiceServer.java index fde49ecb..17b9bb22 100644 --- a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/RemoteObjectServiceServer.java +++ b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/RemoteObjectServiceServer.java @@ -10,7 +10,7 @@ import com.usatiuk.dhfs.objects.repository.autosync.AutoSyncProcessor; import com.usatiuk.dhfs.objects.repository.invalidation.InvalidationQueueService; import com.usatiuk.dhfs.objects.repository.opsupport.Op; import com.usatiuk.dhfs.objects.repository.opsupport.OpObjectRegistry; -import com.usatiuk.utils.StatusRuntimeExceptionNoStacktrace; +import com.usatiuk.dhfs.utils.StatusRuntimeExceptionNoStacktrace; import io.grpc.Status; import io.grpc.StatusRuntimeException; import io.quarkus.grpc.GrpcService; diff --git a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/SyncHandler.java b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/SyncHandler.java index cc88f97d..136041a8 100644 --- a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/SyncHandler.java +++ b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/SyncHandler.java @@ -8,7 +8,7 @@ import com.usatiuk.dhfs.objects.jrepository.JObjectTxManager; import com.usatiuk.dhfs.objects.persistence.JObjectDataP; import com.usatiuk.dhfs.objects.repository.invalidation.InvalidationQueueService; import com.usatiuk.dhfs.objects.repository.opsupport.OpObjectRegistry; -import com.usatiuk.utils.StatusRuntimeExceptionNoStacktrace; +import com.usatiuk.dhfs.utils.StatusRuntimeExceptionNoStacktrace; import io.grpc.Status; import io.quarkus.logging.Log; import jakarta.enterprise.context.ApplicationScoped; diff --git a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/autosync/AutoSyncProcessor.java b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/autosync/AutoSyncProcessor.java index fcc5d702..0220c443 100644 --- a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/autosync/AutoSyncProcessor.java +++ b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/autosync/AutoSyncProcessor.java @@ -3,7 +3,7 @@ package com.usatiuk.dhfs.objects.repository.autosync; import com.usatiuk.dhfs.objects.jrepository.*; import com.usatiuk.dhfs.objects.repository.peersync.PeerDirectory; import com.usatiuk.dhfs.objects.repository.peersync.PersistentPeerInfo; -import com.usatiuk.utils.HashSetDelayedBlockingQueue; +import com.usatiuk.dhfs.utils.HashSetDelayedBlockingQueue; import io.quarkus.logging.Log; import io.quarkus.runtime.ShutdownEvent; import io.quarkus.runtime.Startup; diff --git a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/invalidation/DeferredInvalidationQueueService.java b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/invalidation/DeferredInvalidationQueueService.java index 575f65dc..e62e4d19 100644 --- a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/invalidation/DeferredInvalidationQueueService.java +++ b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/invalidation/DeferredInvalidationQueueService.java @@ -1,6 +1,6 @@ package com.usatiuk.dhfs.objects.repository.invalidation; -import com.usatiuk.dhfs.SerializationHelper; +import com.usatiuk.dhfs.utils.SerializationHelper; import com.usatiuk.dhfs.objects.repository.PeerManager; import io.quarkus.logging.Log; import io.quarkus.runtime.ShutdownEvent; diff --git a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/invalidation/InvalidationQueueService.java b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/invalidation/InvalidationQueueService.java index f754457c..b5424c28 100644 --- a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/invalidation/InvalidationQueueService.java +++ b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/invalidation/InvalidationQueueService.java @@ -6,7 +6,7 @@ import com.usatiuk.dhfs.objects.jrepository.JObjectManager; import com.usatiuk.dhfs.objects.repository.PeerManager; import com.usatiuk.dhfs.objects.repository.PersistentPeerDataService; import com.usatiuk.dhfs.objects.repository.RemoteObjectServiceClient; -import com.usatiuk.utils.HashSetDelayedBlockingQueue; +import com.usatiuk.dhfs.utils.HashSetDelayedBlockingQueue; import io.quarkus.logging.Log; import io.quarkus.runtime.ShutdownEvent; import io.quarkus.runtime.StartupEvent; diff --git a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/opsupport/OpSender.java b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/opsupport/OpSender.java index 9cd68547..3bf3b647 100644 --- a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/opsupport/OpSender.java +++ b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/opsupport/OpSender.java @@ -3,7 +3,7 @@ package com.usatiuk.dhfs.objects.repository.opsupport; import com.usatiuk.dhfs.objects.jrepository.JObjectTxManager; import com.usatiuk.dhfs.objects.repository.PeerManager; import com.usatiuk.dhfs.objects.repository.RemoteObjectServiceClient; -import com.usatiuk.utils.HashSetDelayedBlockingQueue; +import com.usatiuk.dhfs.utils.HashSetDelayedBlockingQueue; import io.quarkus.logging.Log; import io.quarkus.runtime.ShutdownEvent; import io.quarkus.runtime.Startup; diff --git a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/persistence/FileObjectPersistentStore.java b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/persistence/FileObjectPersistentStore.java index 3dd12370..493a8323 100644 --- a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/persistence/FileObjectPersistentStore.java +++ b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/repository/persistence/FileObjectPersistentStore.java @@ -3,13 +3,13 @@ package com.usatiuk.dhfs.objects.repository.persistence; import com.google.protobuf.ByteString; import com.google.protobuf.CodedOutputStream; import com.google.protobuf.UnsafeByteOperations; -import com.usatiuk.dhfs.SerializationHelper; +import com.usatiuk.dhfs.utils.SerializationHelper; import com.usatiuk.dhfs.objects.persistence.JObjectDataP; import com.usatiuk.dhfs.objects.persistence.ObjectMetadataP; import com.usatiuk.dhfs.supportlib.DhfsSupport; import com.usatiuk.dhfs.supportlib.UninitializedByteBuffer; -import com.usatiuk.utils.ByteUtils; -import com.usatiuk.utils.StatusRuntimeExceptionNoStacktrace; +import com.usatiuk.dhfs.utils.ByteUtils; +import com.usatiuk.dhfs.utils.StatusRuntimeExceptionNoStacktrace; import io.grpc.Status; import io.grpc.StatusRuntimeException; import io.quarkus.logging.Log; diff --git a/dhfs-parent/utils/pom.xml b/dhfs-parent/utils/pom.xml new file mode 100644 index 00000000..30db029a --- /dev/null +++ b/dhfs-parent/utils/pom.xml @@ -0,0 +1,59 @@ + + + 4.0.0 + + com.usatiuk.dhfs + parent + 1.0-SNAPSHOT + + + utils + + + 21 + 21 + UTF-8 + + + + + io.quarkus + quarkus-junit5 + test + + + io.quarkus + quarkus-arc + + + io.quarkus + quarkus-grpc + + + org.projectlombok + lombok + provided + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.apache.commons + commons-lang3 + + + org.jboss.slf4j + slf4j-jboss-logmanager + test + + + commons-io + commons-io + + + + \ No newline at end of file diff --git a/dhfs-parent/server/src/main/java/com/usatiuk/utils/ByteUtils.java b/dhfs-parent/utils/src/main/java/com/usatiuk/dhfs/utils/ByteUtils.java similarity index 93% rename from dhfs-parent/server/src/main/java/com/usatiuk/utils/ByteUtils.java rename to dhfs-parent/utils/src/main/java/com/usatiuk/dhfs/utils/ByteUtils.java index f7075b40..dba58508 100644 --- a/dhfs-parent/server/src/main/java/com/usatiuk/utils/ByteUtils.java +++ b/dhfs-parent/utils/src/main/java/com/usatiuk/dhfs/utils/ByteUtils.java @@ -1,4 +1,4 @@ -package com.usatiuk.utils; +package com.usatiuk.dhfs.utils; import java.nio.ByteBuffer; diff --git a/dhfs-parent/server/src/main/java/com/usatiuk/utils/HashSetDelayedBlockingQueue.java b/dhfs-parent/utils/src/main/java/com/usatiuk/dhfs/utils/HashSetDelayedBlockingQueue.java similarity index 98% rename from dhfs-parent/server/src/main/java/com/usatiuk/utils/HashSetDelayedBlockingQueue.java rename to dhfs-parent/utils/src/main/java/com/usatiuk/dhfs/utils/HashSetDelayedBlockingQueue.java index 51d23509..628bf4fd 100644 --- a/dhfs-parent/server/src/main/java/com/usatiuk/utils/HashSetDelayedBlockingQueue.java +++ b/dhfs-parent/utils/src/main/java/com/usatiuk/dhfs/utils/HashSetDelayedBlockingQueue.java @@ -1,4 +1,4 @@ -package com.usatiuk.utils; +package com.usatiuk.dhfs.utils; import jakarta.annotation.Nullable; import lombok.Getter; @@ -11,9 +11,12 @@ import java.util.function.Function; public class HashSetDelayedBlockingQueue { private final LinkedHashMap> _set = new LinkedHashMap<>(); private final Object _sleepSynchronizer = new Object(); - @Getter private long _delay; + public long getDelay() { + return _delay; + } + private boolean _closed = false; public HashSetDelayedBlockingQueue(long delay) { diff --git a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/SerializationHelper.java b/dhfs-parent/utils/src/main/java/com/usatiuk/dhfs/utils/SerializationHelper.java similarity index 90% rename from dhfs-parent/server/src/main/java/com/usatiuk/dhfs/SerializationHelper.java rename to dhfs-parent/utils/src/main/java/com/usatiuk/dhfs/utils/SerializationHelper.java index 977b2307..d285a821 100644 --- a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/SerializationHelper.java +++ b/dhfs-parent/utils/src/main/java/com/usatiuk/dhfs/utils/SerializationHelper.java @@ -1,8 +1,7 @@ -package com.usatiuk.dhfs; +package com.usatiuk.dhfs.utils; import com.google.protobuf.ByteString; import com.google.protobuf.UnsafeByteOperations; -import com.usatiuk.dhfs.files.objects.File; import org.apache.commons.io.input.ClassLoaderObjectInputStream; import org.apache.commons.lang3.SerializationUtils; @@ -12,10 +11,9 @@ import java.io.InputStream; import java.io.Serializable; public abstract class SerializationHelper { - // Taken from SerializationUtils public static T deserialize(final InputStream inputStream) { - try (ClassLoaderObjectInputStream in = new ClassLoaderObjectInputStream(File.class.getClassLoader(), inputStream)) { + try (ClassLoaderObjectInputStream in = new ClassLoaderObjectInputStream(SerializationHelper.class.getClassLoader(), inputStream)) { final T obj = (T) in.readObject(); return obj; } catch (IOException | ClassNotFoundException e) { diff --git a/dhfs-parent/server/src/main/java/com/usatiuk/utils/StatusRuntimeExceptionNoStacktrace.java b/dhfs-parent/utils/src/main/java/com/usatiuk/dhfs/utils/StatusRuntimeExceptionNoStacktrace.java similarity index 94% rename from dhfs-parent/server/src/main/java/com/usatiuk/utils/StatusRuntimeExceptionNoStacktrace.java rename to dhfs-parent/utils/src/main/java/com/usatiuk/dhfs/utils/StatusRuntimeExceptionNoStacktrace.java index 963da69d..40897edc 100644 --- a/dhfs-parent/server/src/main/java/com/usatiuk/utils/StatusRuntimeExceptionNoStacktrace.java +++ b/dhfs-parent/utils/src/main/java/com/usatiuk/dhfs/utils/StatusRuntimeExceptionNoStacktrace.java @@ -1,4 +1,4 @@ -package com.usatiuk.utils; +package com.usatiuk.dhfs.utils; import io.grpc.Metadata; import io.grpc.Status; diff --git a/dhfs-parent/server/src/main/java/com/usatiuk/utils/VoidFn.java b/dhfs-parent/utils/src/main/java/com/usatiuk/dhfs/utils/VoidFn.java similarity index 68% rename from dhfs-parent/server/src/main/java/com/usatiuk/utils/VoidFn.java rename to dhfs-parent/utils/src/main/java/com/usatiuk/dhfs/utils/VoidFn.java index 46f4ff0c..e20d6707 100644 --- a/dhfs-parent/server/src/main/java/com/usatiuk/utils/VoidFn.java +++ b/dhfs-parent/utils/src/main/java/com/usatiuk/dhfs/utils/VoidFn.java @@ -1,4 +1,4 @@ -package com.usatiuk.utils; +package com.usatiuk.dhfs.utils; @FunctionalInterface public interface VoidFn { diff --git a/dhfs-parent/server/src/test/java/com/usatiuk/utils/HashSetDelayedBlockingQueueTest.java b/dhfs-parent/utils/src/test/java/com/usatiuk/dhfs/utils/HashSetDelayedBlockingQueueTest.java similarity index 99% rename from dhfs-parent/server/src/test/java/com/usatiuk/utils/HashSetDelayedBlockingQueueTest.java rename to dhfs-parent/utils/src/test/java/com/usatiuk/dhfs/utils/HashSetDelayedBlockingQueueTest.java index d68998cf..70f36cc9 100644 --- a/dhfs-parent/server/src/test/java/com/usatiuk/utils/HashSetDelayedBlockingQueueTest.java +++ b/dhfs-parent/utils/src/test/java/com/usatiuk/dhfs/utils/HashSetDelayedBlockingQueueTest.java @@ -1,4 +1,4 @@ -package com.usatiuk.utils; +package com.usatiuk.dhfs.utils; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test;