diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/objects/iterators/IterProdFn.java b/dhfs-parent/objects/src/main/java/com/usatiuk/objects/iterators/IterProdFn.java deleted file mode 100644 index 05395201..00000000 --- a/dhfs-parent/objects/src/main/java/com/usatiuk/objects/iterators/IterProdFn.java +++ /dev/null @@ -1,6 +0,0 @@ -package com.usatiuk.objects.iterators; - -@FunctionalInterface -public interface IterProdFn, V> { - CloseableKvIterator get(IteratorStart start, K key); -} diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/objects/iterators/MergingKvIterator.java b/dhfs-parent/objects/src/main/java/com/usatiuk/objects/iterators/MergingKvIterator.java index 49953199..66251e13 100644 --- a/dhfs-parent/objects/src/main/java/com/usatiuk/objects/iterators/MergingKvIterator.java +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/objects/iterators/MergingKvIterator.java @@ -11,17 +11,15 @@ import java.util.TreeMap; public class MergingKvIterator, V> extends ReversibleKvIterator { private final NavigableMap> _sortedIterators = new TreeMap<>(); - private final String _name; private final List> _iterators; - public MergingKvIterator(String name, IteratorStart startType, K startKey, List> iterators) { - _goingForward = true; - _name = name; - // Why streams are so slow? + public MergingKvIterator(IteratorStart startType, K startKey, List> iterators) { + _goingForward = true; + { IteratorEntry[] iteratorEntries = new IteratorEntry[iterators.size()]; for (int i = 0; i < iterators.size(); i++) { - iteratorEntries[i] = new IteratorEntry<>(i, iterators.get(i).get(startType, startKey)); + iteratorEntries[i] = new IteratorEntry<>(i, iterators.get(i)); } _iterators = List.of(iteratorEntries); } @@ -91,8 +89,8 @@ public class MergingKvIterator, V> extends ReversibleKvI } @SafeVarargs - public MergingKvIterator(String name, IteratorStart startType, K startKey, IterProdFn... iterators) { - this(name, startType, startKey, List.of(iterators)); + public MergingKvIterator(IteratorStart startType, K startKey, CloseableKvIterator... iterators) { + this(startType, startKey, List.of(iterators)); } private void advanceIterator(IteratorEntry iteratorEntry) { @@ -151,7 +149,6 @@ public class MergingKvIterator, V> extends ReversibleKvI || (!_goingForward && peekImpl().compareTo(cur.getKey()) >= 0))) { skipImpl(); } - Log.tracev("{0} Reversed to {1}", _name, _sortedIterators); } @Override @@ -199,28 +196,14 @@ public class MergingKvIterator, V> extends ReversibleKvI @Override public String toString() { return "MergingKvIterator{" + - "_name='" + _name + '\'' + ", _sortedIterators=" + _sortedIterators.keySet() + ", _iterators=" + _iterators + '}'; } - private interface FirstMatchState, V> { - } - private record IteratorEntry, V>(int priority, CloseableKvIterator iterator) { public IteratorEntry reversed() { return new IteratorEntry<>(priority, iterator.reversed()); } } - - private record FirstMatchNone, V>() implements FirstMatchState { - } - - private record FirstMatchFound, V>( - CloseableKvIterator iterator) implements FirstMatchState { - } - - private record FirstMatchConsumed, V>() implements FirstMatchState { - } } diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/objects/iterators/TombstoneMergingKvIterator.java b/dhfs-parent/objects/src/main/java/com/usatiuk/objects/iterators/TombstoneMergingKvIterator.java deleted file mode 100644 index 097b23e8..00000000 --- a/dhfs-parent/objects/src/main/java/com/usatiuk/objects/iterators/TombstoneMergingKvIterator.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.usatiuk.objects.iterators; - -import java.util.List; - -public abstract class TombstoneMergingKvIterator { - public static , V> CloseableKvIterator of(String name, IteratorStart startType, K startKey, List>> iterators) { - return new PredicateKvIterator, V>( - new MergingKvIterator>(name + "-merging", startType, startKey, iterators), - startType, startKey, - pair -> { -// Log.tracev("{0} - Processing pair {1}", name, pair); - if (pair instanceof Tombstone) { - return null; - } - return ((Data) pair).value(); - }); - } - - public static , V> CloseableKvIterator of(String name, IteratorStart startType, K startKey, IterProdFn>... iterators) { - return of(name, startType, startKey, List.of(iterators)); - } -} diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/objects/iterators/PredicateKvIterator.java b/dhfs-parent/objects/src/main/java/com/usatiuk/objects/iterators/TombstoneSkippingIterator.java similarity index 61% rename from dhfs-parent/objects/src/main/java/com/usatiuk/objects/iterators/PredicateKvIterator.java rename to dhfs-parent/objects/src/main/java/com/usatiuk/objects/iterators/TombstoneSkippingIterator.java index d042ed1e..50ef7ecc 100644 --- a/dhfs-parent/objects/src/main/java/com/usatiuk/objects/iterators/PredicateKvIterator.java +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/objects/iterators/TombstoneSkippingIterator.java @@ -2,26 +2,26 @@ package com.usatiuk.objects.iterators; import org.apache.commons.lang3.tuple.Pair; +import java.util.List; import java.util.NoSuchElementException; import java.util.function.Function; -public class PredicateKvIterator, V, V_T> extends ReversibleKvIterator { - private final CloseableKvIterator _backing; - private final Function _transformer; - private Pair _next = null; +public class TombstoneSkippingIterator, V> extends ReversibleKvIterator { + private final MergingKvIterator> _backing; + private Pair _next = null; private boolean _checkedNext = false; - public PredicateKvIterator(CloseableKvIterator backing, IteratorStart start, K startKey, Function transformer) { + public TombstoneSkippingIterator(IteratorStart start, K startKey, List>> iterators) { _goingForward = true; - _backing = backing; - _transformer = transformer; + _backing = new MergingKvIterator<>(start, startKey, iterators); if (start == IteratorStart.GE || start == IteratorStart.GT) return; - fillNext(); - boolean shouldGoBack = false; + if (canHaveNext()) + tryFillNext(); + if (start == IteratorStart.LE) { if (_next == null || _next.getKey().compareTo(startKey) > 0) { shouldGoBack = true; @@ -38,34 +38,27 @@ public class PredicateKvIterator, V, V_T> extends Revers _backing.skipPrev(); fillNext(); _goingForward = true; - _backing.skip(); + if (_next != null) + _backing.skip(); fillNext(); } + } + private boolean canHaveNext() { + return (_goingForward ? _backing.hasNext() : _backing.hasPrev()); + } -// switch (start) { -// case LT -> { -//// assert _next == null || _next.getKey().compareTo(startKey) < 0; -// } -// case LE -> { -//// assert _next == null || _next.getKey().compareTo(startKey) <= 0; -// } -// case GT -> { -// assert _next == null || _next.getKey().compareTo(startKey) > 0; -// } -// case GE -> { -// assert _next == null || _next.getKey().compareTo(startKey) >= 0; -// } -// } + private boolean tryFillNext() { + var next = _goingForward ? _backing.next() : _backing.prev(); + if (next.getValue() instanceof Tombstone) + return false; + _next = Pair.of(next.getKey(), ((Data) next.getValue()).value()); + return true; } private void fillNext() { - while ((_goingForward ? _backing.hasNext() : _backing.hasPrev()) && _next == null) { - var next = _goingForward ? _backing.next() : _backing.prev(); - var transformed = _transformer.apply(next.getValue()); - if (transformed == null) - continue; - _next = Pair.of(next.getKey(), transformed); + while (_next == null && canHaveNext()) { + tryFillNext(); } _checkedNext = true; } @@ -80,9 +73,6 @@ public class PredicateKvIterator, V, V_T> extends Revers else if (!_goingForward && !wasAtEnd) _backing.skipPrev(); -// if (!wasAtEnd) -// Log.tracev("Skipped in reverse: {0}", _next); - _next = null; _checkedNext = false; } @@ -117,7 +107,7 @@ public class PredicateKvIterator, V, V_T> extends Revers } @Override - protected Pair nextImpl() { + protected Pair nextImpl() { if (!_checkedNext) fillNext(); @@ -137,7 +127,6 @@ public class PredicateKvIterator, V, V_T> extends Revers @Override public String toString() { return "PredicateKvIterator{" + - "_backing=" + _backing + ", _next=" + _next + '}'; } diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/objects/snapshot/Snapshot.java b/dhfs-parent/objects/src/main/java/com/usatiuk/objects/snapshot/Snapshot.java index 6b1a6bdd..f553fab2 100644 --- a/dhfs-parent/objects/src/main/java/com/usatiuk/objects/snapshot/Snapshot.java +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/objects/snapshot/Snapshot.java @@ -2,13 +2,17 @@ package com.usatiuk.objects.snapshot; import com.usatiuk.objects.iterators.CloseableKvIterator; import com.usatiuk.objects.iterators.IteratorStart; +import com.usatiuk.objects.iterators.MaybeTombstone; +import com.usatiuk.objects.iterators.Tombstone; import com.usatiuk.utils.AutoCloseableNoThrow; import javax.annotation.Nonnull; +import java.util.List; import java.util.Optional; +import java.util.stream.Stream; public interface Snapshot, V> extends AutoCloseableNoThrow { - CloseableKvIterator getIterator(IteratorStart start, K key); + List>> getIterator(IteratorStart start, K key); @Nonnull Optional readObject(K name); diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/objects/stores/CachingObjectPersistentStore.java b/dhfs-parent/objects/src/main/java/com/usatiuk/objects/stores/CachingObjectPersistentStore.java index a04c70e8..85ad5e32 100644 --- a/dhfs-parent/objects/src/main/java/com/usatiuk/objects/stores/CachingObjectPersistentStore.java +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/objects/stores/CachingObjectPersistentStore.java @@ -5,6 +5,7 @@ import com.usatiuk.objects.JDataVersionedWrapperLazy; import com.usatiuk.objects.JObjectKey; import com.usatiuk.objects.iterators.*; import com.usatiuk.objects.snapshot.Snapshot; +import com.usatiuk.utils.ListUtils; import io.quarkus.logging.Log; import io.quarkus.runtime.StartupEvent; import jakarta.annotation.Priority; @@ -16,6 +17,7 @@ import org.eclipse.microprofile.config.inject.ConfigProperty; import org.pcollections.TreePMap; import javax.annotation.Nonnull; +import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -33,6 +35,7 @@ public class CachingObjectPersistentStore { private ExecutorService _statusExecutor; private AtomicLong _cached = new AtomicLong(); private AtomicLong _cacheTries = new AtomicLong(); + public CachingObjectPersistentStore(@ConfigProperty(name = "dhfs.objects.lru.limit") int sizeLimit) { _cache = new AtomicReference<>( new Cache(TreePMap.empty(), 0, -1, sizeLimit) @@ -150,10 +153,12 @@ public class CachingObjectPersistentStore { } @Override - public CloseableKvIterator getIterator(IteratorStart start, JObjectKey key) { - return TombstoneMergingKvIterator.of("cache", start, key, - (mS, mK) -> new NavigableMapKvIterator>(_curCache.map(), mS, mK), - (mS, mK) -> new CachingKvIterator(_backing.getIterator(start, key))); + public List>> getIterator(IteratorStart start, JObjectKey key) { + return ListUtils.prependAndMap( + new NavigableMapKvIterator>(_curCache.map(), start, key), + _backing.getIterator(start, key), + i -> new CachingKvIterator((CloseableKvIterator) (CloseableKvIterator) i) + ); } @Nonnull diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/objects/stores/LmdbObjectPersistentStore.java b/dhfs-parent/objects/src/main/java/com/usatiuk/objects/stores/LmdbObjectPersistentStore.java index c922360b..69e389a8 100644 --- a/dhfs-parent/objects/src/main/java/com/usatiuk/objects/stores/LmdbObjectPersistentStore.java +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/objects/stores/LmdbObjectPersistentStore.java @@ -3,10 +3,7 @@ package com.usatiuk.objects.stores; import com.usatiuk.objects.JObjectKey; import com.usatiuk.objects.JObjectKeyMax; import com.usatiuk.objects.JObjectKeyMin; -import com.usatiuk.objects.iterators.CloseableKvIterator; -import com.usatiuk.objects.iterators.IteratorStart; -import com.usatiuk.objects.iterators.KeyPredicateKvIterator; -import com.usatiuk.objects.iterators.ReversibleKvIterator; +import com.usatiuk.objects.iterators.*; import com.usatiuk.objects.snapshot.Snapshot; import io.quarkus.arc.properties.IfBuildProperty; import io.quarkus.logging.Log; @@ -26,8 +23,10 @@ import java.lang.ref.Cleaner; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.nio.file.Path; +import java.util.List; import java.util.NoSuchElementException; import java.util.Optional; +import java.util.stream.Stream; import static org.lmdbjava.DbiFlags.MDB_CREATE; import static org.lmdbjava.Env.create; @@ -112,9 +111,9 @@ public class LmdbObjectPersistentStore implements ObjectPersistentStore { private boolean _closed = false; @Override - public CloseableKvIterator getIterator(IteratorStart start, JObjectKey key) { + public List>> getIterator(IteratorStart start, JObjectKey key) { assert !_closed; - return new KeyPredicateKvIterator<>(new LmdbKvIterator(_txn, start, key), start, key, (k) -> !k.value().equals(DB_VER_OBJ_NAME_STR)); + return List.of(new KeyPredicateKvIterator<>(new LmdbKvIterator(_txn, start, key), start, key, (k) -> !k.value().equals(DB_VER_OBJ_NAME_STR))); } @Nonnull @@ -195,7 +194,7 @@ public class LmdbObjectPersistentStore implements ObjectPersistentStore { return _root.toFile().getUsableSpace(); } - private class LmdbKvIterator extends ReversibleKvIterator { + private class LmdbKvIterator extends ReversibleKvIterator> { private static final Cleaner CLEANER = Cleaner.create(); private final Txn _txn; // Managed by the snapshot private final Cursor _cursor; @@ -350,13 +349,13 @@ public class LmdbObjectPersistentStore implements ObjectPersistentStore { } @Override - protected Pair nextImpl() { + protected Pair> nextImpl() { if (!_hasNext) { throw new NoSuchElementException("No more elements"); } // TODO: Right now with java serialization it doesn't matter, it's all copied to arrays anyway var val = _cursor.val(); - var ret = Pair.of(JObjectKey.fromByteBuffer(_cursor.key()), val.asReadOnlyBuffer()); + Pair> ret = Pair.of(JObjectKey.fromByteBuffer(_cursor.key()), new DataWrapper<>(val.asReadOnlyBuffer())); if (_goingForward) _hasNext = _cursor.next(); else diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/objects/stores/MemoryObjectPersistentStore.java b/dhfs-parent/objects/src/main/java/com/usatiuk/objects/stores/MemoryObjectPersistentStore.java index 79765b4f..6f39a770 100644 --- a/dhfs-parent/objects/src/main/java/com/usatiuk/objects/stores/MemoryObjectPersistentStore.java +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/objects/stores/MemoryObjectPersistentStore.java @@ -2,10 +2,7 @@ package com.usatiuk.objects.stores; import com.google.protobuf.ByteString; import com.usatiuk.objects.JObjectKey; -import com.usatiuk.objects.iterators.CloseableKvIterator; -import com.usatiuk.objects.iterators.IteratorStart; -import com.usatiuk.objects.iterators.MappingKvIterator; -import com.usatiuk.objects.iterators.NavigableMapKvIterator; +import com.usatiuk.objects.iterators.*; import com.usatiuk.objects.snapshot.Snapshot; import io.quarkus.arc.properties.IfBuildProperty; import jakarta.enterprise.context.ApplicationScoped; @@ -13,8 +10,10 @@ import org.pcollections.TreePMap; import javax.annotation.Nonnull; import java.nio.ByteBuffer; +import java.util.List; import java.util.Optional; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Stream; @ApplicationScoped @IfBuildProperty(name = "dhfs.objects.persistence", stringValue = "memory") @@ -31,8 +30,8 @@ public class MemoryObjectPersistentStore implements ObjectPersistentStore { private final long _lastCommitId = MemoryObjectPersistentStore.this._lastCommitId; @Override - public CloseableKvIterator getIterator(IteratorStart start, JObjectKey key) { - return new MappingKvIterator<>(new NavigableMapKvIterator<>(_objects, start, key), ByteString::asReadOnlyByteBuffer); + public List>> getIterator(IteratorStart start, JObjectKey key) { + return List.of(new MappingKvIterator<>(new NavigableMapKvIterator<>(_objects, start, key), s -> new DataWrapper<>(s.asReadOnlyByteBuffer()))); } @Nonnull diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/objects/stores/SerializingObjectPersistentStore.java b/dhfs-parent/objects/src/main/java/com/usatiuk/objects/stores/SerializingObjectPersistentStore.java index 4af76bde..cbb6637c 100644 --- a/dhfs-parent/objects/src/main/java/com/usatiuk/objects/stores/SerializingObjectPersistentStore.java +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/objects/stores/SerializingObjectPersistentStore.java @@ -1,21 +1,20 @@ package com.usatiuk.objects.stores; -import com.google.protobuf.ByteString; import com.usatiuk.objects.JDataVersionedWrapper; import com.usatiuk.objects.JDataVersionedWrapperSerializer; import com.usatiuk.objects.JObjectKey; -import com.usatiuk.objects.ObjectSerializer; -import com.usatiuk.objects.iterators.CloseableKvIterator; -import com.usatiuk.objects.iterators.IteratorStart; -import com.usatiuk.objects.iterators.MappingKvIterator; +import com.usatiuk.objects.iterators.*; import com.usatiuk.objects.snapshot.Snapshot; +import com.usatiuk.utils.ListUtils; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import org.apache.commons.lang3.tuple.Pair; import javax.annotation.Nonnull; import java.nio.ByteBuffer; +import java.util.List; import java.util.Optional; +import java.util.stream.Stream; @ApplicationScoped public class SerializingObjectPersistentStore { @@ -30,8 +29,10 @@ public class SerializingObjectPersistentStore { private final Snapshot _backing = delegateStore.getSnapshot(); @Override - public CloseableKvIterator getIterator(IteratorStart start, JObjectKey key) { - return new MappingKvIterator<>(_backing.getIterator(start, key), d -> serializer.deserialize(d)); + public List>> getIterator(IteratorStart start, JObjectKey key) { + return ListUtils.map(_backing.getIterator(start, key), + i -> new MappingKvIterator, MaybeTombstone>(i, + d -> serializer.deserialize(((DataWrapper) d).value()))); } @Nonnull diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/objects/stores/WritebackObjectPersistentStore.java b/dhfs-parent/objects/src/main/java/com/usatiuk/objects/stores/WritebackObjectPersistentStore.java index d02816f2..d41599b3 100644 --- a/dhfs-parent/objects/src/main/java/com/usatiuk/objects/stores/WritebackObjectPersistentStore.java +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/objects/stores/WritebackObjectPersistentStore.java @@ -7,6 +7,7 @@ import com.usatiuk.objects.iterators.*; import com.usatiuk.objects.snapshot.Snapshot; import com.usatiuk.objects.transaction.TxCommitException; import com.usatiuk.objects.transaction.TxRecord; +import com.usatiuk.utils.ListUtils; import io.quarkus.logging.Log; import io.quarkus.runtime.ShutdownEvent; import io.quarkus.runtime.StartupEvent; @@ -316,10 +317,8 @@ public class WritebackObjectPersistentStore { private final long txId = finalPw.lastCommittedId(); @Override - public CloseableKvIterator getIterator(IteratorStart start, JObjectKey key) { - return TombstoneMergingKvIterator.of("writeback-ps", start, key, - (tS, tK) -> new NavigableMapKvIterator<>(_pendingWrites, tS, tK), - (tS, tK) -> (CloseableKvIterator>) (CloseableKvIterator) _cache.getIterator(tS, tK)); + public List>> getIterator(IteratorStart start, JObjectKey key) { + return ListUtils.prepend(new NavigableMapKvIterator<>(_pendingWrites, start, key), _cache.getIterator(start, key)); } @Nonnull diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/objects/transaction/TransactionFactoryImpl.java b/dhfs-parent/objects/src/main/java/com/usatiuk/objects/transaction/TransactionFactoryImpl.java index d13f35ac..95594934 100644 --- a/dhfs-parent/objects/src/main/java/com/usatiuk/objects/transaction/TransactionFactoryImpl.java +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/objects/transaction/TransactionFactoryImpl.java @@ -6,6 +6,7 @@ import com.usatiuk.objects.JObjectKey; import com.usatiuk.objects.iterators.*; import com.usatiuk.objects.snapshot.Snapshot; import com.usatiuk.objects.stores.WritebackObjectPersistentStore; +import com.usatiuk.utils.ListUtils; import io.quarkus.logging.Log; import jakarta.inject.Inject; import jakarta.inject.Singleton; @@ -162,16 +163,24 @@ public class TransactionFactoryImpl implements TransactionFactory { @Override public CloseableKvIterator getIterator(IteratorStart start, JObjectKey key) { Log.tracev("Getting tx iterator with start={0}, key={1}", start, key); - return new ReadTrackingIterator(TombstoneMergingKvIterator.of("tx", start, key, - (tS, tK) -> new MappingKvIterator<>(new NavigableMapKvIterator<>(_writes, tS, tK), - t -> switch (t) { - case TxRecord.TxObjectRecordWrite write -> - new DataWrapper<>(new ReadTrackingInternalCrapTx(write.data())); - case TxRecord.TxObjectRecordDeleted deleted -> new TombstoneImpl<>(); - case null, default -> null; - }), - (tS, tK) -> new MappingKvIterator<>(_snapshot.getIterator(tS, tK), - d -> new DataWrapper(new ReadTrackingInternalCrapSource(d))))); + return new ReadTrackingIterator(new TombstoneSkippingIterator(start, key, + ListUtils.prependAndMap( + new MappingKvIterator<>(new NavigableMapKvIterator<>(_writes, start, key), + t -> switch (t) { + case TxRecord.TxObjectRecordWrite write -> + new DataWrapper(new ReadTrackingInternalCrapTx(write.data())); + case TxRecord.TxObjectRecordDeleted deleted -> + new TombstoneImpl(); + case null, default -> null; + }), + _snapshot.getIterator(start, key), + itin -> new MappingKvIterator, MaybeTombstone>(itin, + d -> switch (d) { + case Data w -> + new DataWrapper<>(new ReadTrackingInternalCrapSource(w.value())); + case Tombstone t -> new TombstoneImpl<>(); + case null, default -> null; + })))); } @Override diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/objects/transaction/TransactionManagerImpl.java b/dhfs-parent/objects/src/main/java/com/usatiuk/objects/transaction/TransactionManagerImpl.java index 7aad9dd3..baca7eb6 100644 --- a/dhfs-parent/objects/src/main/java/com/usatiuk/objects/transaction/TransactionManagerImpl.java +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/objects/transaction/TransactionManagerImpl.java @@ -6,6 +6,7 @@ import jakarta.inject.Singleton; import org.apache.commons.lang3.tuple.Pair; import java.util.Collection; +import java.util.concurrent.ExecutorService; @Singleton public class TransactionManagerImpl implements TransactionManager { diff --git a/dhfs-parent/objects/src/test/java/com/usatiuk/objects/iterators/MergingKvIteratorPbtTest.java b/dhfs-parent/objects/src/test/java/com/usatiuk/objects/iterators/MergingKvIteratorPbtTest.java index 3a66148f..36c7a9be 100644 --- a/dhfs-parent/objects/src/test/java/com/usatiuk/objects/iterators/MergingKvIteratorPbtTest.java +++ b/dhfs-parent/objects/src/test/java/com/usatiuk/objects/iterators/MergingKvIteratorPbtTest.java @@ -1,5 +1,6 @@ package com.usatiuk.objects.iterators; +import jnr.ffi.annotations.In; import net.jqwik.api.*; import net.jqwik.api.state.Action; import net.jqwik.api.state.ActionChain; @@ -9,6 +10,7 @@ import org.junit.jupiter.api.Assertions; import java.util.List; import java.util.Map; import java.util.TreeMap; +import java.util.function.BiConsumer; public class MergingKvIteratorPbtTest { @Property @@ -54,8 +56,8 @@ public class MergingKvIteratorPbtTest { } } mergedIterator = new NavigableMapKvIterator<>(perfectMerged, startType, startKey); - mergingIterator = new MergingKvIterator<>("test", startType, startKey, pairs.stream().>map( - list -> (IteratorStart start, Integer key) -> new NavigableMapKvIterator<>(new TreeMap(Map.ofEntries(list.toArray(Map.Entry[]::new))), start, key) + mergingIterator = new MergingKvIterator<>(startType, startKey, pairs.stream().>map( + list -> new NavigableMapKvIterator<>(new TreeMap(Map.ofEntries(list.toArray(Map.Entry[]::new))), startType, startKey) ).toList()); } diff --git a/dhfs-parent/objects/src/test/java/com/usatiuk/objects/iterators/MergingKvIteratorTest.java b/dhfs-parent/objects/src/test/java/com/usatiuk/objects/iterators/MergingKvIteratorTest.java deleted file mode 100644 index 7e8127b0..00000000 --- a/dhfs-parent/objects/src/test/java/com/usatiuk/objects/iterators/MergingKvIteratorTest.java +++ /dev/null @@ -1,348 +0,0 @@ -package com.usatiuk.objects.iterators; - -import com.usatiuk.objects.Just; -import org.apache.commons.lang3.tuple.Pair; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.pcollections.TreePMap; - -import java.util.Iterator; -import java.util.List; -import java.util.NoSuchElementException; - -public class MergingKvIteratorTest { - - @Test - public void testTestIterator() { - var list = List.of(Pair.of(1, 2), Pair.of(3, 4), Pair.of(5, 6)); - var iterator = new SimpleIteratorWrapper<>(list.iterator()); - var realIterator = list.iterator(); - while (realIterator.hasNext()) { - Assertions.assertTrue(iterator.hasNext()); - Assertions.assertEquals(realIterator.next(), iterator.next()); - } - Assertions.assertFalse(iterator.hasNext()); - - var emptyList = List.>of(); - var emptyIterator = new SimpleIteratorWrapper<>(emptyList.iterator()); - Assertions.assertFalse(emptyIterator.hasNext()); - } - - @Test - public void testSimple() { - var source1 = List.of(Pair.of(1, 2), Pair.of(3, 4), Pair.of(5, 6)).iterator(); - var source2 = List.of(Pair.of(2, 3), Pair.of(4, 5), Pair.of(6, 7)).iterator(); - var mergingIterator = new MergingKvIterator<>("test", IteratorStart.GE, 0, (a, b) -> new SimpleIteratorWrapper<>(source1), (a, b) -> new SimpleIteratorWrapper<>(source2)); - var expected = List.of(Pair.of(1, 2), Pair.of(2, 3), Pair.of(3, 4), Pair.of(4, 5), Pair.of(5, 6), Pair.of(6, 7)); - for (var pair : expected) { - Assertions.assertTrue(mergingIterator.hasNext()); - Assertions.assertEquals(pair, mergingIterator.next()); - } - } - - @Test - public void testPriority() { - var source1 = List.of(Pair.of(1, 2), Pair.of(2, 4), Pair.of(5, 6)); - var source2 = List.of(Pair.of(1, 3), Pair.of(2, 5), Pair.of(5, 7)); - var mergingIterator = new MergingKvIterator<>("test", IteratorStart.GE, 0, (a, b) -> new SimpleIteratorWrapper<>(source1.iterator()), (a, b) -> new SimpleIteratorWrapper<>(source2.iterator())); - var expected = List.of(Pair.of(1, 2), Pair.of(2, 4), Pair.of(5, 6)); - for (var pair : expected) { - Assertions.assertTrue(mergingIterator.hasNext()); - Assertions.assertEquals(pair, mergingIterator.next()); - } - Assertions.assertFalse(mergingIterator.hasNext()); - - var mergingIterator2 = new MergingKvIterator<>("test", IteratorStart.GE, 0, (a, b) -> new SimpleIteratorWrapper<>(source2.iterator()), (a, b) -> new SimpleIteratorWrapper<>(source1.iterator())); - var expected2 = List.of(Pair.of(1, 3), Pair.of(2, 5), Pair.of(5, 7)); - for (var pair : expected2) { - Assertions.assertTrue(mergingIterator2.hasNext()); - Assertions.assertEquals(pair, mergingIterator2.next()); - } - Assertions.assertFalse(mergingIterator2.hasNext()); - } - - @Test - public void testPriority2() { - var source1 = List.of(Pair.of(2, 4), Pair.of(5, 6)); - var source2 = List.of(Pair.of(1, 3), Pair.of(2, 5)); - var mergingIterator = new MergingKvIterator<>("test", IteratorStart.GE, 0, (a, b) -> new SimpleIteratorWrapper<>(source1.iterator()), (a, b) -> new SimpleIteratorWrapper<>(source2.iterator())); - var expected = List.of(Pair.of(1, 3), Pair.of(2, 4), Pair.of(5, 6)); - for (var pair : expected) { - Assertions.assertTrue(mergingIterator.hasNext()); - Assertions.assertEquals(pair, mergingIterator.next()); - } - Assertions.assertFalse(mergingIterator.hasNext()); - - var mergingIterator2 = new MergingKvIterator<>("test", IteratorStart.GE, 0, (a, b) -> new SimpleIteratorWrapper<>(source2.iterator()), (a, b) -> new SimpleIteratorWrapper<>(source1.iterator())); - var expected2 = List.of(Pair.of(1, 3), Pair.of(2, 5), Pair.of(5, 6)); - for (var pair : expected2) { - Assertions.assertTrue(mergingIterator2.hasNext()); - Assertions.assertEquals(pair, mergingIterator2.next()); - } - Assertions.assertFalse(mergingIterator2.hasNext()); - } - - @Test - public void testPriorityLe() { - var source1 = TreePMap.empty().plus(2, 4).plus(5, 6); - var source2 = TreePMap.empty().plus(1, 3).plus(2, 5); - var mergingIterator = new MergingKvIterator<>("test", IteratorStart.LE, 5, (mS, mK) -> new NavigableMapKvIterator<>(source1, mS, mK), (mS, mK) -> new NavigableMapKvIterator<>(source2, mS, mK)); - var expected = List.of(Pair.of(5, 6)); - for (var pair : expected) { - Assertions.assertTrue(mergingIterator.hasNext()); - Assertions.assertEquals(pair, mergingIterator.next()); - } - Assertions.assertFalse(mergingIterator.hasNext()); - Just.checkIterator(mergingIterator.reversed(), Pair.of(5, 6), Pair.of(2, 4), Pair.of(1, 3)); - Assertions.assertFalse(mergingIterator.reversed().hasNext()); - Just.checkIterator(mergingIterator, Pair.of(1, 3), Pair.of(2, 4), Pair.of(5, 6)); - Assertions.assertFalse(mergingIterator.hasNext()); - - - var mergingIterator2 = new MergingKvIterator<>("test", IteratorStart.LE, 5, (mS, mK) -> new NavigableMapKvIterator<>(source2, mS, mK), (mS, mK) -> new NavigableMapKvIterator<>(source1, mS, mK)); - var expected2 = List.of(Pair.of(5, 6)); - for (var pair : expected2) { - Assertions.assertTrue(mergingIterator2.hasNext()); - Assertions.assertEquals(pair, mergingIterator2.next()); - } - Assertions.assertFalse(mergingIterator2.hasNext()); - Just.checkIterator(mergingIterator2.reversed(), Pair.of(5, 6), Pair.of(2, 5), Pair.of(1, 3)); - Assertions.assertFalse(mergingIterator2.reversed().hasNext()); - Just.checkIterator(mergingIterator2, Pair.of(1, 3), Pair.of(2, 5), Pair.of(5, 6)); - Assertions.assertFalse(mergingIterator2.hasNext()); - - var mergingIterator3 = new MergingKvIterator<>("test", IteratorStart.LE, 5, (mS, mK) -> new NavigableMapKvIterator<>(source1, mS, mK), (mS, mK) -> new NavigableMapKvIterator<>(source2, mS, mK)); - Assertions.assertEquals(5, mergingIterator3.peekNextKey()); - Assertions.assertEquals(2, mergingIterator3.peekPrevKey()); - Assertions.assertEquals(5, mergingIterator3.peekNextKey()); - Assertions.assertEquals(2, mergingIterator3.peekPrevKey()); - } - - @Test - public void testPriorityLe2() { - var source1 = TreePMap.empty().plus(2, 4).plus(5, 6); - var source2 = TreePMap.empty().plus(1, 3).plus(2, 5).plus(3, 4); - var mergingIterator = new MergingKvIterator<>("test", IteratorStart.LE, 5, (mS, mK) -> new NavigableMapKvIterator<>(source1, mS, mK), (mS, mK) -> new NavigableMapKvIterator<>(source2, mS, mK)); - var expected = List.of(Pair.of(5, 6)); - for (var pair : expected) { - Assertions.assertTrue(mergingIterator.hasNext()); - Assertions.assertEquals(pair, mergingIterator.next()); - } - Assertions.assertFalse(mergingIterator.hasNext()); - } - - @Test - public void testPriorityLe3() { - var source1 = TreePMap.empty().plus(2, 4).plus(5, 6); - var source2 = TreePMap.empty().plus(1, 3).plus(2, 5).plus(6, 8); - var mergingIterator = new MergingKvIterator<>("test", IteratorStart.LE, 5, (mS, mK) -> new NavigableMapKvIterator<>(source1, mS, mK), (mS, mK) -> new NavigableMapKvIterator<>(source2, mS, mK)); - var expected = List.of(Pair.of(5, 6), Pair.of(6, 8)); - for (var pair : expected) { - Assertions.assertTrue(mergingIterator.hasNext()); - Assertions.assertEquals(pair, mergingIterator.next()); - } - Assertions.assertFalse(mergingIterator.hasNext()); - Just.checkIterator(mergingIterator.reversed(), Pair.of(6, 8), Pair.of(5, 6), Pair.of(2, 4), Pair.of(1, 3)); - Assertions.assertFalse(mergingIterator.reversed().hasNext()); - Just.checkIterator(mergingIterator, Pair.of(1, 3), Pair.of(2, 4), Pair.of(5, 6), Pair.of(6, 8)); - Assertions.assertFalse(mergingIterator.hasNext()); - - var mergingIterator2 = new MergingKvIterator<>("test", IteratorStart.LE, 5, (mS, mK) -> new NavigableMapKvIterator<>(source2, mS, mK), (mS, mK) -> new NavigableMapKvIterator<>(source1, mS, mK)); - var expected2 = List.of(Pair.of(5, 6), Pair.of(6, 8)); - for (var pair : expected2) { - Assertions.assertTrue(mergingIterator2.hasNext()); - Assertions.assertEquals(pair, mergingIterator2.next()); - } - Assertions.assertFalse(mergingIterator2.hasNext()); - - var mergingIterator3 = new MergingKvIterator<>("test", IteratorStart.LE, 5, (mS, mK) -> new NavigableMapKvIterator<>(source1, mS, mK), (mS, mK) -> new NavigableMapKvIterator<>(source2, mS, mK)); - Assertions.assertEquals(5, mergingIterator3.peekNextKey()); - Assertions.assertEquals(2, mergingIterator3.peekPrevKey()); - Assertions.assertEquals(5, mergingIterator3.peekNextKey()); - Assertions.assertEquals(2, mergingIterator3.peekPrevKey()); - Assertions.assertTrue(mergingIterator3.hasPrev()); - Assertions.assertTrue(mergingIterator3.hasNext()); - Assertions.assertEquals(5, mergingIterator3.peekNextKey()); - } - - @Test - public void testPriorityLe4() { - var source1 = TreePMap.empty().plus(6, 7); - var source2 = TreePMap.empty().plus(1, 3).plus(2, 5).plus(3, 4); - var mergingIterator = new MergingKvIterator<>("test", IteratorStart.LE, 5, (mS, mK) -> new NavigableMapKvIterator<>(source1, mS, mK), (mS, mK) -> new NavigableMapKvIterator<>(source2, mS, mK)); - var expected = List.of(Pair.of(3, 4), Pair.of(6, 7)); - for (var pair : expected) { - Assertions.assertTrue(mergingIterator.hasNext()); - Assertions.assertEquals(pair, mergingIterator.next()); - } - Assertions.assertFalse(mergingIterator.hasNext()); - - var mergingIterator2 = new MergingKvIterator<>("test", IteratorStart.LE, 5, (mS, mK) -> new NavigableMapKvIterator<>(source2, mS, mK), (mS, mK) -> new NavigableMapKvIterator<>(source1, mS, mK)); - var expected2 = List.of(Pair.of(3, 4), Pair.of(6, 7)); - for (var pair : expected2) { - Assertions.assertTrue(mergingIterator2.hasNext()); - Assertions.assertEquals(pair, mergingIterator2.next()); - } - Assertions.assertFalse(mergingIterator2.hasNext()); - } - - @Test - public void testPriorityLe5() { - var source1 = TreePMap.empty().plus(1, 2).plus(6, 7); - var source2 = TreePMap.empty().plus(1, 3).plus(2, 5).plus(3, 4); - var mergingIterator = new MergingKvIterator<>("test", IteratorStart.LE, 5, (mS, mK) -> new NavigableMapKvIterator<>(source1, mS, mK), (mS, mK) -> new NavigableMapKvIterator<>(source2, mS, mK)); - var expected = List.of(Pair.of(3, 4), Pair.of(6, 7)); - for (var pair : expected) { - Assertions.assertTrue(mergingIterator.hasNext()); - Assertions.assertEquals(pair, mergingIterator.next()); - } - Assertions.assertFalse(mergingIterator.hasNext()); - - var mergingIterator2 = new MergingKvIterator<>("test", IteratorStart.LE, 5, (mS, mK) -> new NavigableMapKvIterator<>(source2, mS, mK), (mS, mK) -> new NavigableMapKvIterator<>(source1, mS, mK)); - var expected2 = List.of(Pair.of(3, 4), Pair.of(6, 7)); - for (var pair : expected2) { - Assertions.assertTrue(mergingIterator2.hasNext()); - Assertions.assertEquals(pair, mergingIterator2.next()); - } - Assertions.assertFalse(mergingIterator2.hasNext()); - } - - @Test - public void testPriorityLe6() { - var source1 = TreePMap.empty().plus(1, 3).plus(2, 5).plus(3, 4); - var source2 = TreePMap.empty().plus(4, 6); - var mergingIterator = new MergingKvIterator<>("test", IteratorStart.LE, 5, (mS, mK) -> new NavigableMapKvIterator<>(source1, mS, mK), (mS, mK) -> new NavigableMapKvIterator<>(source2, mS, mK)); - var expected = List.of(Pair.of(4, 6)); - for (var pair : expected) { - Assertions.assertTrue(mergingIterator.hasNext()); - Assertions.assertEquals(pair, mergingIterator.next()); - } - Assertions.assertFalse(mergingIterator.hasNext()); - - var mergingIterator2 = new MergingKvIterator<>("test", IteratorStart.LE, 5, (mS, mK) -> new NavigableMapKvIterator<>(source2, mS, mK), (mS, mK) -> new NavigableMapKvIterator<>(source1, mS, mK)); - var expected2 = List.of(Pair.of(4, 6)); - for (var pair : expected2) { - Assertions.assertTrue(mergingIterator2.hasNext()); - Assertions.assertEquals(pair, mergingIterator2.next()); - } - Assertions.assertFalse(mergingIterator2.hasNext()); - } - - @Test - public void testPriorityLe7() { - var source1 = TreePMap.empty().plus(1, 3).plus(3, 5).plus(4, 6); - var source2 = TreePMap.empty().plus(1, 4).plus(3, 5).plus(4, 6); - var mergingIterator = new MergingKvIterator<>("test", IteratorStart.LE, 2, (mS, mK) -> new NavigableMapKvIterator<>(source1, mS, mK), (mS, mK) -> new NavigableMapKvIterator<>(source2, mS, mK)); - var expected = List.of(Pair.of(1, 3), Pair.of(3, 5), Pair.of(4, 6)); - for (var pair : expected) { - Assertions.assertTrue(mergingIterator.hasNext()); - Assertions.assertEquals(pair, mergingIterator.next()); - } - Assertions.assertFalse(mergingIterator.hasNext()); - Just.checkIterator(mergingIterator.reversed(), Pair.of(4, 6), Pair.of(3, 5), Pair.of(1, 3)); - Just.checkIterator(mergingIterator, Pair.of(1, 3), Pair.of(3, 5), Pair.of(4, 6)); - - var mergingIterator2 = new MergingKvIterator<>("test", IteratorStart.LE, 2, (mS, mK) -> new NavigableMapKvIterator<>(source2, mS, mK), (mS, mK) -> new NavigableMapKvIterator<>(source1, mS, mK)); - var expected2 = List.of(Pair.of(1, 4), Pair.of(3, 5), Pair.of(4, 6)); - for (var pair : expected2) { - Assertions.assertTrue(mergingIterator2.hasNext()); - Assertions.assertEquals(pair, mergingIterator2.next()); - } - Assertions.assertFalse(mergingIterator2.hasNext()); - } - - @Test - public void testPriorityLt() { - var source1 = TreePMap.empty().plus(2, 4).plus(5, 6); - var source2 = TreePMap.empty().plus(1, 3).plus(2, 5); - var mergingIterator = new MergingKvIterator<>("test", IteratorStart.LT, 5, (mS, mK) -> new NavigableMapKvIterator<>(source1, mS, mK), (mS, mK) -> new NavigableMapKvIterator<>(source2, mS, mK)); - var expected = List.of(Pair.of(2, 4), Pair.of(5, 6)); - for (var pair : expected) { - Assertions.assertTrue(mergingIterator.hasNext()); - Assertions.assertEquals(pair, mergingIterator.next()); - } - Assertions.assertFalse(mergingIterator.hasNext()); - - var mergingIterator2 = new MergingKvIterator<>("test", IteratorStart.LT, 5, (mS, mK) -> new NavigableMapKvIterator<>(source2, mS, mK), (mS, mK) -> new NavigableMapKvIterator<>(source1, mS, mK)); - var expected2 = List.of(Pair.of(2, 5), Pair.of(5, 6)); - for (var pair : expected2) { - Assertions.assertTrue(mergingIterator2.hasNext()); - Assertions.assertEquals(pair, mergingIterator2.next()); - } - Assertions.assertFalse(mergingIterator2.hasNext()); - } - - private class SimpleIteratorWrapper, V> implements CloseableKvIterator { - private final Iterator> _iterator; - private Pair _next; - - public SimpleIteratorWrapper(Iterator> iterator) { - _iterator = iterator; - fillNext(); - } - - private void fillNext() { - while (_iterator.hasNext() && _next == null) { - _next = _iterator.next(); - } - } - - @Override - public K peekNextKey() { - if (_next == null) { - throw new NoSuchElementException(); - } - return _next.getKey(); - } - - @Override - public void skip() { - if (_next == null) { - throw new NoSuchElementException(); - } - _next = null; - fillNext(); - } - - @Override - public K peekPrevKey() { - throw new UnsupportedOperationException(); - } - - @Override - public Pair prev() { - throw new UnsupportedOperationException(); - } - - @Override - public boolean hasPrev() { - throw new UnsupportedOperationException(); - } - - @Override - public void skipPrev() { - throw new UnsupportedOperationException(); - - } - - @Override - public void close() { - } - - @Override - public boolean hasNext() { - return _next != null; - } - - @Override - public Pair next() { - if (_next == null) { - throw new NoSuchElementException("No more elements"); - } - var ret = _next; - _next = null; - fillNext(); - return ret; - } - } -} diff --git a/dhfs-parent/objects/src/test/java/com/usatiuk/objects/iterators/PredicateKvIteratorTest.java b/dhfs-parent/objects/src/test/java/com/usatiuk/objects/iterators/PredicateKvIteratorTest.java deleted file mode 100644 index 5cc3eb97..00000000 --- a/dhfs-parent/objects/src/test/java/com/usatiuk/objects/iterators/PredicateKvIteratorTest.java +++ /dev/null @@ -1,161 +0,0 @@ -package com.usatiuk.objects.iterators; - -import com.usatiuk.objects.Just; -import org.apache.commons.lang3.tuple.Pair; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.pcollections.TreePMap; - -import java.util.List; - -public class PredicateKvIteratorTest { - - @Test - public void simpleTest() { - var source1 = TreePMap.empty().plus(1, 3).plus(3, 5).plus(4, 6); - var pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.GT, 1), - IteratorStart.GE, 1, v -> (v % 2 == 0) ? v : null); - var expected = List.of(Pair.of(4, 6)); - for (var pair : expected) { - Assertions.assertTrue(pit.hasNext()); - Assertions.assertEquals(pair, pit.next()); - } - } - - @Test - public void ltTest() { - var source1 = TreePMap.empty().plus(1, 3).plus(3, 5).plus(4, 6); - var pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 4), - IteratorStart.LT, 4, v -> (v % 2 == 0) ? v : null); - var expected = List.of(Pair.of(4, 6)); - for (var pair : expected) { - Assertions.assertTrue(pit.hasNext()); - Assertions.assertEquals(pair, pit.next()); - } - Assertions.assertFalse(pit.hasNext()); - } - - @Test - public void ltTest2() { - var source1 = TreePMap.empty().plus(1, 3).plus(3, 5).plus(4, 6); - var pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 1), - IteratorStart.LT, 1, v -> (v % 2 == 0) ? v : null); - Just.checkIterator(pit, Pair.of(4, 6)); - Assertions.assertFalse(pit.hasNext()); - - pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 2), - IteratorStart.LT, 2, v -> (v % 2 == 0) ? v : null); - Just.checkIterator(pit, Pair.of(4, 6)); - Assertions.assertFalse(pit.hasNext()); - - pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 4), - IteratorStart.LT, 4, v -> (v % 2 == 0) ? v : null); - Just.checkIterator(pit, Pair.of(4, 6)); - Assertions.assertFalse(pit.hasNext()); - - pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LE, 4), - IteratorStart.LE, 4, v -> (v % 2 == 0) ? v : null); - Just.checkIterator(pit, Pair.of(4, 6)); - Assertions.assertFalse(pit.hasNext()); - } - - @Test - public void ltTest3() { - var source1 = TreePMap.empty().plus(1, 3).plus(3, 5).plus(4, 6).plus(5, 7).plus(6, 8); - var pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 4), - IteratorStart.LT, 4, v -> (v % 2 == 0) ? v : null); - Just.checkIterator(pit, Pair.of(4, 6), Pair.of(6, 8)); - Assertions.assertFalse(pit.hasNext()); - - pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 5), - IteratorStart.LT, 5, v -> (v % 2 == 0) ? v : null); - Just.checkIterator(pit, Pair.of(4, 6), Pair.of(6, 8)); - Assertions.assertFalse(pit.hasNext()); - - pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 6), - IteratorStart.LT, 6, v -> (v % 2 == 0) ? v : null); - Just.checkIterator(pit, Pair.of(4, 6), Pair.of(6, 8)); - Assertions.assertFalse(pit.hasNext()); - - pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 7), - IteratorStart.LT, 7, v -> (v % 2 == 0) ? v : null); - Just.checkIterator(pit, Pair.of(6, 8)); - Assertions.assertFalse(pit.hasNext()); - - pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 8), - IteratorStart.LT, 8, v -> (v % 2 == 0) ? v : null); - Just.checkIterator(pit, Pair.of(6, 8)); - Assertions.assertFalse(pit.hasNext()); - - pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LE, 6), - IteratorStart.LE, 6, v -> (v % 2 == 0) ? v : null); - Just.checkIterator(pit, Pair.of(6, 8)); - Assertions.assertFalse(pit.hasNext()); - - pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 6), - IteratorStart.LT, 6, v -> (v % 2 == 0) ? v : null); - Assertions.assertTrue(pit.hasNext()); - Assertions.assertEquals(4, pit.peekNextKey()); - Assertions.assertFalse(pit.hasPrev()); - Assertions.assertEquals(4, pit.peekNextKey()); - Assertions.assertFalse(pit.hasPrev()); - Assertions.assertEquals(Pair.of(4, 6), pit.next()); - Assertions.assertTrue(pit.hasNext()); - Assertions.assertEquals(6, pit.peekNextKey()); - Assertions.assertEquals(4, pit.peekPrevKey()); - Assertions.assertEquals(6, pit.peekNextKey()); - Assertions.assertEquals(4, pit.peekPrevKey()); - } - - @Test - public void itTest4() { - var source1 = TreePMap.empty().plus(1, 3).plus(3, 5).plus(4, 6).plus(5, 8).plus(6, 10); - var pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 4), - IteratorStart.LT, 4, v -> (v % 2 == 0) ? v : null); - Just.checkIterator(pit, Pair.of(4, 6), Pair.of(5, 8), Pair.of(6, 10)); - Assertions.assertFalse(pit.hasNext()); - - pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 5), - IteratorStart.LT, 5, v -> (v % 2 == 0) ? v : null); - Just.checkIterator(pit, Pair.of(4, 6), Pair.of(5, 8), Pair.of(6, 10)); - Assertions.assertFalse(pit.hasNext()); - - pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 6), - IteratorStart.LT, 6, v -> (v % 2 == 0) ? v : null); - Just.checkIterator(pit, Pair.of(5, 8), Pair.of(6, 10)); - Assertions.assertFalse(pit.hasNext()); - - pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 7), - IteratorStart.LT, 7, v -> (v % 2 == 0) ? v : null); - Just.checkIterator(pit, Pair.of(6, 10)); - Assertions.assertFalse(pit.hasNext()); - Assertions.assertTrue(pit.hasPrev()); - Assertions.assertEquals(6, pit.peekPrevKey()); - Assertions.assertEquals(Pair.of(6, 10), pit.prev()); - Assertions.assertTrue(pit.hasNext()); - Assertions.assertEquals(6, pit.peekNextKey()); - - pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 6), - IteratorStart.LT, 6, v -> (v % 2 == 0) ? v : null); - Assertions.assertTrue(pit.hasNext()); - Assertions.assertEquals(5, pit.peekNextKey()); - Assertions.assertTrue(pit.hasPrev()); - Assertions.assertEquals(4, pit.peekPrevKey()); - Assertions.assertEquals(5, pit.peekNextKey()); - Assertions.assertEquals(4, pit.peekPrevKey()); - Assertions.assertEquals(Pair.of(5, 8), pit.next()); - Assertions.assertTrue(pit.hasNext()); - Assertions.assertEquals(6, pit.peekNextKey()); - Assertions.assertEquals(5, pit.peekPrevKey()); - Assertions.assertEquals(6, pit.peekNextKey()); - Assertions.assertEquals(5, pit.peekPrevKey()); - } - -// @Test -// public void reverseTest() { -// var source1 = TreePMap.empty().plus(1, 3).plus(3, 5).plus(4, 6); -// var pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 4), -// IteratorStart.LT, 4, v -> (v % 2 == 0) ? v : null); -// -// } -} diff --git a/dhfs-parent/objects/src/test/java/com/usatiuk/objects/iterators/TombstoneSkippingIteratorPbtTest.java b/dhfs-parent/objects/src/test/java/com/usatiuk/objects/iterators/TombstoneSkippingIteratorPbtTest.java new file mode 100644 index 00000000..d8bfc13c --- /dev/null +++ b/dhfs-parent/objects/src/test/java/com/usatiuk/objects/iterators/TombstoneSkippingIteratorPbtTest.java @@ -0,0 +1,275 @@ +package com.usatiuk.objects.iterators; + +import net.jqwik.api.*; +import net.jqwik.api.state.Action; +import net.jqwik.api.state.ActionChain; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.Assertions; + +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +public class TombstoneSkippingIteratorPbtTest { + @Property + public void checkMergingIterator(@ForAll("actions") ActionChain actions) { + actions.run(); + } + + @Provide + Arbitrary> actions(@ForAll("lists") List>>> list, + @ForAll IteratorStart iteratorStart, @ForAll("startKey") Integer startKey) { + return ActionChain.startWith(() -> new MergingIteratorModel(list, iteratorStart, startKey)) + .withAction(new NextAction()) + .withAction(new PeekNextKeyAction()) + .withAction(new SkipAction()) + .withAction(new PeekPrevKeyAction()) + .withAction(new SkipPrevAction()) + .withAction(new PrevAction()) + .withAction(new HasNextAction()) + .withAction(new HasPrevAction()); + } + + @Provide + Arbitrary>>>> lists() { + return Arbitraries.entries(Arbitraries.integers().between(-50, 50), + Arbitraries.integers().between(-50, 50).flatMap(i -> Arbitraries.of(true, false).>flatMap( + b -> b ? Arbitraries.just(new DataWrapper(i)) : Arbitraries.just(new TombstoneImpl<>()) + )) + ) + .list().uniqueElements(Map.Entry::getKey).ofMinSize(0).ofMaxSize(20) + .list().ofMinSize(1).ofMaxSize(5); + } + + @Provide + Arbitrary startKey() { + return Arbitraries.integers().between(-51, 51); + } + + static class MergingIteratorModel implements CloseableKvIterator { + private final CloseableKvIterator mergedIterator; + private final CloseableKvIterator mergingIterator; + + private MergingIteratorModel(List>>> pairs, IteratorStart startType, Integer startKey) { + TreeMap> perfectMergedTombstones = new TreeMap<>(); + for (List>> list : pairs) { + for (Map.Entry> pair : list) { + perfectMergedTombstones.putIfAbsent(pair.getKey(), pair.getValue()); + } + } + TreeMap perfectMerged = new TreeMap<>(); + for (var e : perfectMergedTombstones.entrySet()) { + if (e.getValue() instanceof Data data) + perfectMerged.put(e.getKey(), data.value()); + } + + + mergedIterator = new NavigableMapKvIterator<>(perfectMerged, startType, startKey); + mergingIterator = new TombstoneSkippingIterator<>(startType, startKey, pairs.stream().>>map( + list -> new NavigableMapKvIterator>(new TreeMap>(Map.ofEntries(list.toArray(Map.Entry[]::new))), startType, startKey) + ).toList()); + } + + @Override + public Integer peekNextKey() { + var mergedKey = mergedIterator.peekNextKey(); + var mergingKey = mergingIterator.peekNextKey(); + Assertions.assertEquals(mergedKey, mergingKey); + return mergedKey; + } + + @Override + public void skip() { + mergedIterator.skip(); + mergingIterator.skip(); + } + + @Override + public Integer peekPrevKey() { + var mergedKey = mergedIterator.peekPrevKey(); + var mergingKey = mergingIterator.peekPrevKey(); + Assertions.assertEquals(mergedKey, mergingKey); + return mergedKey; + } + + @Override + public Pair prev() { + var mergedKey = mergedIterator.prev(); + var mergingKey = mergingIterator.prev(); + Assertions.assertEquals(mergedKey, mergingKey); + return mergedKey; + } + + @Override + public boolean hasPrev() { + var mergedKey = mergedIterator.hasPrev(); + var mergingKey = mergingIterator.hasPrev(); + Assertions.assertEquals(mergedKey, mergingKey); + return mergedKey; + } + + @Override + public void skipPrev() { + mergedIterator.skipPrev(); + mergingIterator.skipPrev(); + } + + @Override + public void close() { + mergedIterator.close(); + mergingIterator.close(); + } + + @Override + public boolean hasNext() { + var mergedKey = mergedIterator.hasNext(); + var mergingKey = mergingIterator.hasNext(); + Assertions.assertEquals(mergedKey, mergingKey); + return mergedKey; + } + + @Override + public Pair next() { + var mergedKey = mergedIterator.next(); + var mergingKey = mergingIterator.next(); + Assertions.assertEquals(mergedKey, mergingKey); + return mergedKey; + } + } + + static class PeekNextKeyAction extends Action.JustMutate { + @Override + public void mutate(MergingIteratorModel state) { + state.peekNextKey(); + } + + @Override + public boolean precondition(MergingIteratorModel state) { + return state.hasNext(); + } + + @Override + public String description() { + return "Peek next key"; + } + } + + static class SkipAction extends Action.JustMutate { + @Override + public void mutate(MergingIteratorModel state) { + state.skip(); + } + + @Override + public boolean precondition(MergingIteratorModel state) { + return state.hasNext(); + } + + @Override + public String description() { + return "Skip next key"; + } + } + + static class PeekPrevKeyAction extends Action.JustMutate { + @Override + public void mutate(MergingIteratorModel state) { + state.peekPrevKey(); + } + + @Override + public boolean precondition(MergingIteratorModel state) { + return state.hasPrev(); + } + + @Override + public String description() { + return "Peek prev key"; + } + } + + static class SkipPrevAction extends Action.JustMutate { + @Override + public void mutate(MergingIteratorModel state) { + state.skipPrev(); + } + + @Override + public boolean precondition(MergingIteratorModel state) { + return state.hasPrev(); + } + + @Override + public String description() { + return "Skip prev key"; + } + } + + static class PrevAction extends Action.JustMutate { + @Override + public void mutate(MergingIteratorModel state) { + state.prev(); + } + + @Override + public boolean precondition(MergingIteratorModel state) { + return state.hasPrev(); + } + + @Override + public String description() { + return "Prev key"; + } + } + + static class NextAction extends Action.JustMutate { + @Override + public void mutate(MergingIteratorModel state) { + state.next(); + } + + @Override + public boolean precondition(MergingIteratorModel state) { + return state.hasNext(); + } + + @Override + public String description() { + return "Next key"; + } + } + + static class HasNextAction extends Action.JustMutate { + @Override + public void mutate(MergingIteratorModel state) { + state.hasNext(); + } + + @Override + public boolean precondition(MergingIteratorModel state) { + return true; + } + + @Override + public String description() { + return "Has next key"; + } + } + + static class HasPrevAction extends Action.JustMutate { + @Override + public void mutate(MergingIteratorModel state) { + state.hasPrev(); + } + + @Override + public boolean precondition(MergingIteratorModel state) { + return true; + } + + @Override + public String description() { + return "Has prev key"; + } + } +} \ No newline at end of file diff --git a/dhfs-parent/objects/src/test/java/com/usatiuk/objects/stores/LmdbKvIteratorTest.java b/dhfs-parent/objects/src/test/java/com/usatiuk/objects/stores/LmdbKvIteratorTest.java deleted file mode 100644 index c6a113dc..00000000 --- a/dhfs-parent/objects/src/test/java/com/usatiuk/objects/stores/LmdbKvIteratorTest.java +++ /dev/null @@ -1,130 +0,0 @@ -package com.usatiuk.objects.stores; - - -import com.google.protobuf.ByteString; -import com.usatiuk.objects.JObjectKey; -import com.usatiuk.objects.Just; -import com.usatiuk.objects.TempDataProfile; -import com.usatiuk.objects.iterators.IteratorStart; -import io.quarkus.test.junit.QuarkusTest; -import io.quarkus.test.junit.TestProfile; -import jakarta.inject.Inject; -import org.apache.commons.lang3.tuple.Pair; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.RepeatedTest; - -import java.nio.ByteBuffer; -import java.util.List; - -class Profiles { - public static class LmdbKvIteratorTestProfile extends TempDataProfile { - } -} - -@QuarkusTest -@TestProfile(Profiles.LmdbKvIteratorTestProfile.class) -public class LmdbKvIteratorTest { - - @Inject - LmdbObjectPersistentStore store; - - long getNextTxId() { - try (var s = store.getSnapshot()) { - return s.id() + 1; - } - } - - @RepeatedTest(100) - public void iteratorTest1() { - store.prepareTx( - new TxManifestRaw( - List.of(Pair.of(JObjectKey.of(Long.toString(1)), ByteString.copyFrom(new byte[]{2})), - Pair.of(JObjectKey.of(Long.toString(2)), ByteString.copyFrom(new byte[]{3})), - Pair.of(JObjectKey.of(Long.toString(3)), ByteString.copyFrom(new byte[]{4}))), - List.of() - ), getNextTxId() - ).run(); - - try (var snapshot = store.getSnapshot()) { - var iterator = snapshot.getIterator(IteratorStart.GE, JObjectKey.of("")); - Just.checkIterator(iterator, List.of(Pair.of(JObjectKey.of(Long.toString(1)), ByteBuffer.wrap(new byte[]{2})), - Pair.of(JObjectKey.of(Long.toString(2)), ByteBuffer.wrap(new byte[]{3})), - Pair.of(JObjectKey.of(Long.toString(3)), ByteBuffer.wrap(new byte[]{4})))); - Assertions.assertFalse(iterator.hasNext()); - iterator.close(); - - iterator = snapshot.getIterator(IteratorStart.LE, JObjectKey.of(Long.toString(3))); - Just.checkIterator(iterator, Pair.of(JObjectKey.of(Long.toString(3)), ByteBuffer.wrap(new byte[]{4}))); - Assertions.assertFalse(iterator.hasNext()); - iterator.close(); - - iterator = snapshot.getIterator(IteratorStart.LE, JObjectKey.of(Long.toString(2))); - Just.checkIterator(iterator, Pair.of(JObjectKey.of(Long.toString(2)), ByteBuffer.wrap(new byte[]{3})), Pair.of(JObjectKey.of(Long.toString(3)), ByteBuffer.wrap(new byte[]{4}))); - Assertions.assertFalse(iterator.hasNext()); - iterator.close(); - - iterator = snapshot.getIterator(IteratorStart.GE, JObjectKey.of(Long.toString(2))); - Just.checkIterator(iterator, Pair.of(JObjectKey.of(Long.toString(2)), ByteBuffer.wrap(new byte[]{3})), Pair.of(JObjectKey.of(Long.toString(3)), ByteBuffer.wrap(new byte[]{4}))); - Assertions.assertFalse(iterator.hasNext()); - iterator.close(); - - iterator = snapshot.getIterator(IteratorStart.GT, JObjectKey.of(Long.toString(2))); - Just.checkIterator(iterator, Pair.of(JObjectKey.of(Long.toString(3)), ByteBuffer.wrap(new byte[]{4}))); - Assertions.assertFalse(iterator.hasNext()); - iterator.close(); - - iterator = snapshot.getIterator(IteratorStart.LT, JObjectKey.of(Long.toString(3))); - Just.checkIterator(iterator, Pair.of(JObjectKey.of(Long.toString(2)), ByteBuffer.wrap(new byte[]{3})), Pair.of(JObjectKey.of(Long.toString(3)), ByteBuffer.wrap(new byte[]{4}))); - Assertions.assertFalse(iterator.hasNext()); - iterator.close(); - - iterator = snapshot.getIterator(IteratorStart.LT, JObjectKey.of(Long.toString(2))); - Just.checkIterator(iterator, Pair.of(JObjectKey.of(Long.toString(1)), ByteBuffer.wrap(new byte[]{2})), Pair.of(JObjectKey.of(Long.toString(2)), ByteBuffer.wrap(new byte[]{3})), Pair.of(JObjectKey.of(Long.toString(3)), ByteBuffer.wrap(new byte[]{4}))); - Assertions.assertFalse(iterator.hasNext()); - iterator.close(); - - iterator = snapshot.getIterator(IteratorStart.LT, JObjectKey.of(Long.toString(1))); - Just.checkIterator(iterator, Pair.of(JObjectKey.of(Long.toString(1)), ByteBuffer.wrap(new byte[]{2})), Pair.of(JObjectKey.of(Long.toString(2)), ByteBuffer.wrap(new byte[]{3})), Pair.of(JObjectKey.of(Long.toString(3)), ByteBuffer.wrap(new byte[]{4}))); - Assertions.assertFalse(iterator.hasNext()); - iterator.close(); - - iterator = snapshot.getIterator(IteratorStart.LE, JObjectKey.of(Long.toString(1))); - Just.checkIterator(iterator, Pair.of(JObjectKey.of(Long.toString(1)), ByteBuffer.wrap(new byte[]{2})), Pair.of(JObjectKey.of(Long.toString(2)), ByteBuffer.wrap(new byte[]{3})), Pair.of(JObjectKey.of(Long.toString(3)), ByteBuffer.wrap(new byte[]{4}))); - Assertions.assertFalse(iterator.hasNext()); - iterator.close(); - - iterator = snapshot.getIterator(IteratorStart.GT, JObjectKey.of(Long.toString(3))); - Assertions.assertFalse(iterator.hasNext()); - iterator.close(); - - iterator = snapshot.getIterator(IteratorStart.GT, JObjectKey.of(Long.toString(4))); - Assertions.assertFalse(iterator.hasNext()); - iterator.close(); - - iterator = snapshot.getIterator(IteratorStart.LE, JObjectKey.of(Long.toString(0))); - Just.checkIterator(iterator, Pair.of(JObjectKey.of(Long.toString(1)), ByteBuffer.wrap(new byte[]{2})), Pair.of(JObjectKey.of(Long.toString(2)), ByteBuffer.wrap(new byte[]{3})), Pair.of(JObjectKey.of(Long.toString(3)), ByteBuffer.wrap(new byte[]{4}))); - Assertions.assertFalse(iterator.hasNext()); - iterator.close(); - - iterator = snapshot.getIterator(IteratorStart.GE, JObjectKey.of(Long.toString(2))); - Assertions.assertTrue(iterator.hasNext()); - Assertions.assertEquals(JObjectKey.of(Long.toString(2)), iterator.peekNextKey()); - Assertions.assertEquals(JObjectKey.of(Long.toString(1)), iterator.peekPrevKey()); - Assertions.assertEquals(JObjectKey.of(Long.toString(2)), iterator.peekNextKey()); - Assertions.assertEquals(JObjectKey.of(Long.toString(1)), iterator.peekPrevKey()); - Just.checkIterator(iterator.reversed(), Pair.of(JObjectKey.of(Long.toString(1)), ByteBuffer.wrap(new byte[]{2}))); - Just.checkIterator(iterator, Pair.of(JObjectKey.of(Long.toString(1)), ByteBuffer.wrap(new byte[]{2})), Pair.of(JObjectKey.of(Long.toString(2)), ByteBuffer.wrap(new byte[]{3})), Pair.of(JObjectKey.of(Long.toString(3)), ByteBuffer.wrap(new byte[]{4}))); - Assertions.assertEquals(Pair.of(JObjectKey.of(Long.toString(3)), ByteBuffer.wrap(new byte[]{4})), iterator.prev()); - Assertions.assertEquals(Pair.of(JObjectKey.of(Long.toString(2)), ByteBuffer.wrap(new byte[]{3})), iterator.prev()); - Assertions.assertEquals(Pair.of(JObjectKey.of(Long.toString(2)), ByteBuffer.wrap(new byte[]{3})), iterator.next()); - iterator.close(); - } - - store.prepareTx(new TxManifestRaw( - List.of(), - List.of(JObjectKey.of(Long.toString(1)), JObjectKey.of(Long.toString(2)), JObjectKey.of(Long.toString(3))) - ), - getNextTxId() - ).run(); - } -} diff --git a/dhfs-parent/utils/src/main/java/com/usatiuk/utils/ListUtils.java b/dhfs-parent/utils/src/main/java/com/usatiuk/utils/ListUtils.java new file mode 100644 index 00000000..c9b06caa --- /dev/null +++ b/dhfs-parent/utils/src/main/java/com/usatiuk/utils/ListUtils.java @@ -0,0 +1,33 @@ +package com.usatiuk.utils; + +import java.util.List; +import java.util.function.Function; + +public class ListUtils { + + public static List prependAndMap(T_V item, List suffix, Function suffixFn) { + T_V[] arr = (T_V[]) new Object[suffix.size() + 1]; + arr[0] = item; + for (int i = 0; i < suffix.size(); i++) { + arr[i + 1] = suffixFn.apply(suffix.get(i)); + } + return List.of(arr); + } + + public static List prepend(T item, List suffix) { + T[] arr = (T[]) new Object[suffix.size() + 1]; + arr[0] = item; + for (int i = 0; i < suffix.size(); i++) { + arr[i + 1] = suffix.get(i); + } + return List.of(arr); + } + + public static List map(List suffix, Function suffixFn) { + T_V[] arr = (T_V[]) new Object[suffix.size()]; + for (int i = 0; i < suffix.size(); i++) { + arr[i] = suffixFn.apply(suffix.get(i)); + } + return List.of(arr); + } +}