diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/CloseableKvIterator.java b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/CloseableKvIterator.java index 7014f8a2..0314b92b 100644 --- a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/CloseableKvIterator.java +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/CloseableKvIterator.java @@ -8,10 +8,14 @@ import java.util.Iterator; public interface CloseableKvIterator, V> extends Iterator>, AutoCloseableNoThrow { K peekNextKey(); + Class peekNextType(); + void skip(); K peekPrevKey(); + Class peekPrevType(); + Pair prev(); boolean hasPrev(); diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/KeyPredicateKvIterator.java b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/KeyPredicateKvIterator.java index b43308d2..1e2b0037 100644 --- a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/KeyPredicateKvIterator.java +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/KeyPredicateKvIterator.java @@ -114,6 +114,11 @@ public class KeyPredicateKvIterator, V> extends Reversib return got; } + @Override + protected Class peekTypeImpl() { + return _goingForward ? _backing.peekNextType() : _backing.peekPrevType(); + } + @Override public void close() { _backing.close(); diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/MappingKvIterator.java b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/MappingKvIterator.java index eae8f788..03949ae7 100644 --- a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/MappingKvIterator.java +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/MappingKvIterator.java @@ -2,15 +2,18 @@ package com.usatiuk.dhfs.objects; import org.apache.commons.lang3.tuple.Pair; +import java.util.NoSuchElementException; import java.util.function.Function; public class MappingKvIterator, V, V_T> implements CloseableKvIterator { private final CloseableKvIterator _backing; private final Function _transformer; + private final Function, Class> _classMapper; - public MappingKvIterator(CloseableKvIterator backing, Function transformer) { + public MappingKvIterator(CloseableKvIterator backing, Function transformer, Function, Class> classMapper) { _backing = backing; _transformer = transformer; + _classMapper = classMapper; } @Override @@ -18,6 +21,13 @@ public class MappingKvIterator, V, V_T> implements Close return _backing.peekNextKey(); } + @Override + public Class peekNextType() { + if (!hasNext()) + throw new NoSuchElementException(); + return _classMapper.apply(_backing.peekNextType()); + } + @Override public void skip() { _backing.skip(); @@ -38,6 +48,13 @@ public class MappingKvIterator, V, V_T> implements Close return _backing.peekPrevKey(); } + @Override + public Class peekPrevType() { + if (!hasPrev()) + throw new NoSuchElementException(); + return _classMapper.apply(_backing.peekPrevType()); + } + @Override public Pair prev() { var got = _backing.prev(); diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/MergingKvIterator.java b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/MergingKvIterator.java index b6aa55d4..281d240f 100644 --- a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/MergingKvIterator.java +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/MergingKvIterator.java @@ -289,6 +289,28 @@ public class MergingKvIterator, V> extends ReversibleKvI return curVal; } + @Override + protected Class peekTypeImpl() { + switch (_firstMatchState) { + case FirstMatchFound firstMatchFound -> { + return firstMatchFound.iterator().peekNextType(); + } + case FirstMatchConsumed firstMatchConsumed -> { + doHydrate(); + break; + } + default -> { + } + } + + if (_sortedIterators.isEmpty()) + throw new NoSuchElementException(); + + return _goingForward + ? _sortedIterators.firstEntry().getValue().peekNextType() + : _sortedIterators.lastEntry().getValue().peekNextType(); + } + @Override public void close() { diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/NavigableMapKvIterator.java b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/NavigableMapKvIterator.java index c1f07007..8174e3e4 100644 --- a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/NavigableMapKvIterator.java +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/NavigableMapKvIterator.java @@ -91,6 +91,13 @@ public class NavigableMapKvIterator, V> extends Reversib return Pair.of(ret); } + @Override + protected Class peekTypeImpl() { + if (_next == null) + throw new NoSuchElementException("No more elements"); + return (Class) _next.getValue().getClass(); + } + @Override public void close() { } diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/PredicateKvIterator.java b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/PredicateKvIterator.java index a39bfad4..bbe71eaa 100644 --- a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/PredicateKvIterator.java +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/PredicateKvIterator.java @@ -131,6 +131,17 @@ public class PredicateKvIterator, V, V_T> extends Revers return ret; } + @Override + protected Class peekTypeImpl() { + if (!_checkedNext) + fillNext(); + + if (_next == null) + throw new NoSuchElementException("No more elements"); + + return _next.getValue().getClass(); + } + @Override public void close() { _backing.close(); diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/ReversedKvIterator.java b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/ReversedKvIterator.java index 88b23f30..6c5f1e8d 100644 --- a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/ReversedKvIterator.java +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/ReversedKvIterator.java @@ -29,6 +29,11 @@ public class ReversedKvIterator, V> implements Closeable return _backing.peekPrevKey(); } + @Override + public Class peekNextType() { + return _backing.peekPrevType(); + } + @Override public void skip() { _backing.skipPrev(); @@ -39,6 +44,11 @@ public class ReversedKvIterator, V> implements Closeable return _backing.peekNextKey(); } + @Override + public Class peekPrevType() { + return _backing.peekNextType(); + } + @Override public Pair prev() { return _backing.next(); diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/ReversibleKvIterator.java b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/ReversibleKvIterator.java index a13a063d..f707a91b 100644 --- a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/ReversibleKvIterator.java +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/ReversibleKvIterator.java @@ -27,6 +27,8 @@ public abstract class ReversibleKvIterator, V> implement abstract protected Pair nextImpl(); + abstract protected Class peekTypeImpl(); + @Override public K peekNextKey() { ensureForward(); @@ -76,4 +78,15 @@ public abstract class ReversibleKvIterator, V> implement skipImpl(); } + @Override + public Class peekNextType() { + ensureForward(); + return peekTypeImpl(); + } + + @Override + public Class peekPrevType() { + ensureBackward(); + return peekTypeImpl(); + } } diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/TombstoneMergingKvIterator.java b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/TombstoneMergingKvIterator.java index e8e01e27..6f714d64 100644 --- a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/TombstoneMergingKvIterator.java +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/TombstoneMergingKvIterator.java @@ -9,24 +9,25 @@ import java.util.List; public class TombstoneMergingKvIterator, V> implements CloseableKvIterator { private final CloseableKvIterator _backing; private final String _name; + private final Class _returnType; - public TombstoneMergingKvIterator(String name, IteratorStart startType, K startKey, List>> iterators) { + public TombstoneMergingKvIterator(String name, IteratorStart startType, K startKey, List>> iterators, + Class returnType) { _name = name; - _backing = new PredicateKvIterator<>( + _returnType = returnType; + _backing = new MappingKvIterator<>(new TypePredicateKvIterator<>( new MergingKvIterator<>(name + "-merging", startType, startKey, iterators), startType, startKey, - pair -> { - Log.tracev("{0} - Processing pair {1}", _name, pair); - if (pair instanceof Tombstone) { - return null; - } - return ((Data) pair).value(); - }); + k -> { + assert !k.equals(MaybeTombstone.class); + assert Tombstone.class.isAssignableFrom(k) || Data.class.isAssignableFrom(k); + return Data.class.isAssignableFrom(k); + }), t -> (V) returnType.cast(Data.class.cast(t).value()), (t) -> returnType); } @SafeVarargs - public TombstoneMergingKvIterator(String name, IteratorStart startType, K startKey, IterProdFn>... iterators) { - this(name, startType, startKey, List.of(iterators)); + public TombstoneMergingKvIterator(String name, IteratorStart startType, K startKey, Class returnType, IterProdFn>... iterators) { + this(name, startType, startKey, List.of(iterators), returnType); } @Override @@ -34,6 +35,11 @@ public class TombstoneMergingKvIterator, V> implements C return _backing.peekNextKey(); } + @Override + public Class peekNextType() { + return _returnType; + } + @Override public void skip() { _backing.skip(); @@ -44,6 +50,11 @@ public class TombstoneMergingKvIterator, V> implements C return _backing.peekPrevKey(); } + @Override + public Class peekPrevType() { + return _returnType; + } + @Override public Pair prev() { return _backing.prev(); diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/TypePredicateKvIterator.java b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/TypePredicateKvIterator.java new file mode 100644 index 00000000..964cf062 --- /dev/null +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/TypePredicateKvIterator.java @@ -0,0 +1,141 @@ +package com.usatiuk.dhfs.objects; + +import com.usatiuk.dhfs.objects.persistence.IteratorStart; +import org.apache.commons.lang3.tuple.Pair; + +import java.util.NoSuchElementException; +import java.util.function.Function; + +public class TypePredicateKvIterator, V> extends ReversibleKvIterator { + private final CloseableKvIterator _backing; + private final Function, Boolean> _filter; + private K _next; + + public TypePredicateKvIterator(CloseableKvIterator backing, IteratorStart start, K startKey, Function, Boolean> filter) { + _goingForward = true; + _backing = backing; + _filter = filter; + fillNext(); + + boolean shouldGoBack = false; + if (start == IteratorStart.LE) { + if (_next == null || _next.compareTo(startKey) > 0) { + shouldGoBack = true; + } + } else if (start == IteratorStart.LT) { + if (_next == null || _next.compareTo(startKey) >= 0) { + shouldGoBack = true; + } + } + + if (shouldGoBack && _backing.hasPrev()) { + _goingForward = false; + _next = null; + fillNext(); + if (_next != null) + _backing.skipPrev(); + _goingForward = true; +// _backing.skip(); + fillNext(); + } + + + switch (start) { + case LT -> { +// assert _next == null || _next.getKey().compareTo(startKey) < 0; + } + case LE -> { +// assert _next == null || _next.getKey().compareTo(startKey) <= 0; + } + case GT -> { + assert _next == null || _next.compareTo(startKey) > 0; + } + case GE -> { + assert _next == null || _next.compareTo(startKey) >= 0; + } + } + } + + private void fillNext() { + while ((_goingForward ? _backing.hasNext() : _backing.hasPrev()) && _next == null) { + var next = _goingForward ? _backing.peekNextType() : _backing.peekPrevType(); + if (!_filter.apply(next)) { + if (_goingForward) + _backing.skip(); + else + _backing.skipPrev(); + continue; + } else { + _next = _goingForward ? _backing.peekNextKey() : _backing.peekPrevKey(); + } + } + } + + @Override + protected void reverse() { + _goingForward = !_goingForward; + _next = null; + + fillNext(); + } + + @Override + protected K peekImpl() { + if (_next == null) + throw new NoSuchElementException(); + return _next; + } + + @Override + protected void skipImpl() { + if (_next == null) + throw new NoSuchElementException(); + _next = null; + if (_goingForward) + _backing.skip(); + else + _backing.skipPrev(); + fillNext(); + } + + @Override + protected boolean hasImpl() { + return _next != null; + } + + @Override + protected Pair nextImpl() { + if (_next == null) + throw new NoSuchElementException("No more elements"); + var retKey = _next; + _next = null; + var nextType = _goingForward ? _backing.peekNextType() : _backing.peekPrevType(); + var got = _goingForward ? _backing.next() : _backing.prev(); + assert got.getKey().equals(retKey); + assert nextType.equals(got.getValue().getClass()); + assert _filter.apply(got.getValue().getClass()); + fillNext(); + return got; + } + + @Override + protected Class peekTypeImpl() { + if (_next == null) + throw new NoSuchElementException("No more elements"); + + return _goingForward ? _backing.peekNextType() : _backing.peekPrevType(); + } + + @Override + public void close() { + _backing.close(); + } + + @Override + public String toString() { + return "KeyPredicateKvIterator{" + + "_backing=" + _backing + + ", _next=" + _next + + '}'; + } +} diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/WritebackObjectPersistentStore.java b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/WritebackObjectPersistentStore.java index 8a843407..b8330733 100644 --- a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/WritebackObjectPersistentStore.java +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/WritebackObjectPersistentStore.java @@ -465,13 +465,22 @@ public class WritebackObjectPersistentStore { _pendingWritesVersionLock.readLock().lock(); try { var curPending = _pendingWrites.get(); - return new TombstoneMergingKvIterator<>("writeback-ps", start, key, + return new TombstoneMergingKvIterator<>("writeback-ps", start, key, JDataVersionedWrapper.class, (tS, tK) -> new MappingKvIterator<>( new NavigableMapKvIterator<>(curPending, tS, tK), e -> switch (e) { case PendingWrite pw -> new Data<>(pw.data()); case PendingDelete d -> new Tombstone<>(); default -> throw new IllegalStateException("Unexpected value: " + e); + }, + e -> { + if (PendingWrite.class.isAssignableFrom(e)) { + return Data.class; + } else if (PendingDelete.class.isAssignableFrom(e)) { + return Tombstone.class; + } else { + throw new IllegalStateException("Unexpected type: " + e); + } }), (tS, tK) -> cachedStore.getIterator(tS, tK)); } finally { diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/persistence/CachingObjectPersistentStore.java b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/persistence/CachingObjectPersistentStore.java index b8404162..e43fb275 100644 --- a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/persistence/CachingObjectPersistentStore.java +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/persistence/CachingObjectPersistentStore.java @@ -63,7 +63,7 @@ public class CachingObjectPersistentStore { int size = obj.map(JDataVersionedWrapper::estimateSize).orElse(16); _curSize += size; - var entry = new CacheEntry(obj.>map(Data::new).orElse(new Tombstone<>()), size); + CacheEntry entry = obj.map(v -> new CacheEntryYes(v, size)).orElse(new CacheEntryDeleted()); var old = _cache.putLast(key, entry); _sortedCache = _sortedCache.plus(key, entry); @@ -87,7 +87,11 @@ public class CachingObjectPersistentStore { try { var got = _cache.get(name); if (got != null) { - return got.object().opt(); + return switch (got) { + case CacheEntryYes yes -> Optional.of(yes.object()); + case CacheEntryDeleted del -> Optional.empty(); + default -> throw new IllegalStateException("Unexpected value: " + got); + }; } } finally { _lock.readLock().unlock(); @@ -137,6 +141,11 @@ public class CachingObjectPersistentStore { return _delegate.peekNextKey(); } + @Override + public Class peekNextType() { + return _delegate.peekNextType(); + } + @Override public void skip() { _delegate.skip(); @@ -157,6 +166,11 @@ public class CachingObjectPersistentStore { return _delegate.peekPrevKey(); } + @Override + public Class peekPrevType() { + return _delegate.peekPrevType(); + } + private void maybeCache(Pair prev) { _lock.writeLock().lock(); try { @@ -209,19 +223,39 @@ public class CachingObjectPersistentStore { (mS, mK) -> new MappingKvIterator<>( new NavigableMapKvIterator<>(curSortedCache, mS, mK), + e -> switch (e) { + case CacheEntryYes pw -> new Data<>(pw.object()); + case CacheEntryDeleted d -> new Tombstone<>(); + default -> throw new IllegalStateException("Unexpected value: " + e); + }, e -> { - Log.tracev("Taken from cache: {0}", e); - return e.object(); - } - ), + if (CacheEntryYes.class.isAssignableFrom(e)) { + return Data.class; + } else if (CacheEntryDeleted.class.isAssignableFrom(e)) { + return Tombstone.class; + } else { + throw new IllegalStateException("Unexpected type: " + e); + } + }), (mS, mK) - -> new MappingKvIterator<>(new CachingKvIterator(delegate.getIterator(mS, mK)), Data::new)); + -> new MappingKvIterator<>(new CachingKvIterator(delegate.getIterator(mS, mK)), Data::new, (d) -> Data.class)); } finally { _lock.readLock().unlock(); } } - private record CacheEntry(MaybeTombstone object, long size) { + private interface CacheEntry { + long size(); + } + + private record CacheEntryYes(JDataVersionedWrapper object, long size) implements CacheEntry { + } + + private record CacheEntryDeleted() implements CacheEntry { + @Override + public long size() { + return 16; + } } public long getLastTxId() { diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/persistence/LmdbObjectPersistentStore.java b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/persistence/LmdbObjectPersistentStore.java index 17ee7360..428ea47f 100644 --- a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/persistence/LmdbObjectPersistentStore.java +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/persistence/LmdbObjectPersistentStore.java @@ -285,6 +285,14 @@ public class LmdbObjectPersistentStore implements ObjectPersistentStore { Log.tracev("Read: {0}, hasNext: {1}", ret, _hasNext); return ret; } + + @Override + protected Class peekTypeImpl() { + if (!_hasNext) + throw new NoSuchElementException(); + + return ByteString.class; + } } @Override diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/persistence/SerializingObjectPersistentStore.java b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/persistence/SerializingObjectPersistentStore.java index f439731e..09587aa7 100644 --- a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/persistence/SerializingObjectPersistentStore.java +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/persistence/SerializingObjectPersistentStore.java @@ -31,7 +31,7 @@ public class SerializingObjectPersistentStore { // Returns an iterator with a view of all commited objects // Does not have to guarantee consistent view, snapshots are handled by upper layers public CloseableKvIterator getIterator(IteratorStart start, JObjectKey key) { - return new MappingKvIterator<>(delegateStore.getIterator(start, key), d -> serializer.deserialize(d)); + return new MappingKvIterator<>(delegateStore.getIterator(start, key), d -> serializer.deserialize(d), (d) -> JDataVersionedWrapper.class); } public TxManifestRaw prepareManifest(TxManifestObj names) { diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/snapshot/SnapshotKvIterator.java b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/snapshot/SnapshotKvIterator.java index f4db3043..d3f5d04a 100644 --- a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/snapshot/SnapshotKvIterator.java +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/snapshot/SnapshotKvIterator.java @@ -187,4 +187,12 @@ public class SnapshotKvIterator extends ReversibleKvIterator> peekTypeImpl() { + if (_next == null) + throw new NoSuchElementException("No more elements"); + + return (Class>) _next.getValue().getClass(); + } + } diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/snapshot/SnapshotManager.java b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/snapshot/SnapshotManager.java index 77b36f46..31205d98 100644 --- a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/snapshot/SnapshotManager.java +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/snapshot/SnapshotManager.java @@ -243,6 +243,11 @@ public class SnapshotManager { return _backing.peekNextKey(); } + @Override + public Class peekNextType() { + return _backing.peekNextType(); + } + @Override public void skip() { _backing.skip(); @@ -253,6 +258,11 @@ public class SnapshotManager { return _backing.peekPrevKey(); } + @Override + public Class peekPrevType() { + return _backing.peekPrevType(); + } + @Override public Pair prev() { var ret = _backing.prev(); @@ -293,10 +303,10 @@ public class SnapshotManager { try { Log.tracev("Getting snapshot {0} iterator for {1} {2}\n" + "objects in snapshots: {3}", _id, start, key, _objects); - return new CheckingSnapshotKvIterator(new TombstoneMergingKvIterator<>("snapshot", start, key, + return new CheckingSnapshotKvIterator(new TombstoneMergingKvIterator<>("snapshot", start, key, JDataVersionedWrapper.class, (tS, tK) -> new SnapshotKvIterator(_objects, _id, tS, tK), - (tS, tK) -> new MappingKvIterator<>( - writebackStore.getIterator(tS, tK), d -> d.version() <= _id ? new Data<>(d) : new Tombstone<>()) + (tS, tK) -> new PredicateKvIterator<>( + writebackStore.getIterator(tS, tK), tS, tK, d -> d.version() <= _id ? new Data<>(d) : null) )); } finally { _lock.readLock().unlock(); diff --git a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/transaction/TransactionFactoryImpl.java b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/transaction/TransactionFactoryImpl.java index 9288fb5a..14e12435 100644 --- a/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/transaction/TransactionFactoryImpl.java +++ b/dhfs-parent/objects/src/main/java/com/usatiuk/dhfs/objects/transaction/TransactionFactoryImpl.java @@ -6,6 +6,7 @@ import com.usatiuk.dhfs.objects.snapshot.SnapshotManager; import io.quarkus.logging.Log; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; +import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.tuple.Pair; import javax.annotation.Nonnull; @@ -74,6 +75,11 @@ public class TransactionFactoryImpl implements TransactionFactory { return _backing.peekNextKey(); } + @Override + public Class peekNextType() { + throw new NotImplementedException(); + } + @Override public void skip() { _backing.skip(); @@ -84,6 +90,11 @@ public class TransactionFactoryImpl implements TransactionFactory { return _backing.peekPrevKey(); } + @Override + public Class peekPrevType() { + throw new NotImplementedException(); + } + @Override public Pair prev() { var got = _backing.prev(); @@ -221,16 +232,26 @@ public class TransactionFactoryImpl implements TransactionFactory { @Override public CloseableKvIterator getIterator(IteratorStart start, JObjectKey key) { Log.tracev("Getting tx iterator with start={0}, key={1}", start, key); - return new ReadTrackingIterator(new TombstoneMergingKvIterator<>("tx", start, key, - (tS, tK) -> new MappingKvIterator<>(new NavigableMapKvIterator<>(_writes, tS, tK), + return new ReadTrackingIterator(new TombstoneMergingKvIterator<>("tx", start, key, ReadTrackingInternalCrap.class, + (tS, tK) -> new MappingKvIterator<>( + new NavigableMapKvIterator<>(_writes, tS, tK), t -> switch (t) { case TxRecord.TxObjectRecordWrite write -> new Data<>(new ReadTrackingInternalCrapTx(write.data())); case TxRecord.TxObjectRecordDeleted deleted -> new Tombstone<>(); case null, default -> null; + }, + e -> { + if (TxRecord.TxObjectRecordWrite.class.isAssignableFrom(e)) { + return Data.class; + } else if (TxRecord.TxObjectRecordDeleted.class.isAssignableFrom(e)) { + return Tombstone.class; + } else { + throw new IllegalStateException("Unexpected type: " + e); + } }), (tS, tK) -> new MappingKvIterator<>(_snapshot.getIterator(tS, tK), - d -> new Data(new ReadTrackingInternalCrapSource(d))))); + d -> new Data<>(new ReadTrackingInternalCrapSource(d)), (t) -> Data.class))); } @Override diff --git a/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/MergingKvIteratorTest.java b/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/MergingKvIteratorTest.java index 430dc635..db2e58ce 100644 --- a/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/MergingKvIteratorTest.java +++ b/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/MergingKvIteratorTest.java @@ -1,6 +1,7 @@ package com.usatiuk.dhfs.objects; import com.usatiuk.dhfs.objects.persistence.IteratorStart; +import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -35,6 +36,11 @@ public class MergingKvIteratorTest { return _next.getKey(); } + @Override + public Class peekNextType() { + throw new NotImplementedException(); + } + @Override public void skip() { if (_next == null) { @@ -49,6 +55,11 @@ public class MergingKvIteratorTest { throw new UnsupportedOperationException(); } + @Override + public Class peekPrevType() { + throw new NotImplementedException(); + } + @Override public Pair prev() { throw new UnsupportedOperationException(); diff --git a/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/PredicateKvIteratorTest.java b/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/PredicateKvIteratorTest.java deleted file mode 100644 index 05ad6d4b..00000000 --- a/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/PredicateKvIteratorTest.java +++ /dev/null @@ -1,161 +0,0 @@ -package com.usatiuk.dhfs.objects; - -import com.usatiuk.dhfs.objects.persistence.IteratorStart; -import org.apache.commons.lang3.tuple.Pair; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.pcollections.TreePMap; - -import java.util.List; - -public class PredicateKvIteratorTest { - - @Test - public void simpleTest() { - var source1 = TreePMap.empty().plus(1, 3).plus(3, 5).plus(4, 6); - var pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.GT, 1), - IteratorStart.GE, 1, v -> (v % 2 == 0) ? v : null); - var expected = List.of(Pair.of(4, 6)); - for (var pair : expected) { - Assertions.assertTrue(pit.hasNext()); - Assertions.assertEquals(pair, pit.next()); - } - } - - @Test - public void ltTest() { - var source1 = TreePMap.empty().plus(1, 3).plus(3, 5).plus(4, 6); - var pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 4), - IteratorStart.LT, 4, v -> (v % 2 == 0) ? v : null); - var expected = List.of(Pair.of(4, 6)); - for (var pair : expected) { - Assertions.assertTrue(pit.hasNext()); - Assertions.assertEquals(pair, pit.next()); - } - Assertions.assertFalse(pit.hasNext()); - } - - @Test - public void ltTest2() { - var source1 = TreePMap.empty().plus(1, 3).plus(3, 5).plus(4, 6); - var pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 1), - IteratorStart.LT, 1, v -> (v % 2 == 0) ? v : null); - Just.checkIterator(pit, Pair.of(4, 6)); - Assertions.assertFalse(pit.hasNext()); - - pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 2), - IteratorStart.LT, 2, v -> (v % 2 == 0) ? v : null); - Just.checkIterator(pit, Pair.of(4, 6)); - Assertions.assertFalse(pit.hasNext()); - - pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 4), - IteratorStart.LT, 4, v -> (v % 2 == 0) ? v : null); - Just.checkIterator(pit, Pair.of(4, 6)); - Assertions.assertFalse(pit.hasNext()); - - pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LE, 4), - IteratorStart.LE, 4, v -> (v % 2 == 0) ? v : null); - Just.checkIterator(pit, Pair.of(4, 6)); - Assertions.assertFalse(pit.hasNext()); - } - - @Test - public void ltTest3() { - var source1 = TreePMap.empty().plus(1, 3).plus(3, 5).plus(4, 6).plus(5, 7).plus(6, 8); - var pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 4), - IteratorStart.LT, 4, v -> (v % 2 == 0) ? v : null); - Just.checkIterator(pit, Pair.of(4, 6), Pair.of(6, 8)); - Assertions.assertFalse(pit.hasNext()); - - pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 5), - IteratorStart.LT, 5, v -> (v % 2 == 0) ? v : null); - Just.checkIterator(pit, Pair.of(4, 6), Pair.of(6, 8)); - Assertions.assertFalse(pit.hasNext()); - - pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 6), - IteratorStart.LT, 6, v -> (v % 2 == 0) ? v : null); - Just.checkIterator(pit, Pair.of(4, 6), Pair.of(6, 8)); - Assertions.assertFalse(pit.hasNext()); - - pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 7), - IteratorStart.LT, 7, v -> (v % 2 == 0) ? v : null); - Just.checkIterator(pit, Pair.of(6, 8)); - Assertions.assertFalse(pit.hasNext()); - - pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 8), - IteratorStart.LT, 8, v -> (v % 2 == 0) ? v : null); - Just.checkIterator(pit, Pair.of(6, 8)); - Assertions.assertFalse(pit.hasNext()); - - pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LE, 6), - IteratorStart.LE, 6, v -> (v % 2 == 0) ? v : null); - Just.checkIterator(pit, Pair.of(6, 8)); - Assertions.assertFalse(pit.hasNext()); - - pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 6), - IteratorStart.LT, 6, v -> (v % 2 == 0) ? v : null); - Assertions.assertTrue(pit.hasNext()); - Assertions.assertEquals(4, pit.peekNextKey()); - Assertions.assertFalse(pit.hasPrev()); - Assertions.assertEquals(4, pit.peekNextKey()); - Assertions.assertFalse(pit.hasPrev()); - Assertions.assertEquals(Pair.of(4, 6), pit.next()); - Assertions.assertTrue(pit.hasNext()); - Assertions.assertEquals(6, pit.peekNextKey()); - Assertions.assertEquals(4, pit.peekPrevKey()); - Assertions.assertEquals(6, pit.peekNextKey()); - Assertions.assertEquals(4, pit.peekPrevKey()); - } - - @Test - public void itTest4() { - var source1 = TreePMap.empty().plus(1, 3).plus(3, 5).plus(4, 6).plus(5, 8).plus(6, 10); - var pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 4), - IteratorStart.LT, 4, v -> (v % 2 == 0) ? v : null); - Just.checkIterator(pit, Pair.of(4, 6), Pair.of(5, 8), Pair.of(6, 10)); - Assertions.assertFalse(pit.hasNext()); - - pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 5), - IteratorStart.LT, 5, v -> (v % 2 == 0) ? v : null); - Just.checkIterator(pit, Pair.of(4, 6), Pair.of(5, 8), Pair.of(6, 10)); - Assertions.assertFalse(pit.hasNext()); - - pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 6), - IteratorStart.LT, 6, v -> (v % 2 == 0) ? v : null); - Just.checkIterator(pit, Pair.of(5, 8), Pair.of(6, 10)); - Assertions.assertFalse(pit.hasNext()); - - pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 7), - IteratorStart.LT, 7, v -> (v % 2 == 0) ? v : null); - Just.checkIterator(pit, Pair.of(6, 10)); - Assertions.assertFalse(pit.hasNext()); - Assertions.assertTrue(pit.hasPrev()); - Assertions.assertEquals(6, pit.peekPrevKey()); - Assertions.assertEquals(Pair.of(6, 10), pit.prev()); - Assertions.assertTrue(pit.hasNext()); - Assertions.assertEquals(6, pit.peekNextKey()); - - pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 6), - IteratorStart.LT, 6, v -> (v % 2 == 0) ? v : null); - Assertions.assertTrue(pit.hasNext()); - Assertions.assertEquals(5, pit.peekNextKey()); - Assertions.assertTrue(pit.hasPrev()); - Assertions.assertEquals(4, pit.peekPrevKey()); - Assertions.assertEquals(5, pit.peekNextKey()); - Assertions.assertEquals(4, pit.peekPrevKey()); - Assertions.assertEquals(Pair.of(5, 8), pit.next()); - Assertions.assertTrue(pit.hasNext()); - Assertions.assertEquals(6, pit.peekNextKey()); - Assertions.assertEquals(5, pit.peekPrevKey()); - Assertions.assertEquals(6, pit.peekNextKey()); - Assertions.assertEquals(5, pit.peekPrevKey()); - } - -// @Test -// public void reverseTest() { -// var source1 = TreePMap.empty().plus(1, 3).plus(3, 5).plus(4, 6); -// var pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 4), -// IteratorStart.LT, 4, v -> (v % 2 == 0) ? v : null); -// -// } -} diff --git a/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/TypePredicateKvIteratorTest.java b/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/TypePredicateKvIteratorTest.java new file mode 100644 index 00000000..94eb3afe --- /dev/null +++ b/dhfs-parent/objects/src/test/java/com/usatiuk/dhfs/objects/TypePredicateKvIteratorTest.java @@ -0,0 +1,161 @@ +package com.usatiuk.dhfs.objects; + +import com.usatiuk.dhfs.objects.persistence.IteratorStart; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.pcollections.TreePMap; + +import java.util.List; + +public class TypePredicateKvIteratorTest { + +// @Test +// public void simpleTest() { +// var source1 = TreePMap.empty().plus(1, 3).plus(3, 5).plus(4, 6); +// var pit = new TypePredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.GT, 1), +// IteratorStart.GE, 1, v -> (v % 2 == 0) ? v : null); +// var expected = List.of(Pair.of(4, 6)); +// for (var pair : expected) { +// Assertions.assertTrue(pit.hasNext()); +// Assertions.assertEquals(pair, pit.next()); +// } +// } +// +// @Test +// public void ltTest() { +// var source1 = TreePMap.empty().plus(1, 3).plus(3, 5).plus(4, 6); +// var pit = new TypePredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 4), +// IteratorStart.LT, 4, v -> (v % 2 == 0) ? v : null); +// var expected = List.of(Pair.of(4, 6)); +// for (var pair : expected) { +// Assertions.assertTrue(pit.hasNext()); +// Assertions.assertEquals(pair, pit.next()); +// } +// Assertions.assertFalse(pit.hasNext()); +// } +// +// @Test +// public void ltTest2() { +// var source1 = TreePMap.empty().plus(1, 3).plus(3, 5).plus(4, 6); +// var pit = new TypePredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 1), +// IteratorStart.LT, 1, v -> (v % 2 == 0) ? v : null); +// Just.checkIterator(pit, Pair.of(4, 6)); +// Assertions.assertFalse(pit.hasNext()); +// +// pit = new TypePredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 2), +// IteratorStart.LT, 2, v -> (v % 2 == 0) ? v : null); +// Just.checkIterator(pit, Pair.of(4, 6)); +// Assertions.assertFalse(pit.hasNext()); +// +// pit = new TypePredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 4), +// IteratorStart.LT, 4, v -> (v % 2 == 0) ? v : null); +// Just.checkIterator(pit, Pair.of(4, 6)); +// Assertions.assertFalse(pit.hasNext()); +// +// pit = new TypePredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LE, 4), +// IteratorStart.LE, 4, v -> (v % 2 == 0) ? v : null); +// Just.checkIterator(pit, Pair.of(4, 6)); +// Assertions.assertFalse(pit.hasNext()); +// } +// +// @Test +// public void ltTest3() { +// var source1 = TreePMap.empty().plus(1, 3).plus(3, 5).plus(4, 6).plus(5, 7).plus(6, 8); +// var pit = new TypePredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 4), +// IteratorStart.LT, 4, v -> (v % 2 == 0) ? v : null); +// Just.checkIterator(pit, Pair.of(4, 6), Pair.of(6, 8)); +// Assertions.assertFalse(pit.hasNext()); +// +// pit = new TypePredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 5), +// IteratorStart.LT, 5, v -> (v % 2 == 0) ? v : null); +// Just.checkIterator(pit, Pair.of(4, 6), Pair.of(6, 8)); +// Assertions.assertFalse(pit.hasNext()); +// +// pit = new TypePredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 6), +// IteratorStart.LT, 6, v -> (v % 2 == 0) ? v : null); +// Just.checkIterator(pit, Pair.of(4, 6), Pair.of(6, 8)); +// Assertions.assertFalse(pit.hasNext()); +// +// pit = new TypePredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 7), +// IteratorStart.LT, 7, v -> (v % 2 == 0) ? v : null); +// Just.checkIterator(pit, Pair.of(6, 8)); +// Assertions.assertFalse(pit.hasNext()); +// +// pit = new TypePredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 8), +// IteratorStart.LT, 8, v -> (v % 2 == 0) ? v : null); +// Just.checkIterator(pit, Pair.of(6, 8)); +// Assertions.assertFalse(pit.hasNext()); +// +// pit = new TypePredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LE, 6), +// IteratorStart.LE, 6, v -> (v % 2 == 0) ? v : null); +// Just.checkIterator(pit, Pair.of(6, 8)); +// Assertions.assertFalse(pit.hasNext()); +// +// pit = new TypePredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 6), +// IteratorStart.LT, 6, v -> (v % 2 == 0) ? v : null); +// Assertions.assertTrue(pit.hasNext()); +// Assertions.assertEquals(4, pit.peekNextKey()); +// Assertions.assertFalse(pit.hasPrev()); +// Assertions.assertEquals(4, pit.peekNextKey()); +// Assertions.assertFalse(pit.hasPrev()); +// Assertions.assertEquals(Pair.of(4, 6), pit.next()); +// Assertions.assertTrue(pit.hasNext()); +// Assertions.assertEquals(6, pit.peekNextKey()); +// Assertions.assertEquals(4, pit.peekPrevKey()); +// Assertions.assertEquals(6, pit.peekNextKey()); +// Assertions.assertEquals(4, pit.peekPrevKey()); +// } +// +// @Test +// public void itTest4() { +// var source1 = TreePMap.empty().plus(1, 3).plus(3, 5).plus(4, 6).plus(5, 8).plus(6, 10); +// var pit = new TypePredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 4), +// IteratorStart.LT, 4, v -> (v % 2 == 0) ? v : null); +// Just.checkIterator(pit, Pair.of(4, 6), Pair.of(5, 8), Pair.of(6, 10)); +// Assertions.assertFalse(pit.hasNext()); +// +// pit = new TypePredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 5), +// IteratorStart.LT, 5, v -> (v % 2 == 0) ? v : null); +// Just.checkIterator(pit, Pair.of(4, 6), Pair.of(5, 8), Pair.of(6, 10)); +// Assertions.assertFalse(pit.hasNext()); +// +// pit = new TypePredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 6), +// IteratorStart.LT, 6, v -> (v % 2 == 0) ? v : null); +// Just.checkIterator(pit, Pair.of(5, 8), Pair.of(6, 10)); +// Assertions.assertFalse(pit.hasNext()); +// +// pit = new TypePredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 7), +// IteratorStart.LT, 7, v -> (v % 2 == 0) ? v : null); +// Just.checkIterator(pit, Pair.of(6, 10)); +// Assertions.assertFalse(pit.hasNext()); +// Assertions.assertTrue(pit.hasPrev()); +// Assertions.assertEquals(6, pit.peekPrevKey()); +// Assertions.assertEquals(Pair.of(6, 10), pit.prev()); +// Assertions.assertTrue(pit.hasNext()); +// Assertions.assertEquals(6, pit.peekNextKey()); +// +// pit = new TypePredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 6), +// IteratorStart.LT, 6, v -> (v % 2 == 0) ? v : null); +// Assertions.assertTrue(pit.hasNext()); +// Assertions.assertEquals(5, pit.peekNextKey()); +// Assertions.assertTrue(pit.hasPrev()); +// Assertions.assertEquals(4, pit.peekPrevKey()); +// Assertions.assertEquals(5, pit.peekNextKey()); +// Assertions.assertEquals(4, pit.peekPrevKey()); +// Assertions.assertEquals(Pair.of(5, 8), pit.next()); +// Assertions.assertTrue(pit.hasNext()); +// Assertions.assertEquals(6, pit.peekNextKey()); +// Assertions.assertEquals(5, pit.peekPrevKey()); +// Assertions.assertEquals(6, pit.peekNextKey()); +// Assertions.assertEquals(5, pit.peekPrevKey()); +// } + +// @Test +// public void reverseTest() { +// var source1 = TreePMap.empty().plus(1, 3).plus(3, 5).plus(4, 6); +// var pit = new PredicateKvIterator<>(new NavigableMapKvIterator<>(source1, IteratorStart.LT, 4), +// IteratorStart.LT, 4, v -> (v % 2 == 0) ? v : null); +// +// } +} diff --git a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jmap/JMapIterator.java b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jmap/JMapIterator.java index f13f1af7..ccb034a7 100644 --- a/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jmap/JMapIterator.java +++ b/dhfs-parent/server/src/main/java/com/usatiuk/dhfs/objects/jmap/JMapIterator.java @@ -45,6 +45,11 @@ public class JMapIterator> implements Closeabl return keyToKey(_backing.peekNextKey()); } + @Override + public Class> peekNextType() { + throw new NotImplementedException(); + } + @Override public void skip() { if (!_hasNext) { @@ -58,6 +63,11 @@ public class JMapIterator> implements Closeabl throw new NotImplementedException(); } + @Override + public Class> peekPrevType() { + throw new NotImplementedException(); + } + @Override public Pair> prev() { throw new NotImplementedException();