From 480e984417c17915b31e33de193ff08e118fcec9 Mon Sep 17 00:00:00 2001 From: at055612 <22818309+at055612@users.noreply.github.com> Date: Thu, 6 Mar 2025 16:04:06 +0000 Subject: [PATCH 01/21] gh-249 Remove MDB_UNSIGNEDKEY, let CursorIterable call mdb_cmp There are now essentially three ways of configuring comparators when creating a Dbi. **null comparator** LMDB will use its own comparator & CursorIterable will call down to mdb_cmp for comparisons between the current cursor key and the range start/stop key. **provided comparator** LMDB will use its own comparator & CursorIterable will use the provided comparator for comparisons between the current cursor key and the range start/stop key. **provided comparator with nativeCb==true** LMDB will call back to java for all comparator duties. CursorIterable will use the same provided comparator for comparisons between the current cursor key and the range start/stop key. The methods `getSignedComparator()` and `getUnsignedComparator()` have been made public so users of this library can access them. --- src/main/java/org/lmdbjava/BufferProxy.java | 40 +- .../java/org/lmdbjava/ByteArrayProxy.java | 4 +- src/main/java/org/lmdbjava/ByteBufProxy.java | 4 +- .../java/org/lmdbjava/ByteBufferProxy.java | 4 +- src/main/java/org/lmdbjava/Cursor.java | 4 + .../java/org/lmdbjava/CursorIterable.java | 403 +++++++++++------- src/main/java/org/lmdbjava/Dbi.java | 15 +- src/main/java/org/lmdbjava/DbiFlags.java | 21 +- .../java/org/lmdbjava/DirectBufferProxy.java | 4 +- src/main/java/org/lmdbjava/Env.java | 21 +- src/main/java/org/lmdbjava/Key.java | 74 ++++ src/main/java/org/lmdbjava/KeyRangeType.java | 48 +-- src/main/java/org/lmdbjava/Library.java | 2 + .../java/org/lmdbjava/RangeComparator.java | 19 + .../java/org/lmdbjava/ComparatorTest.java | 8 +- .../org/lmdbjava/CursorIterablePerfTest.java | 163 +++++++ .../java/org/lmdbjava/CursorIterableTest.java | 238 ++++++++--- src/test/java/org/lmdbjava/DbiTest.java | 4 +- src/test/java/org/lmdbjava/KeyRangeTest.java | 5 +- src/test/java/org/lmdbjava/TestUtils.java | 2 + 20 files changed, 768 insertions(+), 315 deletions(-) create mode 100644 src/main/java/org/lmdbjava/Key.java create mode 100644 src/main/java/org/lmdbjava/RangeComparator.java create mode 100644 src/test/java/org/lmdbjava/CursorIterablePerfTest.java diff --git a/src/main/java/org/lmdbjava/BufferProxy.java b/src/main/java/org/lmdbjava/BufferProxy.java index e66031d2..26d9db74 100644 --- a/src/main/java/org/lmdbjava/BufferProxy.java +++ b/src/main/java/org/lmdbjava/BufferProxy.java @@ -16,10 +16,6 @@ package org.lmdbjava; import static java.lang.Long.BYTES; -import static org.lmdbjava.DbiFlags.MDB_INTEGERKEY; -import static org.lmdbjava.DbiFlags.MDB_UNSIGNEDKEY; -import static org.lmdbjava.MaskedFlag.isSet; -import static org.lmdbjava.MaskedFlag.mask; import java.util.Comparator; import jnr.ffi.Pointer; @@ -70,36 +66,25 @@ protected BufferProxy() {} */ protected abstract byte[] getBytes(T buffer); - /** - * Get a suitable default {@link Comparator} given the provided flags. - * - *

The provided comparator must strictly match the lexicographical order of keys in the native - * LMDB database. - * - * @param flags for the database - * @return a comparator that can be used (never null) - */ - protected Comparator getComparator(DbiFlags... flags) { - final int intFlag = mask(flags); - - return isSet(intFlag, MDB_INTEGERKEY) || isSet(intFlag, MDB_UNSIGNEDKEY) - ? getUnsignedComparator() - : getSignedComparator(); - } - /** * Get a suitable default {@link Comparator} to compare numeric key values as signed. * + *

+ * Note: LMDB's default comparator is unsigned so if this is used only for the {@link CursorIterable} + * start/stop key comparisons then its behaviour will differ from the iteration order. Use + * with caution. + *

+ * * @return a comparator that can be used (never null) */ - protected abstract Comparator getSignedComparator(); + public abstract Comparator getSignedComparator(); /** * Get a suitable default {@link Comparator} to compare numeric key values as unsigned. * * @return a comparator that can be used (never null) */ - protected abstract Comparator getUnsignedComparator(); + public abstract Comparator getUnsignedComparator(); /** * Called when the MDB_val should be set to reflect the passed buffer. This buffer @@ -140,4 +125,13 @@ protected Comparator getComparator(DbiFlags... flags) { final KeyVal keyVal() { return new KeyVal<>(this); } + + /** + * Create a new {@link Key} to hold pointers for this buffer proxy. + * + * @return a non-null key holder + */ + final Key key() { + return new Key<>(this); + } } diff --git a/src/main/java/org/lmdbjava/ByteArrayProxy.java b/src/main/java/org/lmdbjava/ByteArrayProxy.java index 4a22ab83..3aeba047 100644 --- a/src/main/java/org/lmdbjava/ByteArrayProxy.java +++ b/src/main/java/org/lmdbjava/ByteArrayProxy.java @@ -104,12 +104,12 @@ protected byte[] getBytes(final byte[] buffer) { } @Override - protected Comparator getSignedComparator() { + public Comparator getSignedComparator() { return signedComparator; } @Override - protected Comparator getUnsignedComparator() { + public Comparator getUnsignedComparator() { return unsignedComparator; } diff --git a/src/main/java/org/lmdbjava/ByteBufProxy.java b/src/main/java/org/lmdbjava/ByteBufProxy.java index cac5b97b..933b52a4 100644 --- a/src/main/java/org/lmdbjava/ByteBufProxy.java +++ b/src/main/java/org/lmdbjava/ByteBufProxy.java @@ -114,12 +114,12 @@ protected ByteBuf allocate() { } @Override - protected Comparator getSignedComparator() { + public Comparator getSignedComparator() { return comparator; } @Override - protected Comparator getUnsignedComparator() { + public Comparator getUnsignedComparator() { return comparator; } diff --git a/src/main/java/org/lmdbjava/ByteBufferProxy.java b/src/main/java/org/lmdbjava/ByteBufferProxy.java index 2b7cdf0d..1fce090a 100644 --- a/src/main/java/org/lmdbjava/ByteBufferProxy.java +++ b/src/main/java/org/lmdbjava/ByteBufferProxy.java @@ -182,12 +182,12 @@ protected final ByteBuffer allocate() { } @Override - protected Comparator getSignedComparator() { + public Comparator getSignedComparator() { return signedComparator; } @Override - protected Comparator getUnsignedComparator() { + public Comparator getUnsignedComparator() { return unsignedComparator; } diff --git a/src/main/java/org/lmdbjava/Cursor.java b/src/main/java/org/lmdbjava/Cursor.java index d49a9bed..9070cff6 100644 --- a/src/main/java/org/lmdbjava/Cursor.java +++ b/src/main/java/org/lmdbjava/Cursor.java @@ -196,6 +196,10 @@ public T key() { return kv.key(); } + KeyVal keyVal() { + return kv; + } + /** * Position at last key/data item. * diff --git a/src/main/java/org/lmdbjava/CursorIterable.java b/src/main/java/org/lmdbjava/CursorIterable.java index 6a03bd90..6b92a9cd 100644 --- a/src/main/java/org/lmdbjava/CursorIterable.java +++ b/src/main/java/org/lmdbjava/CursorIterable.java @@ -21,10 +21,14 @@ import static org.lmdbjava.CursorIterable.State.REQUIRES_NEXT_OP; import static org.lmdbjava.CursorIterable.State.TERMINATED; import static org.lmdbjava.GetOp.MDB_SET_RANGE; +import static org.lmdbjava.Library.LIB; import java.util.Comparator; import java.util.Iterator; import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.function.Supplier; +import jnr.ffi.Pointer; import org.lmdbjava.KeyRangeType.CursorOp; import org.lmdbjava.KeyRangeType.IteratorOp; @@ -38,185 +42,266 @@ */ public final class CursorIterable implements Iterable>, AutoCloseable { - private final Comparator comparator; - private final Cursor cursor; - private final KeyVal entry; - private boolean iteratorReturned; - private final KeyRange range; - private State state = REQUIRES_INITIAL_OP; - - CursorIterable( - final Txn txn, final Dbi dbi, final KeyRange range, final Comparator comparator) { - this.cursor = dbi.openCursor(txn); - this.range = range; - this.comparator = comparator; - this.entry = new KeyVal<>(); - } - - @Override - public void close() { - cursor.close(); - } - - /** - * Obtain an iterator. - * - *

As iteration of the returned iterator will cause movement of the underlying LMDB cursor, an - * {@link IllegalStateException} is thrown if an attempt is made to obtain the iterator more than - * once. For advanced cursor control (such as being able to iterate over the same data multiple - * times etc) please instead refer to {@link Dbi#openCursor(org.lmdbjava.Txn)}. - * - * @return an iterator - */ - @Override - public Iterator> iterator() { - if (iteratorReturned) { - throw new IllegalStateException("Iterator can only be returned once"); - } - iteratorReturned = true; + // private final Comparator comparator; + private final RangeComparator rangeComparator; + private final Cursor cursor; + private final Dbi dbi; + private final KeyVal entry; + private boolean iteratorReturned; + private final KeyRange range; + private State state = REQUIRES_INITIAL_OP; + private final Key startKey; + private final Key stopKey; - return new Iterator>() { - @Override - public boolean hasNext() { - while (state != RELEASED && state != TERMINATED) { - update(); - } - return state == RELEASED; - } + CursorIterable( + final Txn txn, + final Dbi dbi, + final KeyRange range, + final Comparator comparator, + final BufferProxy proxy) { + this.cursor = dbi.openCursor(txn); + this.dbi = dbi; + this.range = range; + this.entry = new KeyVal<>(); - @Override - public KeyVal next() { - if (!hasNext()) { - throw new NoSuchElementException(); + if (comparator != null) { + // User supplied java-side comparator so use that + this.rangeComparator = createJavaRangeComparator(range, comparator, entry::key); + this.startKey = null; + this.stopKey = null; + } else { + // No java-side comparator so call down to LMDB to do the comparison + this.rangeComparator = createLmdbDbiComparator(txn.pointer(), dbi.pointer()); + // Allocate buffers for use with the start/stop keys if required. + // Saves us copying bytes on each comparison + this.startKey = createKey(range.getStart(), proxy); + this.stopKey = createKey(range.getStop(), proxy); } - state = REQUIRES_NEXT_OP; - return entry; - } - - @Override - public void remove() { - cursor.delete(); - } - }; - } - - private void executeCursorOp(final CursorOp op) { - final boolean found; - switch (op) { - case FIRST: - found = cursor.first(); - break; - case LAST: - found = cursor.last(); - break; - case NEXT: - found = cursor.next(); - break; - case PREV: - found = cursor.prev(); - break; - case GET_START_KEY: - found = cursor.get(range.getStart(), MDB_SET_RANGE); - break; - case GET_START_KEY_BACKWARD: - found = cursor.get(range.getStart(), MDB_SET_RANGE) || cursor.last(); - break; - default: - throw new IllegalStateException("Unknown cursor operation"); } - entry.setK(found ? cursor.key() : null); - entry.setV(found ? cursor.val() : null); - } - - private void executeIteratorOp() { - final IteratorOp op = - range.getType().iteratorOp(range.getStart(), range.getStop(), entry.key(), comparator); - switch (op) { - case CALL_NEXT_OP: - executeCursorOp(range.getType().nextOp()); - state = REQUIRES_ITERATOR_OP; - break; - case TERMINATE: - state = TERMINATED; - break; - case RELEASE: - state = RELEASED; - break; - default: - throw new IllegalStateException("Unknown operation"); + + private Key createKey(final T keyBuffer, final BufferProxy proxy) { + if (keyBuffer != null) { + final Key key = proxy.key(); + key.keyIn(keyBuffer); + return key; + } else { + return null; + } } - } - - private void update() { - switch (state) { - case REQUIRES_INITIAL_OP: - executeCursorOp(range.getType().initialOp()); - state = REQUIRES_ITERATOR_OP; - break; - case REQUIRES_NEXT_OP: - executeCursorOp(range.getType().nextOp()); - state = REQUIRES_ITERATOR_OP; - break; - case REQUIRES_ITERATOR_OP: - executeIteratorOp(); - break; - case TERMINATED: - break; - default: - throw new IllegalStateException("Unknown state"); + + static RangeComparator createJavaRangeComparator( + final KeyRange range, + final Comparator comparator, + final Supplier currentKeySupplier) { + final T start = range.getStart(); + final T stop = range.getStop(); + return new RangeComparator() { + @Override + public int compareToStartKey() { + return comparator.compare(currentKeySupplier.get(), start); + } + + @Override + public int compareToStopKey() { + return comparator.compare(currentKeySupplier.get(), stop); + } + }; } - } - /** - * Holder for a key and value pair. - * - *

The same holder instance will always be returned for a given iterator. The returned keys and - * values may change or point to different memory locations following changes in the iterator, - * cursor or transaction. - * - * @param buffer type - */ - public static final class KeyVal { + /** + * Calls down to mdb_cmp to make use of the comparator that LMDB uses for insertion order. + * + * @param txnPointer The pointer to the transaction. + * @param dbiPointer The pointer to the Dbi so LMDB can use the comparator of the Dbi + */ + private RangeComparator createLmdbDbiComparator( + final Pointer txnPointer, final Pointer dbiPointer) { + Objects.requireNonNull(txnPointer); + Objects.requireNonNull(dbiPointer); + Objects.requireNonNull(cursor); + + return new RangeComparator() { + @Override + public int compareToStartKey() { + return LIB.mdb_cmp(txnPointer, dbiPointer, cursor.keyVal().pointerKey(), startKey.pointerKey()); + } - private T k; - private T v; + @Override + public int compareToStopKey() { + return LIB.mdb_cmp(txnPointer, dbiPointer, cursor.keyVal().pointerKey(), stopKey.pointerKey()); + } + }; + } - /** Explicitly-defined default constructor to avoid warnings. */ - public KeyVal() {} + @Override + public void close() { + cursor.close(); + } /** - * The key. + * Obtain an iterator. + * + *

As iteration of the returned iterator will cause movement of the underlying LMDB cursor, an + * {@link IllegalStateException} is thrown if an attempt is made to obtain the iterator more than + * once. For advanced cursor control (such as being able to iterate over the same data multiple + * times etc) please instead refer to {@link Dbi#openCursor(org.lmdbjava.Txn)}. * - * @return key + * @return an iterator */ - public T key() { - return k; + @Override + public Iterator> iterator() { + if (iteratorReturned) { + throw new IllegalStateException("Iterator can only be returned once"); + } + iteratorReturned = true; + + return new Iterator>() { + @Override + public boolean hasNext() { + while (state != RELEASED && state != TERMINATED) { + update(); + } + return state == RELEASED; + } + + @Override + public KeyVal next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + state = REQUIRES_NEXT_OP; + return entry; + } + + @Override + public void remove() { + cursor.delete(); + } + }; + } + + private void executeCursorOp(final CursorOp op) { + final boolean found; + switch (op) { + case FIRST: + found = cursor.first(); + break; + case LAST: + found = cursor.last(); + break; + case NEXT: + found = cursor.next(); + break; + case PREV: + found = cursor.prev(); + break; + case GET_START_KEY: + found = cursor.get(range.getStart(), MDB_SET_RANGE); + break; + case GET_START_KEY_BACKWARD: + found = cursor.get(range.getStart(), MDB_SET_RANGE) || cursor.last(); + break; + default: + throw new IllegalStateException("Unknown cursor operation"); + } + entry.setK(found ? cursor.key() : null); + entry.setV(found ? cursor.val() : null); + } + + private void executeIteratorOp() { + final IteratorOp op = + range.getType().iteratorOp(range.getStart(), range.getStop(), entry.key(), rangeComparator); + switch (op) { + case CALL_NEXT_OP: + executeCursorOp(range.getType().nextOp()); + state = REQUIRES_ITERATOR_OP; + break; + case TERMINATE: + state = TERMINATED; + break; + case RELEASE: + state = RELEASED; + break; + default: + throw new IllegalStateException("Unknown operation"); + } + } + + private void update() { + switch (state) { + case REQUIRES_INITIAL_OP: + executeCursorOp(range.getType().initialOp()); + state = REQUIRES_ITERATOR_OP; + break; + case REQUIRES_NEXT_OP: + executeCursorOp(range.getType().nextOp()); + state = REQUIRES_ITERATOR_OP; + break; + case REQUIRES_ITERATOR_OP: + executeIteratorOp(); + break; + case TERMINATED: + break; + default: + throw new IllegalStateException("Unknown state"); + } } /** - * The value. + * Holder for a key and value pair. + * + *

The same holder instance will always be returned for a given iterator. The returned keys and + * values may change or point to different memory locations following changes in the iterator, + * cursor or transaction. * - * @return value + * @param buffer type */ - public T val() { - return v; - } + public static final class KeyVal { + + private T k; + private T v; + + /** + * Explicitly-defined default constructor to avoid warnings. + */ + public KeyVal() { + } + + /** + * The key. + * + * @return key + */ + public T key() { + return k; + } + + /** + * The value. + * + * @return value + */ + public T val() { + return v; + } - void setK(final T key) { - this.k = key; + void setK(final T key) { + this.k = key; + } + + void setV(final T val) { + this.v = val; + } } - void setV(final T val) { - this.v = val; + /** + * Represents the internal {@link CursorIterable} state. + */ + enum State { + REQUIRES_INITIAL_OP, + REQUIRES_NEXT_OP, + REQUIRES_ITERATOR_OP, + RELEASED, + TERMINATED } - } - - /** Represents the internal {@link CursorIterable} state. */ - enum State { - REQUIRES_INITIAL_OP, - REQUIRES_NEXT_OP, - REQUIRES_ITERATOR_OP, - RELEASED, - TERMINATED - } } diff --git a/src/main/java/org/lmdbjava/Dbi.java b/src/main/java/org/lmdbjava/Dbi.java index ad1bb5a7..c622462b 100644 --- a/src/main/java/org/lmdbjava/Dbi.java +++ b/src/main/java/org/lmdbjava/Dbi.java @@ -54,6 +54,7 @@ public final class Dbi { private final Env env; private final byte[] name; private final Pointer ptr; + private final BufferProxy proxy; Dbi( final Env env, @@ -69,16 +70,14 @@ public final class Dbi { } this.env = env; this.name = name == null ? null : Arrays.copyOf(name, name.length); - if (comparator == null) { - this.comparator = proxy.getComparator(flags); - } else { - this.comparator = comparator; - } + this.proxy = proxy; + this.comparator = comparator; final int flagsMask = mask(true, flags); final Pointer dbiPtr = allocateDirect(RUNTIME, ADDRESS); checkRc(LIB.mdb_dbi_open(txn.pointer(), name, flagsMask, dbiPtr)); ptr = dbiPtr.getPointer(0); if (nativeCb) { + requireNonNull(comparator, "comparator cannot be null if nativeCb is set"); this.ccb = (keyA, keyB) -> { final T compKeyA = proxy.allocate(); @@ -96,6 +95,10 @@ public final class Dbi { } } + Pointer pointer() { + return ptr; + } + /** * Close the database handle (normally unnecessary; use with caution). * @@ -275,7 +278,7 @@ public CursorIterable iterate(final Txn txn, final KeyRange range) { env.checkNotClosed(); txn.checkReady(); } - return new CursorIterable<>(txn, this, range, comparator); + return new CursorIterable<>(txn, this, range, comparator, proxy); } /** diff --git a/src/main/java/org/lmdbjava/DbiFlags.java b/src/main/java/org/lmdbjava/DbiFlags.java index 123ec9fd..2f5eadf6 100644 --- a/src/main/java/org/lmdbjava/DbiFlags.java +++ b/src/main/java/org/lmdbjava/DbiFlags.java @@ -55,14 +55,6 @@ public enum DbiFlags implements MaskedFlag { * #MDB_INTEGERKEY} keys. */ MDB_INTEGERDUP(0x20), - /** - * Compare the numeric keys in native byte order and as unsigned. - * - *

This option is applied only to {@link java.nio.ByteBuffer}, {@link org.agrona.DirectBuffer} - * and byte array keys. {@link io.netty.buffer.ByteBuf} keys are always compared in native byte - * order and as unsigned. - */ - MDB_UNSIGNEDKEY(0x30, false), /** * With {@link #MDB_DUPSORT}, use reverse string dups. * @@ -78,24 +70,13 @@ public enum DbiFlags implements MaskedFlag { MDB_CREATE(0x4_0000); private final int mask; - private final boolean propagatedToLmdb; - - DbiFlags(final int mask, final boolean propagatedToLmdb) { - this.mask = mask; - this.propagatedToLmdb = propagatedToLmdb; - } DbiFlags(final int mask) { - this(mask, true); + this.mask = mask; } @Override public int getMask() { return mask; } - - @Override - public boolean isPropagatedToLmdb() { - return propagatedToLmdb; - } } diff --git a/src/main/java/org/lmdbjava/DirectBufferProxy.java b/src/main/java/org/lmdbjava/DirectBufferProxy.java index 156e60e9..5022ed02 100644 --- a/src/main/java/org/lmdbjava/DirectBufferProxy.java +++ b/src/main/java/org/lmdbjava/DirectBufferProxy.java @@ -111,12 +111,12 @@ protected DirectBuffer allocate() { } @Override - protected Comparator getSignedComparator() { + public Comparator getSignedComparator() { return signedComparator; } @Override - protected Comparator getUnsignedComparator() { + public Comparator getUnsignedComparator() { return unsignedComparator; } diff --git a/src/main/java/org/lmdbjava/Env.java b/src/main/java/org/lmdbjava/Env.java index 3db16119..1543282c 100644 --- a/src/main/java/org/lmdbjava/Env.java +++ b/src/main/java/org/lmdbjava/Env.java @@ -256,10 +256,17 @@ public Dbi openDbi(final String name, final DbiFlags... flags) { /** * Convenience method that opens a {@link Dbi} with a UTF-8 database name and associated {@link - * Comparator} that is not invoked from native code. + * Comparator} for use by {@link CursorIterable} when comparing start/stop keys. + *

+ * It is very important that the passed comparator behaves in the same way as the comparator + * LMDB uses for its insertion order (for the type of data that will be stored in the database), + * or you fully understand the implications of them behaving differently. + * LMDB's comparator is unsigned lexicographical, unless {@link DbiFlags#MDB_INTEGERKEY} is used. + *

* * @param name name of the database (or null if no name is required) - * @param comparator custom comparator callback (or null to use default) + * @param comparator custom comparator for cursor start/stop key comparisons. If null, + * LMDB's comparator will be used. * @param flags to open the database with * @return a database that is ready to use */ @@ -271,11 +278,15 @@ public Dbi openDbi( /** * Convenience method that opens a {@link Dbi} with a UTF-8 database name and associated {@link - * Comparator} that may be invoked from native code if specified. + * Comparator}. The comparator will be used by {@link CursorIterable} when comparing start/stop keys + * as a minimum. If nativeCb is {@code true}, this comparator will also be called by LMDB to determine + * insertion/iteration order. Calling back to a java comparator may significantly impact performance. * * @param name name of the database (or null if no name is required) - * @param comparator custom comparator callback (or null to use default) - * @param nativeCb whether native code calls back to the Java comparator + * @param comparator custom comparator for cursor start/stop key comparisons and optionally for + * LMDB to call back to. If null, + * LMDB's comparator will be used. + * @param nativeCb whether LMDB native code calls back to the Java comparator * @param flags to open the database with * @return a database that is ready to use */ diff --git a/src/main/java/org/lmdbjava/Key.java b/src/main/java/org/lmdbjava/Key.java new file mode 100644 index 00000000..7fd8bbe2 --- /dev/null +++ b/src/main/java/org/lmdbjava/Key.java @@ -0,0 +1,74 @@ +/* + * Copyright © 2016-2025 The LmdbJava Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lmdbjava; + +import static java.util.Objects.requireNonNull; +import static org.lmdbjava.BufferProxy.MDB_VAL_STRUCT_SIZE; +import static org.lmdbjava.Library.RUNTIME; + +import jnr.ffi.Pointer; +import jnr.ffi.provider.MemoryManager; + +/** + * Represents off-heap memory holding a key only. + * + * @param buffer type + */ +final class Key implements AutoCloseable { + + private static final MemoryManager MEM_MGR = RUNTIME.getMemoryManager(); + private boolean closed; + private T k; + private final BufferProxy proxy; + private final Pointer ptrArray; + private final Pointer ptrKey; + private final long ptrKeyAddr; + + Key(final BufferProxy proxy) { + requireNonNull(proxy); + this.proxy = proxy; + this.k = proxy.allocate(); + ptrKey = MEM_MGR.allocateTemporary(MDB_VAL_STRUCT_SIZE, false); + ptrKeyAddr = ptrKey.address(); + ptrArray = MEM_MGR.allocateTemporary(MDB_VAL_STRUCT_SIZE * 2, false); + } + + @Override + public void close() { + if (closed) { + return; + } + closed = true; + proxy.deallocate(k); + } + + T key() { + return k; + } + + void keyIn(final T key) { + proxy.in(key, ptrKey, ptrKeyAddr); + } + + T keyOut() { + k = proxy.out(k, ptrKey, ptrKeyAddr); + return k; + } + + Pointer pointerKey() { + return ptrKey; + } +} diff --git a/src/main/java/org/lmdbjava/KeyRangeType.java b/src/main/java/org/lmdbjava/KeyRangeType.java index ad67286d..07123e9a 100644 --- a/src/main/java/org/lmdbjava/KeyRangeType.java +++ b/src/main/java/org/lmdbjava/KeyRangeType.java @@ -322,12 +322,12 @@ CursorOp initialOp() { * @param start start buffer * @param stop stop buffer * @param buffer current key returned by LMDB (may be null) - * @param c comparator (required) + * @param rangeComparator comparator (required) * @return response to this key */ > IteratorOp iteratorOp( - final T start, final T stop, final T buffer, final C c) { - requireNonNull(c, "Comparator required"); + final T start, final T stop, final T buffer, final RangeComparator rangeComparator) { + requireNonNull(rangeComparator, "Comparator required"); if (buffer == null) { return TERMINATE; } @@ -337,55 +337,55 @@ > IteratorOp iteratorOp( case FORWARD_AT_LEAST: return RELEASE; case FORWARD_AT_MOST: - return c.compare(buffer, stop) > 0 ? TERMINATE : RELEASE; + return rangeComparator.compareToStopKey() > 0 ? TERMINATE : RELEASE; case FORWARD_CLOSED: - return c.compare(buffer, stop) > 0 ? TERMINATE : RELEASE; + return rangeComparator.compareToStopKey() > 0 ? TERMINATE : RELEASE; case FORWARD_CLOSED_OPEN: - return c.compare(buffer, stop) >= 0 ? TERMINATE : RELEASE; + return rangeComparator.compareToStopKey() >= 0 ? TERMINATE : RELEASE; case FORWARD_GREATER_THAN: - return c.compare(buffer, start) == 0 ? CALL_NEXT_OP : RELEASE; + return rangeComparator.compareToStartKey() == 0 ? CALL_NEXT_OP : RELEASE; case FORWARD_LESS_THAN: - return c.compare(buffer, stop) >= 0 ? TERMINATE : RELEASE; + return rangeComparator.compareToStopKey() >= 0 ? TERMINATE : RELEASE; case FORWARD_OPEN: - if (c.compare(buffer, start) == 0) { + if (rangeComparator.compareToStartKey() == 0) { return CALL_NEXT_OP; } - return c.compare(buffer, stop) >= 0 ? TERMINATE : RELEASE; + return rangeComparator.compareToStopKey() >= 0 ? TERMINATE : RELEASE; case FORWARD_OPEN_CLOSED: - if (c.compare(buffer, start) == 0) { + if (rangeComparator.compareToStartKey() == 0) { return CALL_NEXT_OP; } - return c.compare(buffer, stop) > 0 ? TERMINATE : RELEASE; + return rangeComparator.compareToStopKey() > 0 ? TERMINATE : RELEASE; case BACKWARD_ALL: return RELEASE; case BACKWARD_AT_LEAST: - return c.compare(buffer, start) > 0 ? CALL_NEXT_OP : RELEASE; // rewind + return rangeComparator.compareToStartKey() > 0 ? CALL_NEXT_OP : RELEASE; // rewind case BACKWARD_AT_MOST: - return c.compare(buffer, stop) >= 0 ? RELEASE : TERMINATE; + return rangeComparator.compareToStopKey() >= 0 ? RELEASE : TERMINATE; case BACKWARD_CLOSED: - if (c.compare(buffer, start) > 0) { + if (rangeComparator.compareToStartKey() > 0) { return CALL_NEXT_OP; // rewind } - return c.compare(buffer, stop) >= 0 ? RELEASE : TERMINATE; + return rangeComparator.compareToStopKey() >= 0 ? RELEASE : TERMINATE; case BACKWARD_CLOSED_OPEN: - if (c.compare(buffer, start) > 0) { + if (rangeComparator.compareToStartKey() > 0) { return CALL_NEXT_OP; // rewind } - return c.compare(buffer, stop) > 0 ? RELEASE : TERMINATE; + return rangeComparator.compareToStopKey() > 0 ? RELEASE : TERMINATE; case BACKWARD_GREATER_THAN: - return c.compare(buffer, start) >= 0 ? CALL_NEXT_OP : RELEASE; + return rangeComparator.compareToStartKey() >= 0 ? CALL_NEXT_OP : RELEASE; case BACKWARD_LESS_THAN: - return c.compare(buffer, stop) > 0 ? RELEASE : TERMINATE; + return rangeComparator.compareToStopKey() > 0 ? RELEASE : TERMINATE; case BACKWARD_OPEN: - if (c.compare(buffer, start) >= 0) { + if (rangeComparator.compareToStartKey() >= 0) { return CALL_NEXT_OP; // rewind } - return c.compare(buffer, stop) > 0 ? RELEASE : TERMINATE; + return rangeComparator.compareToStopKey() > 0 ? RELEASE : TERMINATE; case BACKWARD_OPEN_CLOSED: - if (c.compare(buffer, start) >= 0) { + if (rangeComparator.compareToStartKey() >= 0) { return CALL_NEXT_OP; // rewind } - return c.compare(buffer, stop) >= 0 ? RELEASE : TERMINATE; + return rangeComparator.compareToStopKey() >= 0 ? RELEASE : TERMINATE; default: throw new IllegalStateException("Invalid type"); } diff --git a/src/main/java/org/lmdbjava/Library.java b/src/main/java/org/lmdbjava/Library.java index ef9b9b35..6d8122d2 100644 --- a/src/main/java/org/lmdbjava/Library.java +++ b/src/main/java/org/lmdbjava/Library.java @@ -235,6 +235,8 @@ public interface Lmdb { void mdb_txn_reset(@In Pointer txn); + int mdb_cmp(@In Pointer txn, @In Pointer dbi, @In Pointer key1, @In Pointer key2); + Pointer mdb_version(IntByReference major, IntByReference minor, IntByReference patch); } } diff --git a/src/main/java/org/lmdbjava/RangeComparator.java b/src/main/java/org/lmdbjava/RangeComparator.java new file mode 100644 index 00000000..162584b1 --- /dev/null +++ b/src/main/java/org/lmdbjava/RangeComparator.java @@ -0,0 +1,19 @@ +package org.lmdbjava; + +/** + * For comparing a cursor's current key against a {@link KeyRange}'s start/stop key. + */ +interface RangeComparator { + + /** + * Compare the cursor's current key to the range start key. Equivalent to compareTo(currentKey, + * startKey) + */ + int compareToStartKey(); + + /** + * Compare the cursor's current key to the range stop key. Equivalent to compareTo(currentKey, + * stopKey) + */ + int compareToStopKey(); +} diff --git a/src/test/java/org/lmdbjava/ComparatorTest.java b/src/test/java/org/lmdbjava/ComparatorTest.java index 3e265cee..dad84ed2 100644 --- a/src/test/java/org/lmdbjava/ComparatorTest.java +++ b/src/test/java/org/lmdbjava/ComparatorTest.java @@ -135,7 +135,7 @@ private static final class ByteArrayRunner implements ComparatorRunner { @Override public int compare(final byte[] o1, final byte[] o2) { - final Comparator c = PROXY_BA.getComparator(); + final Comparator c = PROXY_BA.getUnsignedComparator(); return c.compare(o1, o2); } } @@ -145,7 +145,7 @@ private static final class ByteBufferRunner implements ComparatorRunner { @Override public int compare(final byte[] o1, final byte[] o2) { - final Comparator c = PROXY_OPTIMAL.getComparator(); + final Comparator c = PROXY_OPTIMAL.getUnsignedComparator(); // Convert arrays to buffers that are larger than the array, with // limit set at the array length. One buffer bigger than the other. @@ -189,7 +189,7 @@ private static final class DirectBufferRunner implements ComparatorRunner { public int compare(final byte[] o1, final byte[] o2) { final DirectBuffer o1b = new UnsafeBuffer(o1); final DirectBuffer o2b = new UnsafeBuffer(o2); - final Comparator c = PROXY_DB.getComparator(); + final Comparator c = PROXY_DB.getUnsignedComparator(); return c.compare(o1b, o2b); } } @@ -223,7 +223,7 @@ public int compare(final byte[] o1, final byte[] o2) { final ByteBuf o2b = DEFAULT.directBuffer(o2.length); o1b.writeBytes(o1); o2b.writeBytes(o2); - final Comparator c = PROXY_NETTY.getComparator(); + final Comparator c = PROXY_NETTY.getUnsignedComparator(); return c.compare(o1b, o2b); } } diff --git a/src/test/java/org/lmdbjava/CursorIterablePerfTest.java b/src/test/java/org/lmdbjava/CursorIterablePerfTest.java new file mode 100644 index 00000000..a9647d56 --- /dev/null +++ b/src/test/java/org/lmdbjava/CursorIterablePerfTest.java @@ -0,0 +1,163 @@ +package org.lmdbjava; + +import static com.jakewharton.byteunits.BinaryByteUnit.GIBIBYTES; +import static org.lmdbjava.DbiFlags.MDB_CREATE; +import static org.lmdbjava.Env.create; +import static org.lmdbjava.EnvFlags.MDB_NOSUBDIR; +import static org.lmdbjava.PutFlags.MDB_APPEND; +import static org.lmdbjava.PutFlags.MDB_NOOVERWRITE; +import static org.lmdbjava.TestUtils.POSIX_MODE; +import static org.lmdbjava.TestUtils.bb; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class CursorIterablePerfTest { + + @Rule + public final TemporaryFolder tmp = new TemporaryFolder(); + +// private static final int ITERATIONS = 5_000_000; + private static final int ITERATIONS = 100_000; +// private static final int ITERATIONS = 10; + + private Dbi dbJavaComparator; + private Dbi dbLmdbComparator; + private Dbi dbCallbackComparator; + private List> dbs = new ArrayList<>(); + private Env env; + private List data = new ArrayList<>(ITERATIONS); + + @Before + public void before() throws IOException { + final File path = tmp.newFile(); + final BufferProxy bufferProxy = ByteBufferProxy.PROXY_OPTIMAL; + env = + create(bufferProxy) + .setMapSize(GIBIBYTES.toBytes(1)) + .setMaxReaders(1) + .setMaxDbs(3) + .open(path, POSIX_MODE, MDB_NOSUBDIR); + + // Use a java comparator for start/stop keys only + dbJavaComparator = env.openDbi("JavaComparator", bufferProxy.getUnsignedComparator(), MDB_CREATE); + // Use LMDB comparator for start/stop keys + dbLmdbComparator = env.openDbi("LmdbComparator", MDB_CREATE); + // Use a java comparator for start/stop keys and as a callback comparator + dbCallbackComparator = env.openDbi( + "CallBackComparator", bufferProxy.getUnsignedComparator(), true, MDB_CREATE); + + dbs.add(dbJavaComparator); + dbs.add(dbLmdbComparator); + dbs.add(dbCallbackComparator); + + populateList(); + } + + private void populateList() { + for (int i = 0; i < ITERATIONS * 2; i+=2) { + data.add(i); + } + } + + private void populateDatabases(final boolean randomOrder) { + System.out.println("Clear then populate databases"); + + final List data; + if (randomOrder) { + data = new ArrayList<>(this.data); + Collections.shuffle(data); + } else { + data = this.data; + } + + for (int round = 0; round < 3; round++) { + System.out.println("round: " + round + " -----------------------------------------"); + + for (final Dbi db : dbs) { + // Clean out the db first + try (Txn txn = env.txnWrite(); + final Cursor cursor = db.openCursor(txn)) { + while (cursor.next()) { + cursor.delete(); + } + } + + final String dbName = new String(db.getName(), StandardCharsets.UTF_8); + final Instant start = Instant.now(); + try (Txn txn = env.txnWrite()) { + for (final Integer i : data) { + if (randomOrder) { + db.put(txn, bb(i), bb(i + 1), MDB_NOOVERWRITE); + } else { + db.put(txn, bb(i), bb(i + 1), MDB_NOOVERWRITE, MDB_APPEND); + } + } + txn.commit(); + } + final Duration duration = Duration.between(start, Instant.now()); + System.out.println("DB: " + dbName + + " - Loaded in duration: " + duration + + ", millis: " + duration.toMillis()); + } + } + } + + @After + public void after() { + env.close(); + tmp.delete(); + } + + @Test + public void comparePerf_sequential() { + comparePerf(false); + } + + @Test + public void comparePerf_random() { + comparePerf(true); + } + + public void comparePerf(final boolean randomOrder) { + populateDatabases(randomOrder); + final ByteBuffer startKeyBuf = bb(data.getFirst()); + final ByteBuffer stopKeyBuf = bb(data.getLast()); + final KeyRange keyRange = KeyRange.closed(startKeyBuf, stopKeyBuf); + + System.out.println("\nIterating over all entries"); + for (int round = 0; round < 3; round++) { + System.out.println("round: " + round + " -----------------------------------------"); + for (final Dbi db : dbs) { + final String dbName = new String(db.getName(), StandardCharsets.UTF_8); + + final Instant start = Instant.now(); + int cnt = 0; + // Exercise the stop key comparator on every entry + try (Txn txn = env.txnRead(); + CursorIterable c = db.iterate(txn, keyRange)) { + for (final CursorIterable.KeyVal kv : c) { + cnt++; + } + } + final Duration duration = Duration.between(start, Instant.now()); + System.out.println("DB: " + dbName + + " - Iterated in duration: " + duration + + ", millis: " + duration.toMillis() + + ", cnt: " + cnt); + } + } + } +} diff --git a/src/test/java/org/lmdbjava/CursorIterableTest.java b/src/test/java/org/lmdbjava/CursorIterableTest.java index bd23bc55..96be0816 100644 --- a/src/test/java/org/lmdbjava/CursorIterableTest.java +++ b/src/test/java/org/lmdbjava/CursorIterableTest.java @@ -43,6 +43,8 @@ import static org.lmdbjava.KeyRange.openClosedBackward; import static org.lmdbjava.PutFlags.MDB_NOOVERWRITE; import static org.lmdbjava.TestUtils.DB_1; +import static org.lmdbjava.TestUtils.DB_2; +import static org.lmdbjava.TestUtils.DB_3; import static org.lmdbjava.TestUtils.POSIX_MODE; import static org.lmdbjava.TestUtils.bb; @@ -69,7 +71,10 @@ public final class CursorIterableTest { @Rule public final TemporaryFolder tmp = new TemporaryFolder(); - private Dbi db; + private Dbi dbJavaComparator; + private Dbi dbLmdbComparator; + private Dbi dbCallbackComparator; + private List> dbs = new ArrayList<>(); private Env env; private Deque list; @@ -116,19 +121,39 @@ public void atMostTest() { @Before public void before() throws IOException { final File path = tmp.newFile(); + final BufferProxy bufferProxy = ByteBufferProxy.PROXY_OPTIMAL; env = - create() + create(bufferProxy) .setMapSize(KIBIBYTES.toBytes(256)) .setMaxReaders(1) - .setMaxDbs(1) + .setMaxDbs(3) .open(path, POSIX_MODE, MDB_NOSUBDIR); - db = env.openDbi(DB_1, MDB_CREATE); - populateDatabase(db); + + // Use a java comparator for start/stop keys only + dbJavaComparator = env.openDbi(DB_1, bufferProxy.getUnsignedComparator(), MDB_CREATE); + // Use LMDB comparator for start/stop keys + dbLmdbComparator = env.openDbi(DB_2, MDB_CREATE); + // Use a java comparator for start/stop keys and as a callback comparaotr + dbCallbackComparator = env.openDbi( + DB_3, bufferProxy.getUnsignedComparator(), true, MDB_CREATE); + + populateList(); + + populateDatabase(dbJavaComparator); + populateDatabase(dbLmdbComparator); + populateDatabase(dbCallbackComparator); + + dbs.add(dbJavaComparator); + dbs.add(dbLmdbComparator); + dbs.add(dbCallbackComparator); } - private void populateDatabase(final Dbi dbi) { + private void populateList() { list = new LinkedList<>(); list.addAll(asList(2, 3, 4, 5, 6, 7, 8, 9)); + } + + private void populateDatabase(final Dbi dbi) { try (Txn txn = env.txnWrite()) { final Cursor c = dbi.openCursor(txn); c.put(bb(2), bb(3), MDB_NOOVERWRITE); @@ -166,6 +191,14 @@ public void closedTest() { verify(closed(bb(1), bb(7)), 2, 4, 6); } + public void closedTest1() { + verify(dbLmdbComparator, closed(bb(3), bb(7)), 4, 6); + } + + public void closedTest2() { + verify(dbJavaComparator, closed(bb(3), bb(7)), 4, 6); + } + @Test public void greaterThanBackwardTest() { verify(greaterThanBackward(bb(6)), 4, 2); @@ -181,30 +214,39 @@ public void greaterThanTest() { @Test(expected = IllegalStateException.class) public void iterableOnlyReturnedOnce() { - try (Txn txn = env.txnRead(); - CursorIterable c = db.iterate(txn)) { - c.iterator(); // ok - c.iterator(); // fails + for (final Dbi db : dbs) { + try (Txn txn = env.txnRead(); + CursorIterable c = db.iterate(txn)) { + c.iterator(); // ok + c.iterator(); // fails + } } } @Test public void iterate() { - try (Txn txn = env.txnRead(); - CursorIterable c = db.iterate(txn)) { - for (final KeyVal kv : c) { - assertThat(kv.key().getInt(), is(list.pollFirst())); - assertThat(kv.val().getInt(), is(list.pollFirst())); + for (final Dbi db : dbs) { + populateList(); + try (Txn txn = env.txnRead(); + CursorIterable c = db.iterate(txn)) { + + int cnt = 0; + for (final KeyVal kv : c) { + assertThat(kv.key().getInt(), is(list.pollFirst())); + assertThat(kv.val().getInt(), is(list.pollFirst())); + } } } } @Test(expected = IllegalStateException.class) public void iteratorOnlyReturnedOnce() { - try (Txn txn = env.txnRead(); - CursorIterable c = db.iterate(txn)) { - c.iterator(); // ok - c.iterator(); // fails + for (final Dbi db : dbs) { + try (Txn txn = env.txnRead(); + CursorIterable c = db.iterate(txn)) { + c.iterator(); // ok + c.iterator(); // fails + } } } @@ -222,16 +264,19 @@ public void lessThanTest() { @Test(expected = NoSuchElementException.class) public void nextThrowsNoSuchElementExceptionIfNoMoreElements() { - try (Txn txn = env.txnRead(); - CursorIterable c = db.iterate(txn)) { - final Iterator> i = c.iterator(); - while (i.hasNext()) { - final KeyVal kv = i.next(); - assertThat(kv.key().getInt(), is(list.pollFirst())); - assertThat(kv.val().getInt(), is(list.pollFirst())); + for (final Dbi db : dbs) { + populateList(); + try (Txn txn = env.txnRead(); + CursorIterable c = db.iterate(txn)) { + final Iterator> i = c.iterator(); + while (i.hasNext()) { + final KeyVal kv = i.next(); + assertThat(kv.key().getInt(), is(list.pollFirst())); + assertThat(kv.val().getInt(), is(list.pollFirst())); + } + assertThat(i.hasNext(), is(false)); + i.next(); } - assertThat(i.hasNext(), is(false)); - i.next(); } } @@ -284,81 +329,148 @@ public void openTest() { @Test public void removeOddElements() { - verify(all(), 2, 4, 6, 8); - int idx = -1; - try (Txn txn = env.txnWrite()) { - try (CursorIterable ci = db.iterate(txn)) { - final Iterator> c = ci.iterator(); - while (c.hasNext()) { - c.next(); - idx++; - if (idx % 2 == 0) { - c.remove(); + for (final Dbi db : dbs) { + verify(db, all(), 2, 4, 6, 8); + int idx = -1; + try (Txn txn = env.txnWrite()) { + try (CursorIterable ci = db.iterate(txn)) { + final Iterator> c = ci.iterator(); + while (c.hasNext()) { + c.next(); + idx++; + if (idx % 2 == 0) { + c.remove(); + } } } + txn.commit(); } - txn.commit(); + verify(db, all(), 4, 8); } - verify(all(), 4, 8); } @Test(expected = Env.AlreadyClosedException.class) public void nextWithClosedEnvTest() { - try (Txn txn = env.txnRead()) { - try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { - final Iterator> c = ci.iterator(); + for (final Dbi db : dbs) { + try (Txn txn = env.txnRead()) { + try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { + final Iterator> c = ci.iterator(); - env.close(); - c.next(); + env.close(); + c.next(); + } } } } @Test(expected = Env.AlreadyClosedException.class) public void removeWithClosedEnvTest() { - try (Txn txn = env.txnWrite()) { - try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { - final Iterator> c = ci.iterator(); + for (final Dbi db : dbs) { + try (Txn txn = env.txnWrite()) { + try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { + final Iterator> c = ci.iterator(); - final KeyVal keyVal = c.next(); - assertThat(keyVal, Matchers.notNullValue()); + final KeyVal keyVal = c.next(); + assertThat(keyVal, Matchers.notNullValue()); - env.close(); - c.remove(); + env.close(); + c.remove(); + } } } } @Test(expected = Env.AlreadyClosedException.class) public void hasNextWithClosedEnvTest() { - try (Txn txn = env.txnRead()) { - try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { - final Iterator> c = ci.iterator(); + for (final Dbi db : dbs) { + try (Txn txn = env.txnRead()) { + try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { + final Iterator> c = ci.iterator(); - env.close(); - c.hasNext(); + env.close(); + c.hasNext(); + } } } } @Test(expected = Env.AlreadyClosedException.class) public void forEachRemainingWithClosedEnvTest() { - try (Txn txn = env.txnRead()) { - try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { - final Iterator> c = ci.iterator(); + for (final Dbi db : dbs) { + try (Txn txn = env.txnRead()) { + try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { + final Iterator> c = ci.iterator(); - env.close(); - c.forEachRemaining(keyVal -> {}); + env.close(); + c.forEachRemaining(keyVal -> {}); + } + } + } + } + + @Test + public void testSignedVsUnsigned() { + final ByteBuffer val1 = bb(1); + final ByteBuffer val2 = bb(2); + final ByteBuffer val110 = bb(110); + final ByteBuffer val111 = bb(111); + final ByteBuffer val150 = bb(150); + + final BufferProxy bufferProxy = ByteBufferProxy.PROXY_OPTIMAL; + final Comparator unsignedComparator = bufferProxy.getUnsignedComparator(); + final Comparator signedComparator = bufferProxy.getSignedComparator(); + + // Compare the same + assertThat( + unsignedComparator.compare(val1, val2), + Matchers.is(signedComparator.compare(val1, val2))); + + // Compare differently + assertThat( + unsignedComparator.compare(val110, val150), + Matchers.not(signedComparator.compare(val110, val150))); + + // Compare differently + assertThat( + unsignedComparator.compare(val111, val150), + Matchers.not(signedComparator.compare(val111, val150))); + + // This will fail if the db is using a signed comparator for the start/stop keys + for (final Dbi db : dbs) { + db.put(val110, val110); + db.put(val150, val150); + + final ByteBuffer startKeyBuf = val111; + KeyRange keyRange = KeyRange.atLeastBackward(startKeyBuf); + + try (Txn txn = env.txnRead(); + CursorIterable c = db.iterate(txn, keyRange)) { + for (final CursorIterable.KeyVal kv : c) { + final int key = kv.key().getInt(); + final int val = kv.val().getInt(); +// System.out.println("key: " + key + " val: " + val); + assertThat(key, is(110)); + break; + } } } } private void verify(final KeyRange range, final int... expected) { - verify(range, db, expected); + // Verify using all comparator types + for (final Dbi db : dbs) { + verify(range, db, expected); + } + } + + private void verify( + final Dbi dbi, final KeyRange range, final int... expected) { + verify(range, dbi, expected); } private void verify( final KeyRange range, final Dbi dbi, final int... expected) { + final List results = new ArrayList<>(); try (Txn txn = env.txnRead(); diff --git a/src/test/java/org/lmdbjava/DbiTest.java b/src/test/java/org/lmdbjava/DbiTest.java index 1fa80f6e..9c5cdb2e 100644 --- a/src/test/java/org/lmdbjava/DbiTest.java +++ b/src/test/java/org/lmdbjava/DbiTest.java @@ -111,7 +111,7 @@ public void close() { public void customComparator() { final Comparator reverseOrder = (o1, o2) -> { - final int lexical = PROXY_OPTIMAL.getComparator().compare(o1, o2); + final int lexical = PROXY_OPTIMAL.getUnsignedComparator().compare(o1, o2); if (lexical == 0) { return 0; } @@ -144,7 +144,7 @@ public void dbOpenMaxDatabases() { @Test public void dbiWithComparatorThreadSafety() { final DbiFlags[] flags = new DbiFlags[] {MDB_CREATE, MDB_INTEGERKEY}; - final Comparator c = PROXY_OPTIMAL.getComparator(flags); + final Comparator c = PROXY_OPTIMAL.getUnsignedComparator(); final Dbi db = env.openDbi(DB_1, c, true, flags); final List keys = range(0, 1_000).boxed().collect(toList()); diff --git a/src/test/java/org/lmdbjava/KeyRangeTest.java b/src/test/java/org/lmdbjava/KeyRangeTest.java index 6e104bbf..0197bf11 100644 --- a/src/test/java/org/lmdbjava/KeyRangeTest.java +++ b/src/test/java/org/lmdbjava/KeyRangeTest.java @@ -195,7 +195,10 @@ private void verify(final KeyRange range, final int... expected) { IteratorOp op; do { - op = range.getType().iteratorOp(range.getStart(), range.getStop(), buff, Integer::compare); + final Integer finalBuff = buff; + final RangeComparator rangeComparator = + CursorIterable.createJavaRangeComparator(range, Integer::compareTo, () -> finalBuff); + op = range.getType().iteratorOp(range.getStart(), range.getStop(), buff, rangeComparator); switch (op) { case CALL_NEXT_OP: buff = cursor.apply(range.getType().nextOp(), range.getStart()); diff --git a/src/test/java/org/lmdbjava/TestUtils.java b/src/test/java/org/lmdbjava/TestUtils.java index 42dcf052..f3d3974b 100644 --- a/src/test/java/org/lmdbjava/TestUtils.java +++ b/src/test/java/org/lmdbjava/TestUtils.java @@ -30,6 +30,8 @@ final class TestUtils { public static final String DB_1 = "test-db-1"; + public static final String DB_2 = "test-db-2"; + public static final String DB_3 = "test-db-3"; public static final int POSIX_MODE = 0664; From 46f8d08194d989529f87b447f48fad0214c10e80 Mon Sep 17 00:00:00 2001 From: at055612 <22818309+at055612@users.noreply.github.com> Date: Thu, 6 Mar 2025 17:27:25 +0000 Subject: [PATCH 02/21] gh-249 Remove non-J8 features, use indent size 2 --- .../org/lmdbjava/CursorIterablePerfTest.java | 220 +++++++++--------- 1 file changed, 110 insertions(+), 110 deletions(-) diff --git a/src/test/java/org/lmdbjava/CursorIterablePerfTest.java b/src/test/java/org/lmdbjava/CursorIterablePerfTest.java index a9647d56..ab94f85f 100644 --- a/src/test/java/org/lmdbjava/CursorIterablePerfTest.java +++ b/src/test/java/org/lmdbjava/CursorIterablePerfTest.java @@ -29,135 +29,135 @@ public class CursorIterablePerfTest { @Rule public final TemporaryFolder tmp = new TemporaryFolder(); -// private static final int ITERATIONS = 5_000_000; - private static final int ITERATIONS = 100_000; -// private static final int ITERATIONS = 10; - - private Dbi dbJavaComparator; - private Dbi dbLmdbComparator; - private Dbi dbCallbackComparator; - private List> dbs = new ArrayList<>(); - private Env env; - private List data = new ArrayList<>(ITERATIONS); - - @Before - public void before() throws IOException { - final File path = tmp.newFile(); - final BufferProxy bufferProxy = ByteBufferProxy.PROXY_OPTIMAL; - env = - create(bufferProxy) - .setMapSize(GIBIBYTES.toBytes(1)) - .setMaxReaders(1) - .setMaxDbs(3) - .open(path, POSIX_MODE, MDB_NOSUBDIR); - - // Use a java comparator for start/stop keys only + // private static final int ITERATIONS = 5_000_000; + private static final int ITERATIONS = 100_000; + // private static final int ITERATIONS = 10; + + private Dbi dbJavaComparator; + private Dbi dbLmdbComparator; + private Dbi dbCallbackComparator; + private List> dbs = new ArrayList<>(); + private Env env; + private List data = new ArrayList<>(ITERATIONS); + + @Before + public void before() throws IOException { + final File path = tmp.newFile(); + final BufferProxy bufferProxy = ByteBufferProxy.PROXY_OPTIMAL; + env = + create(bufferProxy) + .setMapSize(GIBIBYTES.toBytes(1)) + .setMaxReaders(1) + .setMaxDbs(3) + .open(path, POSIX_MODE, MDB_NOSUBDIR); + + // Use a java comparator for start/stop keys only dbJavaComparator = env.openDbi("JavaComparator", bufferProxy.getUnsignedComparator(), MDB_CREATE); - // Use LMDB comparator for start/stop keys - dbLmdbComparator = env.openDbi("LmdbComparator", MDB_CREATE); - // Use a java comparator for start/stop keys and as a callback comparator + // Use LMDB comparator for start/stop keys + dbLmdbComparator = env.openDbi("LmdbComparator", MDB_CREATE); + // Use a java comparator for start/stop keys and as a callback comparator dbCallbackComparator = env.openDbi( "CallBackComparator", bufferProxy.getUnsignedComparator(), true, MDB_CREATE); - dbs.add(dbJavaComparator); - dbs.add(dbLmdbComparator); - dbs.add(dbCallbackComparator); + dbs.add(dbJavaComparator); + dbs.add(dbLmdbComparator); + dbs.add(dbCallbackComparator); - populateList(); + populateList(); + } + + private void populateList() { + for (int i = 0; i < ITERATIONS * 2; i += 2) { + data.add(i); } + } - private void populateList() { - for (int i = 0; i < ITERATIONS * 2; i+=2) { - data.add(i); - } + private void populateDatabases(final boolean randomOrder) { + System.out.println("Clear then populate databases"); + + final List data; + if (randomOrder) { + data = new ArrayList<>(this.data); + Collections.shuffle(data); + } else { + data = this.data; } - private void populateDatabases(final boolean randomOrder) { - System.out.println("Clear then populate databases"); + for (int round = 0; round < 3; round++) { + System.out.println("round: " + round + " -----------------------------------------"); - final List data; - if (randomOrder) { - data = new ArrayList<>(this.data); - Collections.shuffle(data); - } else { - data = this.data; + for (final Dbi db : dbs) { + // Clean out the db first + try (Txn txn = env.txnWrite(); + final Cursor cursor = db.openCursor(txn)) { + while (cursor.next()) { + cursor.delete(); + } } - for (int round = 0; round < 3; round++) { - System.out.println("round: " + round + " -----------------------------------------"); - - for (final Dbi db : dbs) { - // Clean out the db first - try (Txn txn = env.txnWrite(); - final Cursor cursor = db.openCursor(txn)) { - while (cursor.next()) { - cursor.delete(); - } - } - - final String dbName = new String(db.getName(), StandardCharsets.UTF_8); - final Instant start = Instant.now(); - try (Txn txn = env.txnWrite()) { - for (final Integer i : data) { - if (randomOrder) { - db.put(txn, bb(i), bb(i + 1), MDB_NOOVERWRITE); - } else { - db.put(txn, bb(i), bb(i + 1), MDB_NOOVERWRITE, MDB_APPEND); - } - } - txn.commit(); - } - final Duration duration = Duration.between(start, Instant.now()); + final String dbName = new String(db.getName(), StandardCharsets.UTF_8); + final Instant start = Instant.now(); + try (Txn txn = env.txnWrite()) { + for (final Integer i : data) { + if (randomOrder) { + db.put(txn, bb(i), bb(i + 1), MDB_NOOVERWRITE); + } else { + db.put(txn, bb(i), bb(i + 1), MDB_NOOVERWRITE, MDB_APPEND); + } + } + txn.commit(); + } + final Duration duration = Duration.between(start, Instant.now()); System.out.println("DB: " + dbName + " - Loaded in duration: " + duration + ", millis: " + duration.toMillis()); - } - } - } - - @After - public void after() { - env.close(); - tmp.delete(); + } } - - @Test - public void comparePerf_sequential() { - comparePerf(false); - } - - @Test - public void comparePerf_random() { - comparePerf(true); - } - - public void comparePerf(final boolean randomOrder) { - populateDatabases(randomOrder); - final ByteBuffer startKeyBuf = bb(data.getFirst()); - final ByteBuffer stopKeyBuf = bb(data.getLast()); - final KeyRange keyRange = KeyRange.closed(startKeyBuf, stopKeyBuf); - - System.out.println("\nIterating over all entries"); - for (int round = 0; round < 3; round++) { - System.out.println("round: " + round + " -----------------------------------------"); - for (final Dbi db : dbs) { - final String dbName = new String(db.getName(), StandardCharsets.UTF_8); - - final Instant start = Instant.now(); - int cnt = 0; - // Exercise the stop key comparator on every entry - try (Txn txn = env.txnRead(); - CursorIterable c = db.iterate(txn, keyRange)) { - for (final CursorIterable.KeyVal kv : c) { - cnt++; - } - } - final Duration duration = Duration.between(start, Instant.now()); + } + + @After + public void after() { + env.close(); + tmp.delete(); + } + + @Test + public void comparePerf_sequential() { + comparePerf(false); + } + + @Test + public void comparePerf_random() { + comparePerf(true); + } + + public void comparePerf(final boolean randomOrder) { + populateDatabases(randomOrder); + final ByteBuffer startKeyBuf = bb(data.get(0)); + final ByteBuffer stopKeyBuf = bb(data.get(data.size() - 1)); + final KeyRange keyRange = KeyRange.closed(startKeyBuf, stopKeyBuf); + + System.out.println("\nIterating over all entries"); + for (int round = 0; round < 3; round++) { + System.out.println("round: " + round + " -----------------------------------------"); + for (final Dbi db : dbs) { + final String dbName = new String(db.getName(), StandardCharsets.UTF_8); + + final Instant start = Instant.now(); + int cnt = 0; + // Exercise the stop key comparator on every entry + try (Txn txn = env.txnRead(); + CursorIterable c = db.iterate(txn, keyRange)) { + for (final CursorIterable.KeyVal kv : c) { + cnt++; + } + } + final Duration duration = Duration.between(start, Instant.now()); System.out.println("DB: " + dbName + " - Iterated in duration: " + duration + ", millis: " + duration.toMillis() + ", cnt: " + cnt); - } - } + } } + } } From f92012ecc079149b2414925e0a077a75f82ba043 Mon Sep 17 00:00:00 2001 From: at055612 <22818309+at055612@users.noreply.github.com> Date: Thu, 6 Mar 2025 17:33:40 +0000 Subject: [PATCH 03/21] gh-249 Fix indents --- .../org/lmdbjava/CursorIterablePerfTest.java | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/test/java/org/lmdbjava/CursorIterablePerfTest.java b/src/test/java/org/lmdbjava/CursorIterablePerfTest.java index ab94f85f..99667b1d 100644 --- a/src/test/java/org/lmdbjava/CursorIterablePerfTest.java +++ b/src/test/java/org/lmdbjava/CursorIterablePerfTest.java @@ -26,8 +26,8 @@ public class CursorIterablePerfTest { - @Rule - public final TemporaryFolder tmp = new TemporaryFolder(); + @Rule + public final TemporaryFolder tmp = new TemporaryFolder(); // private static final int ITERATIONS = 5_000_000; private static final int ITERATIONS = 100_000; @@ -52,12 +52,12 @@ public void before() throws IOException { .open(path, POSIX_MODE, MDB_NOSUBDIR); // Use a java comparator for start/stop keys only - dbJavaComparator = env.openDbi("JavaComparator", bufferProxy.getUnsignedComparator(), MDB_CREATE); + dbJavaComparator = env.openDbi("JavaComparator", bufferProxy.getUnsignedComparator(), MDB_CREATE); // Use LMDB comparator for start/stop keys dbLmdbComparator = env.openDbi("LmdbComparator", MDB_CREATE); // Use a java comparator for start/stop keys and as a callback comparator - dbCallbackComparator = env.openDbi( - "CallBackComparator", bufferProxy.getUnsignedComparator(), true, MDB_CREATE); + dbCallbackComparator = env.openDbi( + "CallBackComparator", bufferProxy.getUnsignedComparator(), true, MDB_CREATE); dbs.add(dbJavaComparator); dbs.add(dbLmdbComparator); @@ -89,7 +89,7 @@ private void populateDatabases(final boolean randomOrder) { for (final Dbi db : dbs) { // Clean out the db first try (Txn txn = env.txnWrite(); - final Cursor cursor = db.openCursor(txn)) { + final Cursor cursor = db.openCursor(txn)) { while (cursor.next()) { cursor.delete(); } @@ -108,9 +108,9 @@ private void populateDatabases(final boolean randomOrder) { txn.commit(); } final Duration duration = Duration.between(start, Instant.now()); - System.out.println("DB: " + dbName - + " - Loaded in duration: " + duration - + ", millis: " + duration.toMillis()); + System.out.println("DB: " + dbName + + " - Loaded in duration: " + duration + + ", millis: " + duration.toMillis()); } } } @@ -147,16 +147,16 @@ public void comparePerf(final boolean randomOrder) { int cnt = 0; // Exercise the stop key comparator on every entry try (Txn txn = env.txnRead(); - CursorIterable c = db.iterate(txn, keyRange)) { + CursorIterable c = db.iterate(txn, keyRange)) { for (final CursorIterable.KeyVal kv : c) { cnt++; } } final Duration duration = Duration.between(start, Instant.now()); - System.out.println("DB: " + dbName - + " - Iterated in duration: " + duration - + ", millis: " + duration.toMillis() - + ", cnt: " + cnt); + System.out.println("DB: " + dbName + + " - Iterated in duration: " + duration + + ", millis: " + duration.toMillis() + + ", cnt: " + cnt); } } } From e1756d633d7cbd1d23a33ffa0fcaaaa44c06aacd Mon Sep 17 00:00:00 2001 From: at055612 <22818309+at055612@users.noreply.github.com> Date: Thu, 6 Mar 2025 19:02:46 +0000 Subject: [PATCH 04/21] gh-249 Tidy code and refactor RangeComparator impls --- src/main/java/org/lmdbjava/BufferProxy.java | 8 +- .../java/org/lmdbjava/CursorIterable.java | 544 ++++++++++-------- src/main/java/org/lmdbjava/Env.java | 23 +- src/main/java/org/lmdbjava/Key.java | 4 +- src/main/java/org/lmdbjava/KeyRangeType.java | 4 +- .../java/org/lmdbjava/RangeComparator.java | 26 +- .../org/lmdbjava/CursorIterablePerfTest.java | 37 +- .../java/org/lmdbjava/CursorIterableTest.java | 18 +- src/test/java/org/lmdbjava/KeyRangeTest.java | 4 +- 9 files changed, 376 insertions(+), 292 deletions(-) diff --git a/src/main/java/org/lmdbjava/BufferProxy.java b/src/main/java/org/lmdbjava/BufferProxy.java index 26d9db74..ab7ba3a4 100644 --- a/src/main/java/org/lmdbjava/BufferProxy.java +++ b/src/main/java/org/lmdbjava/BufferProxy.java @@ -69,11 +69,9 @@ protected BufferProxy() {} /** * Get a suitable default {@link Comparator} to compare numeric key values as signed. * - *

- * Note: LMDB's default comparator is unsigned so if this is used only for the {@link CursorIterable} - * start/stop key comparisons then its behaviour will differ from the iteration order. Use - * with caution. - *

+ *

Note: LMDB's default comparator is unsigned so if this is used only for the {@link + * CursorIterable} start/stop key comparisons then its behaviour will differ from the iteration + * order. Use with caution. * * @return a comparator that can be used (never null) */ diff --git a/src/main/java/org/lmdbjava/CursorIterable.java b/src/main/java/org/lmdbjava/CursorIterable.java index 6b92a9cd..7c487bae 100644 --- a/src/main/java/org/lmdbjava/CursorIterable.java +++ b/src/main/java/org/lmdbjava/CursorIterable.java @@ -42,266 +42,352 @@ */ public final class CursorIterable implements Iterable>, AutoCloseable { - // private final Comparator comparator; - private final RangeComparator rangeComparator; - private final Cursor cursor; - private final Dbi dbi; - private final KeyVal entry; - private boolean iteratorReturned; - private final KeyRange range; - private State state = REQUIRES_INITIAL_OP; - private final Key startKey; - private final Key stopKey; + // private final Comparator comparator; + private final RangeComparator rangeComparator; + private final Cursor cursor; + private final KeyVal entry; + private boolean iteratorReturned; + private final KeyRange range; + private State state = REQUIRES_INITIAL_OP; - CursorIterable( - final Txn txn, - final Dbi dbi, - final KeyRange range, - final Comparator comparator, - final BufferProxy proxy) { - this.cursor = dbi.openCursor(txn); - this.dbi = dbi; - this.range = range; - this.entry = new KeyVal<>(); - - if (comparator != null) { - // User supplied java-side comparator so use that - this.rangeComparator = createJavaRangeComparator(range, comparator, entry::key); - this.startKey = null; - this.stopKey = null; - } else { - // No java-side comparator so call down to LMDB to do the comparison - this.rangeComparator = createLmdbDbiComparator(txn.pointer(), dbi.pointer()); - // Allocate buffers for use with the start/stop keys if required. - // Saves us copying bytes on each comparison - this.startKey = createKey(range.getStart(), proxy); - this.stopKey = createKey(range.getStop(), proxy); - } + CursorIterable( + final Txn txn, + final Dbi dbi, + final KeyRange range, + final Comparator comparator, + final BufferProxy proxy) { + this.cursor = dbi.openCursor(txn); + this.range = range; + this.entry = new KeyVal<>(); + + if (comparator != null) { + // User supplied java-side comparator so use that + this.rangeComparator = new JavaRangeComparator<>(range, comparator, entry::key); + } else { + // No java-side comparator so call down to LMDB to do the comparison + this.rangeComparator = new LmdbRangeComparator<>(txn, dbi, cursor, range, proxy); } + } - private Key createKey(final T keyBuffer, final BufferProxy proxy) { - if (keyBuffer != null) { - final Key key = proxy.key(); - key.keyIn(keyBuffer); - return key; - } else { - return null; + // static RangeComparator createJavaRangeComparator( + // final KeyRange range, + // final Comparator comparator, + // final Supplier currentKeySupplier) { + // final T start = range.getStart(); + // final T stop = range.getStop(); + // return new RangeComparator() { + // @Override + // public int compareToStartKey() { + // return comparator.compare(currentKeySupplier.get(), start); + // } + // + // @Override + // public int compareToStopKey() { + // return comparator.compare(currentKeySupplier.get(), stop); + // } + // }; + // } + + // /** + // * Calls down to mdb_cmp to make use of the comparator that LMDB uses for insertion order. + // * + // * @param txnPointer The pointer to the transaction. + // * @param dbiPointer The pointer to the Dbi so LMDB can use the comparator of the Dbi + // * @param proxy + // */ + // private RangeComparator createLmdbDbiComparator(final Pointer txnPointer, + // final Pointer dbiPointer, + // final Pointer cursorKeyPointer, + // final KeyRange range, + // final BufferProxy proxy) { + // Objects.requireNonNull(txnPointer); + // Objects.requireNonNull(dbiPointer); + // Objects.requireNonNull(range); + // Objects.requireNonNull(cursor); + // // Allocate buffers for use with the start/stop keys if required. + // // Saves us copying bytes on each comparison + // final Key startKey = createKey(range.getStart(), proxy); + // final Key stopKey = createKey(range.getStop(), proxy); + // + // return new RangeComparator() { + // @Override + // public int compareToStartKey() { + // return LIB.mdb_cmp(txnPointer, dbiPointer, cursor.keyVal().pointerKey(), + // startKey.pointer()); + // } + // + // @Override + // public int compareToStopKey() { + // return LIB.mdb_cmp(txnPointer, dbiPointer, cursor.keyVal().pointerKey(), + // stopKey.pointer()); + // } + // }; + // } + + @Override + public void close() { + cursor.close(); + try { + rangeComparator.close(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * Obtain an iterator. + * + *

As iteration of the returned iterator will cause movement of the underlying LMDB cursor, an + * {@link IllegalStateException} is thrown if an attempt is made to obtain the iterator more than + * once. For advanced cursor control (such as being able to iterate over the same data multiple + * times etc) please instead refer to {@link Dbi#openCursor(org.lmdbjava.Txn)}. + * + * @return an iterator + */ + @Override + public Iterator> iterator() { + if (iteratorReturned) { + throw new IllegalStateException("Iterator can only be returned once"); + } + iteratorReturned = true; + + return new Iterator>() { + @Override + public boolean hasNext() { + while (state != RELEASED && state != TERMINATED) { + update(); + } + return state == RELEASED; + } + + @Override + public KeyVal next() { + if (!hasNext()) { + throw new NoSuchElementException(); } + state = REQUIRES_NEXT_OP; + return entry; + } + + @Override + public void remove() { + cursor.delete(); + } + }; + } + + private void executeCursorOp(final CursorOp op) { + final boolean found; + switch (op) { + case FIRST: + found = cursor.first(); + break; + case LAST: + found = cursor.last(); + break; + case NEXT: + found = cursor.next(); + break; + case PREV: + found = cursor.prev(); + break; + case GET_START_KEY: + found = cursor.get(range.getStart(), MDB_SET_RANGE); + break; + case GET_START_KEY_BACKWARD: + found = cursor.get(range.getStart(), MDB_SET_RANGE) || cursor.last(); + break; + default: + throw new IllegalStateException("Unknown cursor operation"); } + entry.setK(found ? cursor.key() : null); + entry.setV(found ? cursor.val() : null); + } - static RangeComparator createJavaRangeComparator( - final KeyRange range, - final Comparator comparator, - final Supplier currentKeySupplier) { - final T start = range.getStart(); - final T stop = range.getStop(); - return new RangeComparator() { - @Override - public int compareToStartKey() { - return comparator.compare(currentKeySupplier.get(), start); - } - - @Override - public int compareToStopKey() { - return comparator.compare(currentKeySupplier.get(), stop); - } - }; + private void executeIteratorOp() { + final IteratorOp op = range.getType().iteratorOp(entry.key(), rangeComparator); + switch (op) { + case CALL_NEXT_OP: + executeCursorOp(range.getType().nextOp()); + state = REQUIRES_ITERATOR_OP; + break; + case TERMINATE: + state = TERMINATED; + break; + case RELEASE: + state = RELEASED; + break; + default: + throw new IllegalStateException("Unknown operation"); } + } + + private void update() { + switch (state) { + case REQUIRES_INITIAL_OP: + executeCursorOp(range.getType().initialOp()); + state = REQUIRES_ITERATOR_OP; + break; + case REQUIRES_NEXT_OP: + executeCursorOp(range.getType().nextOp()); + state = REQUIRES_ITERATOR_OP; + break; + case REQUIRES_ITERATOR_OP: + executeIteratorOp(); + break; + case TERMINATED: + break; + default: + throw new IllegalStateException("Unknown state"); + } + } + + /** + * Holder for a key and value pair. + * + *

The same holder instance will always be returned for a given iterator. The returned keys and + * values may change or point to different memory locations following changes in the iterator, + * cursor or transaction. + * + * @param buffer type + */ + public static final class KeyVal { + + private T k; + private T v; + + /** Explicitly-defined default constructor to avoid warnings. */ + public KeyVal() {} /** - * Calls down to mdb_cmp to make use of the comparator that LMDB uses for insertion order. + * The key. * - * @param txnPointer The pointer to the transaction. - * @param dbiPointer The pointer to the Dbi so LMDB can use the comparator of the Dbi + * @return key */ - private RangeComparator createLmdbDbiComparator( - final Pointer txnPointer, final Pointer dbiPointer) { - Objects.requireNonNull(txnPointer); - Objects.requireNonNull(dbiPointer); - Objects.requireNonNull(cursor); - - return new RangeComparator() { - @Override - public int compareToStartKey() { - return LIB.mdb_cmp(txnPointer, dbiPointer, cursor.keyVal().pointerKey(), startKey.pointerKey()); - } - - @Override - public int compareToStopKey() { - return LIB.mdb_cmp(txnPointer, dbiPointer, cursor.keyVal().pointerKey(), stopKey.pointerKey()); - } - }; - } - - @Override - public void close() { - cursor.close(); + public T key() { + return k; } /** - * Obtain an iterator. + * The value. * - *

As iteration of the returned iterator will cause movement of the underlying LMDB cursor, an - * {@link IllegalStateException} is thrown if an attempt is made to obtain the iterator more than - * once. For advanced cursor control (such as being able to iterate over the same data multiple - * times etc) please instead refer to {@link Dbi#openCursor(org.lmdbjava.Txn)}. - * - * @return an iterator + * @return value */ - @Override - public Iterator> iterator() { - if (iteratorReturned) { - throw new IllegalStateException("Iterator can only be returned once"); - } - iteratorReturned = true; - - return new Iterator>() { - @Override - public boolean hasNext() { - while (state != RELEASED && state != TERMINATED) { - update(); - } - return state == RELEASED; - } - - @Override - public KeyVal next() { - if (!hasNext()) { - throw new NoSuchElementException(); - } - state = REQUIRES_NEXT_OP; - return entry; - } - - @Override - public void remove() { - cursor.delete(); - } - }; + public T val() { + return v; } - private void executeCursorOp(final CursorOp op) { - final boolean found; - switch (op) { - case FIRST: - found = cursor.first(); - break; - case LAST: - found = cursor.last(); - break; - case NEXT: - found = cursor.next(); - break; - case PREV: - found = cursor.prev(); - break; - case GET_START_KEY: - found = cursor.get(range.getStart(), MDB_SET_RANGE); - break; - case GET_START_KEY_BACKWARD: - found = cursor.get(range.getStart(), MDB_SET_RANGE) || cursor.last(); - break; - default: - throw new IllegalStateException("Unknown cursor operation"); - } - entry.setK(found ? cursor.key() : null); - entry.setV(found ? cursor.val() : null); + void setK(final T key) { + this.k = key; } - private void executeIteratorOp() { - final IteratorOp op = - range.getType().iteratorOp(range.getStart(), range.getStop(), entry.key(), rangeComparator); - switch (op) { - case CALL_NEXT_OP: - executeCursorOp(range.getType().nextOp()); - state = REQUIRES_ITERATOR_OP; - break; - case TERMINATE: - state = TERMINATED; - break; - case RELEASE: - state = RELEASED; - break; - default: - throw new IllegalStateException("Unknown operation"); - } + void setV(final T val) { + this.v = val; } + } - private void update() { - switch (state) { - case REQUIRES_INITIAL_OP: - executeCursorOp(range.getType().initialOp()); - state = REQUIRES_ITERATOR_OP; - break; - case REQUIRES_NEXT_OP: - executeCursorOp(range.getType().nextOp()); - state = REQUIRES_ITERATOR_OP; - break; - case REQUIRES_ITERATOR_OP: - executeIteratorOp(); - break; - case TERMINATED: - break; - default: - throw new IllegalStateException("Unknown state"); - } + /** Represents the internal {@link CursorIterable} state. */ + enum State { + REQUIRES_INITIAL_OP, + REQUIRES_NEXT_OP, + REQUIRES_ITERATOR_OP, + RELEASED, + TERMINATED + } + + static class JavaRangeComparator implements RangeComparator { + + private final Comparator comparator; + private final Supplier currentKeySupplier; + private final T start; + private final T stop; + + JavaRangeComparator( + final KeyRange range, + final Comparator comparator, + final Supplier currentKeySupplier) { + this.comparator = comparator; + this.currentKeySupplier = currentKeySupplier; + this.start = range.getStart(); + this.stop = range.getStop(); } - /** - * Holder for a key and value pair. - * - *

The same holder instance will always be returned for a given iterator. The returned keys and - * values may change or point to different memory locations following changes in the iterator, - * cursor or transaction. - * - * @param buffer type - */ - public static final class KeyVal { + @Override + public int compareToStartKey() { + return comparator.compare(currentKeySupplier.get(), start); + } - private T k; - private T v; + @Override + public int compareToStopKey() { + return comparator.compare(currentKeySupplier.get(), stop); + } - /** - * Explicitly-defined default constructor to avoid warnings. - */ - public KeyVal() { - } + @Override + public void close() throws Exception { + // Nothing to close + } + } - /** - * The key. - * - * @return key - */ - public T key() { - return k; - } + /** + * Calls down to mdb_cmp to make use of the comparator that LMDB uses for insertion order. Has a + * very slight overhead as compared to {@link JavaRangeComparator}. + */ + private static class LmdbRangeComparator implements RangeComparator { - /** - * The value. - * - * @return value - */ - public T val() { - return v; - } + private final Pointer txnPointer; + private final Pointer dbiPointer; + private final Pointer cursorKeyPointer; + private final Key startKey; + private final Key stopKey; + private final Pointer startKeyPointer; + private final Pointer stopKeyPointer; - void setK(final T key) { - this.k = key; - } + public LmdbRangeComparator( + final Txn txn, + final Dbi dbi, + final Cursor cursor, + final KeyRange range, + final BufferProxy proxy) { + txnPointer = Objects.requireNonNull(txn).pointer(); + dbiPointer = Objects.requireNonNull(dbi).pointer(); + cursorKeyPointer = Objects.requireNonNull(cursor).keyVal().pointerKey(); + // Allocate buffers for use with the start/stop keys if required. + // Saves us copying bytes on each comparison + Objects.requireNonNull(range); + startKey = createKey(range.getStart(), proxy); + stopKey = createKey(range.getStop(), proxy); + startKeyPointer = startKey != null ? startKey.pointer() : null; + stopKeyPointer = stopKey != null ? stopKey.pointer() : null; + } - void setV(final T val) { - this.v = val; - } + @Override + public int compareToStartKey() { + return LIB.mdb_cmp(txnPointer, dbiPointer, cursorKeyPointer, startKeyPointer); } - /** - * Represents the internal {@link CursorIterable} state. - */ - enum State { - REQUIRES_INITIAL_OP, - REQUIRES_NEXT_OP, - REQUIRES_ITERATOR_OP, - RELEASED, - TERMINATED + @Override + public int compareToStopKey() { + return LIB.mdb_cmp(txnPointer, dbiPointer, cursorKeyPointer, stopKeyPointer); + } + + @Override + public void close() { + if (startKey != null) { + startKey.close(); + } + if (stopKey != null) { + stopKey.close(); + } + } + + private Key createKey(final T keyBuffer, final BufferProxy proxy) { + if (keyBuffer != null) { + final Key key = proxy.key(); + key.keyIn(keyBuffer); + return key; + } else { + return null; + } } + } } diff --git a/src/main/java/org/lmdbjava/Env.java b/src/main/java/org/lmdbjava/Env.java index 1543282c..2e4822b7 100644 --- a/src/main/java/org/lmdbjava/Env.java +++ b/src/main/java/org/lmdbjava/Env.java @@ -257,16 +257,15 @@ public Dbi openDbi(final String name, final DbiFlags... flags) { /** * Convenience method that opens a {@link Dbi} with a UTF-8 database name and associated {@link * Comparator} for use by {@link CursorIterable} when comparing start/stop keys. - *

- * It is very important that the passed comparator behaves in the same way as the comparator + * + *

It is very important that the passed comparator behaves in the same way as the comparator * LMDB uses for its insertion order (for the type of data that will be stored in the database), - * or you fully understand the implications of them behaving differently. - * LMDB's comparator is unsigned lexicographical, unless {@link DbiFlags#MDB_INTEGERKEY} is used. - *

+ * or you fully understand the implications of them behaving differently. LMDB's comparator is + * unsigned lexicographical, unless {@link DbiFlags#MDB_INTEGERKEY} is used. * * @param name name of the database (or null if no name is required) - * @param comparator custom comparator for cursor start/stop key comparisons. If null, - * LMDB's comparator will be used. + * @param comparator custom comparator for cursor start/stop key comparisons. If null, LMDB's + * comparator will be used. * @param flags to open the database with * @return a database that is ready to use */ @@ -278,14 +277,14 @@ public Dbi openDbi( /** * Convenience method that opens a {@link Dbi} with a UTF-8 database name and associated {@link - * Comparator}. The comparator will be used by {@link CursorIterable} when comparing start/stop keys - * as a minimum. If nativeCb is {@code true}, this comparator will also be called by LMDB to determine - * insertion/iteration order. Calling back to a java comparator may significantly impact performance. + * Comparator}. The comparator will be used by {@link CursorIterable} when comparing start/stop + * keys as a minimum. If nativeCb is {@code true}, this comparator will also be called by LMDB to + * determine insertion/iteration order. Calling back to a java comparator may significantly impact + * performance. * * @param name name of the database (or null if no name is required) * @param comparator custom comparator for cursor start/stop key comparisons and optionally for - * LMDB to call back to. If null, - * LMDB's comparator will be used. + * LMDB to call back to. If null, LMDB's comparator will be used. * @param nativeCb whether LMDB native code calls back to the Java comparator * @param flags to open the database with * @return a database that is ready to use diff --git a/src/main/java/org/lmdbjava/Key.java b/src/main/java/org/lmdbjava/Key.java index 7fd8bbe2..12c54290 100644 --- a/src/main/java/org/lmdbjava/Key.java +++ b/src/main/java/org/lmdbjava/Key.java @@ -33,7 +33,6 @@ final class Key implements AutoCloseable { private boolean closed; private T k; private final BufferProxy proxy; - private final Pointer ptrArray; private final Pointer ptrKey; private final long ptrKeyAddr; @@ -43,7 +42,6 @@ final class Key implements AutoCloseable { this.k = proxy.allocate(); ptrKey = MEM_MGR.allocateTemporary(MDB_VAL_STRUCT_SIZE, false); ptrKeyAddr = ptrKey.address(); - ptrArray = MEM_MGR.allocateTemporary(MDB_VAL_STRUCT_SIZE * 2, false); } @Override @@ -68,7 +66,7 @@ T keyOut() { return k; } - Pointer pointerKey() { + Pointer pointer() { return ptrKey; } } diff --git a/src/main/java/org/lmdbjava/KeyRangeType.java b/src/main/java/org/lmdbjava/KeyRangeType.java index 07123e9a..26f09636 100644 --- a/src/main/java/org/lmdbjava/KeyRangeType.java +++ b/src/main/java/org/lmdbjava/KeyRangeType.java @@ -319,14 +319,12 @@ CursorOp initialOp() { * * @param buffer type * @param comparator for the buffers - * @param start start buffer - * @param stop stop buffer * @param buffer current key returned by LMDB (may be null) * @param rangeComparator comparator (required) * @return response to this key */ > IteratorOp iteratorOp( - final T start, final T stop, final T buffer, final RangeComparator rangeComparator) { + final T buffer, final RangeComparator rangeComparator) { requireNonNull(rangeComparator, "Comparator required"); if (buffer == null) { return TERMINATE; diff --git a/src/main/java/org/lmdbjava/RangeComparator.java b/src/main/java/org/lmdbjava/RangeComparator.java index 162584b1..59d46015 100644 --- a/src/main/java/org/lmdbjava/RangeComparator.java +++ b/src/main/java/org/lmdbjava/RangeComparator.java @@ -1,19 +1,17 @@ package org.lmdbjava; -/** - * For comparing a cursor's current key against a {@link KeyRange}'s start/stop key. - */ -interface RangeComparator { +/** For comparing a cursor's current key against a {@link KeyRange}'s start/stop key. */ +interface RangeComparator extends AutoCloseable { - /** - * Compare the cursor's current key to the range start key. Equivalent to compareTo(currentKey, - * startKey) - */ - int compareToStartKey(); + /** + * Compare the cursor's current key to the range start key. Equivalent to compareTo(currentKey, + * startKey) + */ + int compareToStartKey(); - /** - * Compare the cursor's current key to the range stop key. Equivalent to compareTo(currentKey, - * stopKey) - */ - int compareToStopKey(); + /** + * Compare the cursor's current key to the range stop key. Equivalent to compareTo(currentKey, + * stopKey) + */ + int compareToStopKey(); } diff --git a/src/test/java/org/lmdbjava/CursorIterablePerfTest.java b/src/test/java/org/lmdbjava/CursorIterablePerfTest.java index 99667b1d..19e5a9b9 100644 --- a/src/test/java/org/lmdbjava/CursorIterablePerfTest.java +++ b/src/test/java/org/lmdbjava/CursorIterablePerfTest.java @@ -26,8 +26,7 @@ public class CursorIterablePerfTest { - @Rule - public final TemporaryFolder tmp = new TemporaryFolder(); + @Rule public final TemporaryFolder tmp = new TemporaryFolder(); // private static final int ITERATIONS = 5_000_000; private static final int ITERATIONS = 100_000; @@ -52,12 +51,13 @@ public void before() throws IOException { .open(path, POSIX_MODE, MDB_NOSUBDIR); // Use a java comparator for start/stop keys only - dbJavaComparator = env.openDbi("JavaComparator", bufferProxy.getUnsignedComparator(), MDB_CREATE); + dbJavaComparator = + env.openDbi("JavaComparator", bufferProxy.getUnsignedComparator(), MDB_CREATE); // Use LMDB comparator for start/stop keys dbLmdbComparator = env.openDbi("LmdbComparator", MDB_CREATE); // Use a java comparator for start/stop keys and as a callback comparator - dbCallbackComparator = env.openDbi( - "CallBackComparator", bufferProxy.getUnsignedComparator(), true, MDB_CREATE); + dbCallbackComparator = + env.openDbi("CallBackComparator", bufferProxy.getUnsignedComparator(), true, MDB_CREATE); dbs.add(dbJavaComparator); dbs.add(dbLmdbComparator); @@ -89,7 +89,7 @@ private void populateDatabases(final boolean randomOrder) { for (final Dbi db : dbs) { // Clean out the db first try (Txn txn = env.txnWrite(); - final Cursor cursor = db.openCursor(txn)) { + final Cursor cursor = db.openCursor(txn)) { while (cursor.next()) { cursor.delete(); } @@ -108,9 +108,13 @@ private void populateDatabases(final boolean randomOrder) { txn.commit(); } final Duration duration = Duration.between(start, Instant.now()); - System.out.println("DB: " + dbName - + " - Loaded in duration: " + duration - + ", millis: " + duration.toMillis()); + System.out.println( + "DB: " + + dbName + + " - Loaded in duration: " + + duration + + ", millis: " + + duration.toMillis()); } } } @@ -147,16 +151,21 @@ public void comparePerf(final boolean randomOrder) { int cnt = 0; // Exercise the stop key comparator on every entry try (Txn txn = env.txnRead(); - CursorIterable c = db.iterate(txn, keyRange)) { + CursorIterable c = db.iterate(txn, keyRange)) { for (final CursorIterable.KeyVal kv : c) { cnt++; } } final Duration duration = Duration.between(start, Instant.now()); - System.out.println("DB: " + dbName - + " - Iterated in duration: " + duration - + ", millis: " + duration.toMillis() - + ", cnt: " + cnt); + System.out.println( + "DB: " + + dbName + + " - Iterated in duration: " + + duration + + ", millis: " + + duration.toMillis() + + ", cnt: " + + cnt); } } } diff --git a/src/test/java/org/lmdbjava/CursorIterableTest.java b/src/test/java/org/lmdbjava/CursorIterableTest.java index 96be0816..22cf7361 100644 --- a/src/test/java/org/lmdbjava/CursorIterableTest.java +++ b/src/test/java/org/lmdbjava/CursorIterableTest.java @@ -134,8 +134,7 @@ public void before() throws IOException { // Use LMDB comparator for start/stop keys dbLmdbComparator = env.openDbi(DB_2, MDB_CREATE); // Use a java comparator for start/stop keys and as a callback comparaotr - dbCallbackComparator = env.openDbi( - DB_3, bufferProxy.getUnsignedComparator(), true, MDB_CREATE); + dbCallbackComparator = env.openDbi(DB_3, bufferProxy.getUnsignedComparator(), true, MDB_CREATE); populateList(); @@ -422,18 +421,17 @@ public void testSignedVsUnsigned() { // Compare the same assertThat( - unsignedComparator.compare(val1, val2), - Matchers.is(signedComparator.compare(val1, val2))); + unsignedComparator.compare(val1, val2), Matchers.is(signedComparator.compare(val1, val2))); // Compare differently assertThat( - unsignedComparator.compare(val110, val150), - Matchers.not(signedComparator.compare(val110, val150))); + unsignedComparator.compare(val110, val150), + Matchers.not(signedComparator.compare(val110, val150))); // Compare differently assertThat( - unsignedComparator.compare(val111, val150), - Matchers.not(signedComparator.compare(val111, val150))); + unsignedComparator.compare(val111, val150), + Matchers.not(signedComparator.compare(val111, val150))); // This will fail if the db is using a signed comparator for the start/stop keys for (final Dbi db : dbs) { @@ -444,11 +442,11 @@ public void testSignedVsUnsigned() { KeyRange keyRange = KeyRange.atLeastBackward(startKeyBuf); try (Txn txn = env.txnRead(); - CursorIterable c = db.iterate(txn, keyRange)) { + CursorIterable c = db.iterate(txn, keyRange)) { for (final CursorIterable.KeyVal kv : c) { final int key = kv.key().getInt(); final int val = kv.val().getInt(); -// System.out.println("key: " + key + " val: " + val); + // System.out.println("key: " + key + " val: " + val); assertThat(key, is(110)); break; } diff --git a/src/test/java/org/lmdbjava/KeyRangeTest.java b/src/test/java/org/lmdbjava/KeyRangeTest.java index 0197bf11..c474e982 100644 --- a/src/test/java/org/lmdbjava/KeyRangeTest.java +++ b/src/test/java/org/lmdbjava/KeyRangeTest.java @@ -197,8 +197,8 @@ private void verify(final KeyRange range, final int... expected) { do { final Integer finalBuff = buff; final RangeComparator rangeComparator = - CursorIterable.createJavaRangeComparator(range, Integer::compareTo, () -> finalBuff); - op = range.getType().iteratorOp(range.getStart(), range.getStop(), buff, rangeComparator); + new CursorIterable.JavaRangeComparator<>(range, Integer::compareTo, () -> finalBuff); + op = range.getType().iteratorOp(buff, rangeComparator); switch (op) { case CALL_NEXT_OP: buff = cursor.apply(range.getType().nextOp(), range.getStart()); From 9509f6bea115c7839e14881e4ecb4459211820d4 Mon Sep 17 00:00:00 2001 From: at055612 <22818309+at055612@users.noreply.github.com> Date: Thu, 6 Mar 2025 19:05:48 +0000 Subject: [PATCH 05/21] gh-249 Remove commented code --- .../java/org/lmdbjava/CursorIterable.java | 56 ------------------- 1 file changed, 56 deletions(-) diff --git a/src/main/java/org/lmdbjava/CursorIterable.java b/src/main/java/org/lmdbjava/CursorIterable.java index 7c487bae..b0de7d06 100644 --- a/src/main/java/org/lmdbjava/CursorIterable.java +++ b/src/main/java/org/lmdbjava/CursorIterable.java @@ -42,7 +42,6 @@ */ public final class CursorIterable implements Iterable>, AutoCloseable { - // private final Comparator comparator; private final RangeComparator rangeComparator; private final Cursor cursor; private final KeyVal entry; @@ -69,61 +68,6 @@ public final class CursorIterable implements Iterable RangeComparator createJavaRangeComparator( - // final KeyRange range, - // final Comparator comparator, - // final Supplier currentKeySupplier) { - // final T start = range.getStart(); - // final T stop = range.getStop(); - // return new RangeComparator() { - // @Override - // public int compareToStartKey() { - // return comparator.compare(currentKeySupplier.get(), start); - // } - // - // @Override - // public int compareToStopKey() { - // return comparator.compare(currentKeySupplier.get(), stop); - // } - // }; - // } - - // /** - // * Calls down to mdb_cmp to make use of the comparator that LMDB uses for insertion order. - // * - // * @param txnPointer The pointer to the transaction. - // * @param dbiPointer The pointer to the Dbi so LMDB can use the comparator of the Dbi - // * @param proxy - // */ - // private RangeComparator createLmdbDbiComparator(final Pointer txnPointer, - // final Pointer dbiPointer, - // final Pointer cursorKeyPointer, - // final KeyRange range, - // final BufferProxy proxy) { - // Objects.requireNonNull(txnPointer); - // Objects.requireNonNull(dbiPointer); - // Objects.requireNonNull(range); - // Objects.requireNonNull(cursor); - // // Allocate buffers for use with the start/stop keys if required. - // // Saves us copying bytes on each comparison - // final Key startKey = createKey(range.getStart(), proxy); - // final Key stopKey = createKey(range.getStop(), proxy); - // - // return new RangeComparator() { - // @Override - // public int compareToStartKey() { - // return LIB.mdb_cmp(txnPointer, dbiPointer, cursor.keyVal().pointerKey(), - // startKey.pointer()); - // } - // - // @Override - // public int compareToStopKey() { - // return LIB.mdb_cmp(txnPointer, dbiPointer, cursor.keyVal().pointerKey(), - // stopKey.pointer()); - // } - // }; - // } - @Override public void close() { cursor.close(); From 67e2df1002ee86534ad082beb83a1c753e2ab6b5 Mon Sep 17 00:00:00 2001 From: at055612 <22818309+at055612@users.noreply.github.com> Date: Fri, 7 Mar 2025 10:31:12 +0000 Subject: [PATCH 06/21] gh-249 Add CursorIterableIntegerKeyTest --- .../CursorIterableIntegerKeyTest.java | 493 ++++++++++++++++++ .../org/lmdbjava/CursorIterablePerfTest.java | 2 +- src/test/java/org/lmdbjava/TestUtils.java | 15 + 3 files changed, 509 insertions(+), 1 deletion(-) create mode 100644 src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java diff --git a/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java b/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java new file mode 100644 index 00000000..431c6e51 --- /dev/null +++ b/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java @@ -0,0 +1,493 @@ +/* + * Copyright © 2016-2025 The LmdbJava Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lmdbjava; + +import static com.jakewharton.byteunits.BinaryByteUnit.KIBIBYTES; +import static java.util.Arrays.asList; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasSize; +import static org.lmdbjava.DbiFlags.MDB_CREATE; +import static org.lmdbjava.DbiFlags.MDB_INTEGERKEY; +import static org.lmdbjava.Env.create; +import static org.lmdbjava.EnvFlags.MDB_NOSUBDIR; +import static org.lmdbjava.KeyRange.all; +import static org.lmdbjava.KeyRange.allBackward; +import static org.lmdbjava.KeyRange.atLeast; +import static org.lmdbjava.KeyRange.atLeastBackward; +import static org.lmdbjava.KeyRange.atMost; +import static org.lmdbjava.KeyRange.atMostBackward; +import static org.lmdbjava.KeyRange.closed; +import static org.lmdbjava.KeyRange.closedBackward; +import static org.lmdbjava.KeyRange.closedOpen; +import static org.lmdbjava.KeyRange.closedOpenBackward; +import static org.lmdbjava.KeyRange.greaterThan; +import static org.lmdbjava.KeyRange.greaterThanBackward; +import static org.lmdbjava.KeyRange.lessThan; +import static org.lmdbjava.KeyRange.lessThanBackward; +import static org.lmdbjava.KeyRange.open; +import static org.lmdbjava.KeyRange.openBackward; +import static org.lmdbjava.KeyRange.openClosed; +import static org.lmdbjava.KeyRange.openClosedBackward; +import static org.lmdbjava.PutFlags.MDB_NOOVERWRITE; +import static org.lmdbjava.TestUtils.DB_1; +import static org.lmdbjava.TestUtils.DB_2; +import static org.lmdbjava.TestUtils.DB_3; +import static org.lmdbjava.TestUtils.POSIX_MODE; +import static org.lmdbjava.TestUtils.bb; +import static org.lmdbjava.TestUtils.bbNative; +import static org.lmdbjava.TestUtils.getNativeInt; + +import com.google.common.primitives.UnsignedBytes; +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Deque; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.NoSuchElementException; +import org.hamcrest.Matchers; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.lmdbjava.CursorIterable.KeyVal; + +/** Test {@link CursorIterable} using {@link DbiFlags#MDB_INTEGERKEY} to ensure that + * comparators work with native order integer keys. */ +public final class CursorIterableIntegerKeyTest { + + @Rule public final TemporaryFolder tmp = new TemporaryFolder(); + private Dbi dbJavaComparator; + private Dbi dbLmdbComparator; + private Dbi dbCallbackComparator; + private List> dbs = new ArrayList<>(); + private Env env; + private Deque list; + + @After + public void after() { + env.close(); + } + + @Test + public void allBackwardTest() { + verify(allBackward(), 8, 6, 4, 2); + } + + @Test + public void allTest() { + verify(all(), 2, 4, 6, 8); + } + + @Test + public void atLeastBackwardTest() { + verify(atLeastBackward(bbNative(5)), 4, 2); + verify(atLeastBackward(bbNative(6)), 6, 4, 2); + verify(atLeastBackward(bbNative(9)), 8, 6, 4, 2); + } + + @Test + public void atLeastTest() { + verify(atLeast(bbNative(5)), 6, 8); + verify(atLeast(bbNative(6)), 6, 8); + } + + @Test + public void atMostBackwardTest() { + verify(atMostBackward(bbNative(5)), 8, 6); + verify(atMostBackward(bbNative(6)), 8, 6); + } + + @Test + public void atMostTest() { + verify(atMost(bbNative(5)), 2, 4); + verify(atMost(bbNative(6)), 2, 4, 6); + } + + @Before + public void before() throws IOException { + final File path = tmp.newFile(); + final BufferProxy bufferProxy = ByteBufferProxy.PROXY_OPTIMAL; + env = + create(bufferProxy) + .setMapSize(KIBIBYTES.toBytes(256)) + .setMaxReaders(1) + .setMaxDbs(3) + .open(path, POSIX_MODE, MDB_NOSUBDIR); + + // Use a java comparator for start/stop keys only + dbJavaComparator = env.openDbi(DB_1, + bufferProxy.getUnsignedComparator(), + MDB_CREATE, + MDB_INTEGERKEY); + // Use LMDB comparator for start/stop keys + dbLmdbComparator = env.openDbi(DB_2, MDB_CREATE, MDB_INTEGERKEY); + // Use a java comparator for start/stop keys and as a callback comparaotr + dbCallbackComparator = env.openDbi(DB_3, + bufferProxy.getUnsignedComparator(), + true, + MDB_CREATE, + MDB_INTEGERKEY); + + populateList(); + + populateDatabase(dbJavaComparator); + populateDatabase(dbLmdbComparator); + populateDatabase(dbCallbackComparator); + + dbs.add(dbJavaComparator); + dbs.add(dbLmdbComparator); + dbs.add(dbCallbackComparator); + } + + private void populateList() { + list = new LinkedList<>(); + list.addAll(asList(2, 3, 4, 5, 6, 7, 8, 9)); + } + + private void populateDatabase(final Dbi dbi) { + try (Txn txn = env.txnWrite()) { + final Cursor c = dbi.openCursor(txn); + c.put(bbNative(2), bb(3), MDB_NOOVERWRITE); + c.put(bbNative(4), bb(5)); + c.put(bbNative(6), bb(7)); + c.put(bbNative(8), bb(9)); + txn.commit(); + } + } + + @Test + public void closedBackwardTest() { + verify(closedBackward(bbNative(7), bbNative(3)), 6, 4); + verify(closedBackward(bbNative(6), bbNative(2)), 6, 4, 2); + verify(closedBackward(bbNative(9), bbNative(3)), 8, 6, 4); + } + + @Test + public void closedOpenBackwardTest() { + verify(closedOpenBackward(bbNative(8), bbNative(3)), 8, 6, 4); + verify(closedOpenBackward(bbNative(7), bbNative(2)), 6, 4); + verify(closedOpenBackward(bbNative(9), bbNative(3)), 8, 6, 4); + } + + @Test + public void closedOpenTest() { + verify(closedOpen(bbNative(3), bbNative(8)), 4, 6); + verify(closedOpen(bbNative(2), bbNative(6)), 2, 4); + } + + @Test + public void closedTest() { + verify(closed(bbNative(3), bbNative(7)), 4, 6); + verify(closed(bbNative(2), bbNative(6)), 2, 4, 6); + verify(closed(bbNative(1), bbNative(7)), 2, 4, 6); + } + + @Test + public void greaterThanBackwardTest() { + verify(greaterThanBackward(bbNative(6)), 4, 2); + verify(greaterThanBackward(bbNative(7)), 6, 4, 2); + verify(greaterThanBackward(bbNative(9)), 8, 6, 4, 2); + } + + @Test + public void greaterThanTest() { + verify(greaterThan(bbNative(4)), 6, 8); + verify(greaterThan(bbNative(3)), 4, 6, 8); + } + + @Test(expected = IllegalStateException.class) + public void iterableOnlyReturnedOnce() { + for (final Dbi db : dbs) { + try (Txn txn = env.txnRead(); + CursorIterable c = db.iterate(txn)) { + c.iterator(); // ok + c.iterator(); // fails + } + } + } + + @Test + public void iterate() { + for (final Dbi db : dbs) { + populateList(); + try (Txn txn = env.txnRead(); + CursorIterable c = db.iterate(txn)) { + + int cnt = 0; + for (final KeyVal kv : c) { + assertThat(getNativeInt(kv.key()), is(list.pollFirst())); + assertThat(kv.val().getInt(), is(list.pollFirst())); + } + } + } + } + + @Test(expected = IllegalStateException.class) + public void iteratorOnlyReturnedOnce() { + for (final Dbi db : dbs) { + try (Txn txn = env.txnRead(); + CursorIterable c = db.iterate(txn)) { + c.iterator(); // ok + c.iterator(); // fails + } + } + } + + @Test + public void lessThanBackwardTest() { + verify(lessThanBackward(bbNative(5)), 8, 6); + verify(lessThanBackward(bbNative(2)), 8, 6, 4); + } + + @Test + public void lessThanTest() { + verify(lessThan(bbNative(5)), 2, 4); + verify(lessThan(bbNative(8)), 2, 4, 6); + } + + @Test(expected = NoSuchElementException.class) + public void nextThrowsNoSuchElementExceptionIfNoMoreElements() { + for (final Dbi db : dbs) { + populateList(); + try (Txn txn = env.txnRead(); + CursorIterable c = db.iterate(txn)) { + final Iterator> i = c.iterator(); + while (i.hasNext()) { + final KeyVal kv = i.next(); + assertThat(TestUtils.getNativeInt(kv.key()), is(list.pollFirst())); + assertThat(kv.val().getInt(), is(list.pollFirst())); + } + assertThat(i.hasNext(), is(false)); + i.next(); + } + } + } + + @Test + public void openBackwardTest() { + verify(openBackward(bbNative(7), bbNative(2)), 6, 4); + verify(openBackward(bbNative(8), bbNative(1)), 6, 4, 2); + verify(openBackward(bbNative(9), bbNative(4)), 8, 6); + } + + @Test + public void openClosedBackwardTest() { + verify(openClosedBackward(bbNative(7), bbNative(2)), 6, 4, 2); + verify(openClosedBackward(bbNative(8), bbNative(4)), 6, 4); + verify(openClosedBackward(bbNative(9), bbNative(4)), 8, 6, 4); + } + + @Test + public void openClosedBackwardTestWithGuava() { + final Comparator guava = UnsignedBytes.lexicographicalComparator(); + final Comparator comparator = + (bb1, bb2) -> { + final byte[] array1 = new byte[bb1.remaining()]; + final byte[] array2 = new byte[bb2.remaining()]; + bb1.mark(); + bb2.mark(); + bb1.get(array1); + bb2.get(array2); + bb1.reset(); + bb2.reset(); + return guava.compare(array1, array2); + }; + final Dbi guavaDbi = env.openDbi(DB_1, comparator, MDB_CREATE); + populateDatabase(guavaDbi); + verify(openClosedBackward(bbNative(7), bbNative(2)), guavaDbi, 6, 4, 2); + verify(openClosedBackward(bbNative(8), bbNative(4)), guavaDbi, 6, 4); + } + + @Test + public void openClosedTest() { + verify(openClosed(bbNative(3), bbNative(8)), 4, 6, 8); + verify(openClosed(bbNative(2), bbNative(6)), 4, 6); + } + + @Test + public void openTest() { + verify(open(bbNative(3), bbNative(7)), 4, 6); + verify(open(bbNative(2), bbNative(8)), 4, 6); + } + + @Test + public void removeOddElements() { + for (final Dbi db : dbs) { + verify(db, all(), 2, 4, 6, 8); + int idx = -1; + try (Txn txn = env.txnWrite()) { + try (CursorIterable ci = db.iterate(txn)) { + final Iterator> c = ci.iterator(); + while (c.hasNext()) { + c.next(); + idx++; + if (idx % 2 == 0) { + c.remove(); + } + } + } + txn.commit(); + } + verify(db, all(), 4, 8); + } + } + + @Test(expected = Env.AlreadyClosedException.class) + public void nextWithClosedEnvTest() { + for (final Dbi db : dbs) { + try (Txn txn = env.txnRead()) { + try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { + final Iterator> c = ci.iterator(); + + env.close(); + c.next(); + } + } + } + } + + @Test(expected = Env.AlreadyClosedException.class) + public void removeWithClosedEnvTest() { + for (final Dbi db : dbs) { + try (Txn txn = env.txnWrite()) { + try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { + final Iterator> c = ci.iterator(); + + final KeyVal keyVal = c.next(); + assertThat(keyVal, Matchers.notNullValue()); + + env.close(); + c.remove(); + } + } + } + } + + @Test(expected = Env.AlreadyClosedException.class) + public void hasNextWithClosedEnvTest() { + for (final Dbi db : dbs) { + try (Txn txn = env.txnRead()) { + try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { + final Iterator> c = ci.iterator(); + + env.close(); + c.hasNext(); + } + } + } + } + + @Test(expected = Env.AlreadyClosedException.class) + public void forEachRemainingWithClosedEnvTest() { + for (final Dbi db : dbs) { + try (Txn txn = env.txnRead()) { + try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { + final Iterator> c = ci.iterator(); + + env.close(); + c.forEachRemaining(keyVal -> {}); + } + } + } + } + + @Test + public void testSignedVsUnsigned() { + final ByteBuffer val1 = bbNative(1); + final ByteBuffer val2 = bbNative(2); + final ByteBuffer val110 = bbNative(110); + final ByteBuffer val111 = bbNative(111); + final ByteBuffer val150 = bbNative(150); + + final BufferProxy bufferProxy = ByteBufferProxy.PROXY_OPTIMAL; + final Comparator unsignedComparator = bufferProxy.getUnsignedComparator(); + final Comparator signedComparator = bufferProxy.getSignedComparator(); + + // Compare the same + assertThat( + unsignedComparator.compare(val1, val2), Matchers.is(signedComparator.compare(val1, val2))); + + // Compare differently + assertThat( + unsignedComparator.compare(val110, val150), + Matchers.not(signedComparator.compare(val110, val150))); + + // Compare differently + assertThat( + unsignedComparator.compare(val111, val150), + Matchers.not(signedComparator.compare(val111, val150))); + + // This will fail if the db is using a signed comparator for the start/stop keys + for (final Dbi db : dbs) { + db.put(val110, val110); + db.put(val150, val150); + + final ByteBuffer startKeyBuf = val111; + KeyRange keyRange = KeyRange.atLeastBackward(startKeyBuf); + + try (Txn txn = env.txnRead(); + CursorIterable c = db.iterate(txn, keyRange)) { + for (final KeyVal kv : c) { + final int key = getNativeInt(kv.key()); + final int val = kv.val().getInt(); + // System.out.println("key: " + key + " val: " + val); + assertThat(key, is(110)); + break; + } + } + } + } + + private void verify(final KeyRange range, final int... expected) { + // Verify using all comparator types + for (final Dbi db : dbs) { + verify(range, db, expected); + } + } + + private void verify( + final Dbi dbi, final KeyRange range, final int... expected) { + verify(range, dbi, expected); + } + + private void verify( + final KeyRange range, final Dbi dbi, final int... expected) { + + final List results = new ArrayList<>(); + + try (Txn txn = env.txnRead(); + CursorIterable c = dbi.iterate(txn, range)) { + for (final KeyVal kv : c) { + final int key = kv.key().order(ByteOrder.nativeOrder()).getInt(); + final int val = kv.val().getInt(); + results.add(key); + assertThat(val, is(key + 1)); + } + } + + assertThat(results, hasSize(expected.length)); + for (int idx = 0; idx < results.size(); idx++) { + assertThat(results.get(idx), is(expected[idx])); + } + } +} diff --git a/src/test/java/org/lmdbjava/CursorIterablePerfTest.java b/src/test/java/org/lmdbjava/CursorIterablePerfTest.java index 19e5a9b9..257ee705 100644 --- a/src/test/java/org/lmdbjava/CursorIterablePerfTest.java +++ b/src/test/java/org/lmdbjava/CursorIterablePerfTest.java @@ -73,7 +73,7 @@ private void populateList() { } private void populateDatabases(final boolean randomOrder) { - System.out.println("Clear then populate databases"); + System.out.println("Clear then populate databases (randomOrder=" + randomOrder + ")"); final List data; if (randomOrder) { diff --git a/src/test/java/org/lmdbjava/TestUtils.java b/src/test/java/org/lmdbjava/TestUtils.java index f3d3974b..ff84946f 100644 --- a/src/test/java/org/lmdbjava/TestUtils.java +++ b/src/test/java/org/lmdbjava/TestUtils.java @@ -23,6 +23,7 @@ import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import org.agrona.MutableDirectBuffer; import org.agrona.concurrent.UnsafeBuffer; @@ -49,6 +50,20 @@ static ByteBuffer bb(final int value) { return bb; } + static ByteBuffer bbNative(final int value) { + final ByteBuffer bb = allocateDirect(Long.BYTES) + .order(ByteOrder.nativeOrder()); + bb.putInt(value).flip(); + return bb; + } + + static int getNativeInt(final ByteBuffer bb) { + final int val = bb.order(ByteOrder.nativeOrder()) + .getInt(); + bb.rewind(); + return val; + } + static void invokePrivateConstructor(final Class clazz) { try { final Constructor c = clazz.getDeclaredConstructor(); From 620a89fc4bdc292b3f5cace5631a5af6afd6f11e Mon Sep 17 00:00:00 2001 From: at055612 <22818309+at055612@users.noreply.github.com> Date: Thu, 5 Jun 2025 10:03:12 +0100 Subject: [PATCH 07/21] gh-249 Remove MaskedFlag.isPropagatedToLmdb, add DbiBuilder WIP --- src/main/java/org/lmdbjava/Cursor.java | 8 +- src/main/java/org/lmdbjava/Dbi.java | 6 +- src/main/java/org/lmdbjava/DbiBuilder.java | 335 +++++++++++++++++++++ src/main/java/org/lmdbjava/DbiFlags.java | 3 + src/main/java/org/lmdbjava/Env.java | 27 +- src/main/java/org/lmdbjava/MaskedFlag.java | 72 +---- src/main/java/org/lmdbjava/Txn.java | 2 +- 7 files changed, 382 insertions(+), 71 deletions(-) create mode 100644 src/main/java/org/lmdbjava/DbiBuilder.java diff --git a/src/main/java/org/lmdbjava/Cursor.java b/src/main/java/org/lmdbjava/Cursor.java index 9070cff6..69f84c88 100644 --- a/src/main/java/org/lmdbjava/Cursor.java +++ b/src/main/java/org/lmdbjava/Cursor.java @@ -112,7 +112,7 @@ public void delete(final PutFlags... f) { txn.checkReady(); txn.checkWritesAllowed(); } - final int flags = mask(true, f); + final int flags = mask(f); checkRc(LIB.mdb_cursor_del(ptrCursor, flags)); } @@ -249,7 +249,7 @@ public boolean put(final T key, final T val, final PutFlags... op) { } kv.keyIn(key); kv.valIn(val); - final int mask = mask(true, op); + final int mask = mask(op); final int rc = LIB.mdb_cursor_put(ptrCursor, kv.pointerKey(), kv.pointerVal(), mask); if (rc == MDB_KEYEXIST) { if (isSet(mask, MDB_NOOVERWRITE)) { @@ -287,7 +287,7 @@ public void putMultiple(final T key, final T val, final int elements, final PutF txn.checkReady(); txn.checkWritesAllowed(); } - final int mask = mask(true, op); + final int mask = mask(op); if (SHOULD_CHECK && !isSet(mask, MDB_MULTIPLE)) { throw new IllegalArgumentException("Must set " + MDB_MULTIPLE + " flag"); } @@ -346,7 +346,7 @@ public T reserve(final T key, final int size, final PutFlags... op) { } kv.keyIn(key); kv.valIn(size); - final int flags = mask(true, op) | MDB_RESERVE.getMask(); + final int flags = mask(op) | MDB_RESERVE.getMask(); checkRc(LIB.mdb_cursor_put(ptrCursor, kv.pointerKey(), kv.pointerVal(), flags)); kv.valOut(); ReferenceUtil.reachabilityFence0(key); diff --git a/src/main/java/org/lmdbjava/Dbi.java b/src/main/java/org/lmdbjava/Dbi.java index c622462b..8560e34c 100644 --- a/src/main/java/org/lmdbjava/Dbi.java +++ b/src/main/java/org/lmdbjava/Dbi.java @@ -72,7 +72,7 @@ public final class Dbi { this.name = name == null ? null : Arrays.copyOf(name, name.length); this.proxy = proxy; this.comparator = comparator; - final int flagsMask = mask(true, flags); + final int flagsMask = mask(flags); final Pointer dbiPtr = allocateDirect(RUNTIME, ADDRESS); checkRc(LIB.mdb_dbi_open(txn.pointer(), name, flagsMask, dbiPtr)); ptr = dbiPtr.getPointer(0); @@ -371,7 +371,7 @@ public boolean put(final Txn txn, final T key, final T val, final PutFlags... } txn.kv().keyIn(key); txn.kv().valIn(val); - final int mask = mask(true, flags); + final int mask = mask(flags); final int rc = LIB.mdb_put(txn.pointer(), ptr, txn.kv().pointerKey(), txn.kv().pointerVal(), mask); if (rc == MDB_KEYEXIST) { @@ -413,7 +413,7 @@ public T reserve(final Txn txn, final T key, final int size, final PutFlags.. } txn.kv().keyIn(key); txn.kv().valIn(size); - final int flags = mask(true, op) | MDB_RESERVE.getMask(); + final int flags = mask(op) | MDB_RESERVE.getMask(); checkRc(LIB.mdb_put(txn.pointer(), ptr, txn.kv().pointerKey(), txn.kv().pointerVal(), flags)); txn.kv().valOut(); // marked as in,out in LMDB C docs ReferenceUtil.reachabilityFence0(key); diff --git a/src/main/java/org/lmdbjava/DbiBuilder.java b/src/main/java/org/lmdbjava/DbiBuilder.java new file mode 100644 index 00000000..a33ca746 --- /dev/null +++ b/src/main/java/org/lmdbjava/DbiBuilder.java @@ -0,0 +1,335 @@ +package org.lmdbjava; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collection; +import java.util.Comparator; +import java.util.EnumSet; +import java.util.Objects; +import java.util.Set; + +/** + * Staged builder for building a {@link Dbi} + * + * @param buffer type + */ +public class DbiBuilder { + + private final Env env; + private final BufferProxy proxy; + private final boolean readOnly; + private byte[] name; + + DbiBuilder(final Env env, + final BufferProxy proxy, + final boolean readOnly) { + this.env = Objects.requireNonNull(env); + this.proxy = Objects.requireNonNull(proxy); + this.readOnly = readOnly; + } + + /** + *

+ * Create the {@link Dbi} with the passed name. + *

+ *

+ * The name will be converted into bytes using {@link StandardCharsets#UTF_8}. + *

+ */ + public RequireComparator withDbName(final String name) { + // Null name is allowed so no null check + final byte[] nameBytes = name == null + ? null + : name.getBytes(StandardCharsets.UTF_8); + return withDbName(nameBytes); + } + + /** + * Create the {@link Dbi} with the passed name in byte[] form. + */ + public RequireComparator withDbName(final byte[] name) { + // Null name is allowed so no null check + this.name = name; + return new RequireComparator<>(this); + } + + /** + *

+ * Create the {@link Dbi} without a name. + *

+ *

+ * Equivalent to passing null to + * {@link DbiBuilder#withDbName(String)} or {@link DbiBuilder#withDbName(byte[])}. + *

+ */ + public RequireComparator withoutDbName() { + return withDbName((byte[]) null); + } + + + // -------------------------------------------------------------------------------- + + + /** + * Intermediate builder stage for constructing a {@link Dbi}. + * + * @param buffer type + */ + public static class RequireComparator { + + private final DbiBuilder dbiBuilder; + + private Comparator comparator; + private boolean useNativeCallback; + + private RequireComparator(final DbiBuilder dbiBuilder) { + this.dbiBuilder = dbiBuilder; + } + + /** + *

+ * {@link CursorIterable} will call down to LMDB's {@code mdb_cmp} method when + * comparing entries to start/stop keys. This ensures LmdbJava is comparing start/stop + * keys using the same comparator that is used for insert order into the db. + *

+ *

+ * This option may be slightly less performant than when using + * {@link RequireComparator#withDefaultJavaComparator()} as it need to call down + * to LMDB to perform the comparisons, however it guarantees that {@link CursorIterable} + * key comparison matches LMDB key comparison. + *

+ *

+ * If you do not intend to use {@link CursorIterable} then it doesn't matter whether + * you choose {@link RequireComparator#withNativeComparator()}, + * {@link RequireComparator#withDefaultJavaComparator()} or + * {@link RequireComparator#withIteratorComparator(Comparator)} as these comparators will + * never be used. + *

+ * + * @return this builder instance. + */ + public FinalStage withNativeComparator() { + this.comparator = null; + this.useNativeCallback = false; + return new FinalStage<>(this); + } + + /** + *

+ * {@link CursorIterable} will make use of the default Java-side comparators when + * comparing entries to start/stop keys. + *

+ *

+ * This option may be slightly more performant than when using + * {@link RequireComparator#withNativeComparator()} but it relies on the default comparator + * in LmdbJava behaving identically to the comparator in LMDB. + *

+ *

+ * If you do not intend to use {@link CursorIterable} then it doesn't matter whether + * you choose {@link RequireComparator#withNativeComparator()}, + * {@link RequireComparator#withDefaultJavaComparator()} or + * {@link RequireComparator#withIteratorComparator(Comparator)} as these comparators will + * never be used. + *

+ * + * @return this builder instance. + */ + public FinalStage withDefaultJavaComparator() { + this.comparator = dbiBuilder.proxy.getUnsignedComparator(); + this.useNativeCallback = false; + return new FinalStage<>(this); + } + + /** + * Provide a java-side {@link Comparator} that LMDB will call back to in order to + * manage database insertion/iteration order. It will also be used for {@link CursorIterable} + * start/stop key comparisons. + *

+ * Due to calling back to java, this will be less performant than using LMDB's + * default comparator, but allows for total control over the order in which entries + * are stored in the database. + *

+ * + * @param comparator for all key comparison operations. + * @return this builder instance. + */ + public FinalStage withCallbackComparator(final Comparator comparator) { + this.comparator = Objects.requireNonNull(comparator); + this.useNativeCallback = true; + return new FinalStage<>(this); + } + + /** + *

+ * {@link CursorIterable} will make use of the passed comparator for + * comparing entries to start/stop keys. It has NO bearing on the insert/iteration + * order of the db. + *

+ *

+ * WARNING: Only call this method if you fully understand the implications + * of using a comparator for the {@link CursorIterable} start/stop keys that behaves + * differently to the comparator in LMDB that controls the insert/iteration order. + *

+ *

+ * This option may be slightly less performant than when using + * {@link RequireComparator#withDefaultJavaComparator()} as it need to call down + * to LMDB to perform the comparisons, however it guarantees that {@link CursorIterable} + * key comparison matches LMDB key comparison. + *

+ *

+ * If you do not intend to use {@link CursorIterable} then it doesn't matter whether + * you choose {@link RequireComparator#withNativeComparator()}, + * {@link RequireComparator#withDefaultJavaComparator()} or + * {@link RequireComparator#withIteratorComparator(Comparator)} as these comparators will + * never be used. + *

+ * + * @param comparator The comparator to use with {@link CursorIterable}. + * @return this builder instance. + */ + public FinalStage withIteratorComparator(final Comparator comparator) { + this.comparator = Objects.requireNonNull(comparator); + this.useNativeCallback = false; + return new FinalStage<>(this); + } + } + + + // -------------------------------------------------------------------------------- + + + /** + * Final stage builder for constructing a {@link Dbi}. + * + * @param buffer type + */ + public static class FinalStage { + + private final RequireComparator requireComparator; + private Set dbiFlags = null; + private Txn txn = null; + + private FinalStage(RequireComparator requireComparator) { + this.requireComparator = requireComparator; + } + + private void initDbiFlags() { + if (dbiFlags == null) { + dbiFlags = EnumSet.noneOf(DbiFlags.class); + } + } + + /** + *

+ * Apply all the dbi flags supplied in dbiFlags. + *

+ *

+ * Replaces any flags applies in previous calls to + * {@link FinalStage#withDbiFlags(Collection)}, {@link FinalStage#withDbiFlags(DbiFlags...)} + * or {@link FinalStage#addDbiFlag(DbiFlags)}. + *

+ * + * @param dbiFlags to open the database with. + */ + public FinalStage withDbiFlags(final Collection dbiFlags) { + initDbiFlags(); + if (dbiFlags != null) { + this.dbiFlags.addAll(dbiFlags); + } + return this; + } + + /** + *

+ * Apply all the dbi flags supplied in dbiFlags. + *

+ *

+ * Replaces any flags applies in previous calls to + * {@link FinalStage#withDbiFlags(Collection)}, {@link FinalStage#withDbiFlags(DbiFlags...)} + * or {@link FinalStage#addDbiFlag(DbiFlags)}. + *

+ * + * @param dbiFlags to open the database with. + */ + public FinalStage withDbiFlags(final DbiFlags... dbiFlags) { + initDbiFlags(); + if (dbiFlags != null) { + Arrays.stream(dbiFlags) + .filter(Objects::nonNull) + .forEach(this.dbiFlags::add); + } + return this; + } + + /** + * Adds dbiFlag to those flags already added to this builder. + * + * @param dbiFlag to open the database with. + * @return this builder instance. + */ + public FinalStage addDbiFlag(final DbiFlags dbiFlag) { + initDbiFlags(); + if (dbiFlags != null) { + this.dbiFlags.add(dbiFlag); + } + return this; + } + + /** + * Use the supplied transaction to open the {@link Dbi}. + *

+ * The caller must commit the transaction after calling {@link FinalStage#open()} + * in order to retain the Dbi in the Env. + *

+ * + * @param txn transaction to use (required; not closed) + * @return this builder instance. + */ + public FinalStage withTxn(final Txn txn) { + this.txn = Objects.requireNonNull(txn); + return this; + } + + /** + * Construct and open the {@link Dbi}. + *

+ * If a {@link Txn} was supplied to the builder, it should be committed upon return from + * this method. + *

+ * + * @return A newly constructed and opened {@link Dbi}. + */ + public Dbi open() { + final DbiBuilder dbiBuilder = requireComparator.dbiBuilder; + if (txn == null) { + try (final Txn txn = getTxn(dbiBuilder)) { + return open(txn, dbiBuilder); + } + } else { + return open(txn, dbiBuilder); + } + } + + private Txn getTxn(final DbiBuilder dbiBuilder) { + return dbiBuilder.readOnly + ? dbiBuilder.env.txnRead() + : dbiBuilder.env.txnWrite(); + } + + private Dbi open(final Txn txn, + final DbiBuilder dbiBuilder) { + final DbiFlags[] dbiFlagsArr = dbiFlags != null && !dbiFlags.isEmpty() + ? this.dbiFlags.toArray(new DbiFlags[0]) + : null; + + return new Dbi<>( + dbiBuilder.env, + txn, + dbiBuilder.name, + requireComparator.comparator, + requireComparator.useNativeCallback, + dbiBuilder.proxy, + dbiFlagsArr); + } + } +} diff --git a/src/main/java/org/lmdbjava/DbiFlags.java b/src/main/java/org/lmdbjava/DbiFlags.java index 2f5eadf6..af6eaeaa 100644 --- a/src/main/java/org/lmdbjava/DbiFlags.java +++ b/src/main/java/org/lmdbjava/DbiFlags.java @@ -31,6 +31,9 @@ public enum DbiFlags implements MaskedFlag { *

Duplicate keys may be used in the database. Or, from another perspective, keys may have * multiple data items, stored in sorted order. By default keys must be unique and may have only a * single data item. + *

+ * + *

*/ MDB_DUPSORT(0x04), /** diff --git a/src/main/java/org/lmdbjava/Env.java b/src/main/java/org/lmdbjava/Env.java index 2e4822b7..d7c7b269 100644 --- a/src/main/java/org/lmdbjava/Env.java +++ b/src/main/java/org/lmdbjava/Env.java @@ -145,7 +145,7 @@ public void close() { public void copy(final File path, final CopyFlags... flags) { requireNonNull(path); validatePath(path); - final int flagsMask = mask(true, flags); + final int flagsMask = mask(flags); checkRc(LIB.mdb_env_copy2(ptr, path.getAbsolutePath(), flagsMask)); } @@ -241,6 +241,15 @@ public boolean isReadOnly() { return readOnly; } + /** + * Open (and optionally creates, if {@link DbiFlags#MDB_CREATE} is set) + * a {@link Dbi} using a builder. + * @return A new builder instance for creating/opening a {@link Dbi}. + */ + public DbiBuilder buildDbi() { + return new DbiBuilder<>(this, proxy, readOnly); + } + /** * Convenience method that opens a {@link Dbi} with a UTF-8 database name and default {@link * Comparator} that is not invoked from native code. @@ -248,7 +257,9 @@ public boolean isReadOnly() { * @param name name of the database (or null if no name is required) * @param flags to open the database with * @return a database that is ready to use + * @deprecated Instead use {@link Env#buildDbi()} */ + @Deprecated() public Dbi openDbi(final String name, final DbiFlags... flags) { final byte[] nameBytes = name == null ? null : name.getBytes(UTF_8); return openDbi(nameBytes, null, false, flags); @@ -268,7 +279,9 @@ public Dbi openDbi(final String name, final DbiFlags... flags) { * comparator will be used. * @param flags to open the database with * @return a database that is ready to use + * @deprecated Instead use {@link Env#buildDbi()} */ + @Deprecated() public Dbi openDbi( final String name, final Comparator comparator, final DbiFlags... flags) { final byte[] nameBytes = name == null ? null : name.getBytes(UTF_8); @@ -288,7 +301,9 @@ public Dbi openDbi( * @param nativeCb whether LMDB native code calls back to the Java comparator * @param flags to open the database with * @return a database that is ready to use + * @deprecated Instead use {@link Env#buildDbi()} */ + @Deprecated() public Dbi openDbi( final String name, final Comparator comparator, @@ -305,7 +320,9 @@ public Dbi openDbi( * @param name name of the database (or null if no name is required) * @param flags to open the database with * @return a database that is ready to use + * @deprecated Instead use {@link Env#buildDbi()} */ + @Deprecated() public Dbi openDbi(final byte[] name, final DbiFlags... flags) { return openDbi(name, null, false, flags); } @@ -318,7 +335,9 @@ public Dbi openDbi(final byte[] name, final DbiFlags... flags) { * @param comparator custom comparator callback (or null to use LMDB default) * @param flags to open the database with * @return a database that is ready to use + * @deprecated Instead use {@link Env#buildDbi()} */ + @Deprecated() public Dbi openDbi( final byte[] name, final Comparator comparator, final DbiFlags... flags) { return openDbi(name, comparator, false, flags); @@ -336,7 +355,9 @@ public Dbi openDbi( * @param nativeCb whether native code calls back to the Java comparator * @param flags to open the database with * @return a database that is ready to use + * @deprecated Instead use {@link Env#buildDbi()} */ + @Deprecated() public Dbi openDbi( final byte[] name, final Comparator comparator, @@ -375,7 +396,9 @@ public Dbi openDbi( * @param nativeCb whether native code should call back to the comparator * @param flags to open the database with * @return a database that is ready to use + * @deprecated Instead use {@link Env#buildDbi()} */ + @Deprecated() public Dbi openDbi( final Txn txn, final byte[] name, @@ -557,7 +580,7 @@ public Env open(final File path, final int mode, final EnvFlags... flags) { checkRc(LIB.mdb_env_set_mapsize(ptr, mapSize)); checkRc(LIB.mdb_env_set_maxdbs(ptr, maxDbs)); checkRc(LIB.mdb_env_set_maxreaders(ptr, maxReaders)); - final int flagsMask = mask(true, flags); + final int flagsMask = mask(flags); final boolean readOnly = isSet(flagsMask, MDB_RDONLY_ENV); final boolean noSubDir = isSet(flagsMask, MDB_NOSUBDIR); checkRc(LIB.mdb_env_open(ptr, path.getAbsolutePath(), flagsMask, mode)); diff --git a/src/main/java/org/lmdbjava/MaskedFlag.java b/src/main/java/org/lmdbjava/MaskedFlag.java index 00556ecb..4dc47b20 100644 --- a/src/main/java/org/lmdbjava/MaskedFlag.java +++ b/src/main/java/org/lmdbjava/MaskedFlag.java @@ -17,11 +17,6 @@ import static java.util.Objects.requireNonNull; -import java.util.Arrays; -import java.util.Objects; -import java.util.function.Predicate; -import java.util.stream.Stream; - /** Indicates an enum that can provide integers for each of its values. */ public interface MaskedFlag { @@ -32,15 +27,6 @@ public interface MaskedFlag { */ int getMask(); - /** - * Indicates if the flag must be propagated to the underlying C code of LMDB or not. - * - * @return the boolean value indicating the propagation - */ - default boolean isPropagatedToLmdb() { - return true; - } - /** * Fetch the integer mask for all presented flags. * @@ -50,54 +36,18 @@ default boolean isPropagatedToLmdb() { */ @SafeVarargs static int mask(final M... flags) { - return mask(false, flags); - } - - /** - * Fetch the integer mask for all presented flags. - * - * @param flag type - * @param flags to mask (null or empty returns zero) - * @return the integer mask for use in C - */ - static int mask(final Stream flags) { - return mask(false, flags); - } - - /** - * Fetch the integer mask for the presented flags. - * - * @param flag type - * @param onlyPropagatedToLmdb if to include only the flags which are also propagate to the C code - * or all of them - * @param flags to mask (null or empty returns zero) - * @return the integer mask for use in C - */ - @SafeVarargs - static int mask(final boolean onlyPropagatedToLmdb, final M... flags) { - return flags == null ? 0 : mask(onlyPropagatedToLmdb, Arrays.stream(flags)); - } - - /** - * Fetch the integer mask for all presented flags. - * - * @param flag type - * @param onlyPropagatedToLmdb if to include only the flags which are also propagate to the C code - * or all of them - * @param flags to mask - * @return the integer mask for use in C - */ - static int mask( - final boolean onlyPropagatedToLmdb, final Stream flags) { - final Predicate filter = onlyPropagatedToLmdb ? MaskedFlag::isPropagatedToLmdb : f -> true; + if (flags == null || flags.length == 0) { + return 0; + } - return flags == null - ? 0 - : flags - .filter(Objects::nonNull) - .filter(filter) - .map(M::getMask) - .reduce(0, (f1, f2) -> f1 | f2); + int result = 0; + for (MaskedFlag flag : flags) { + if (flag == null) { + continue; + } + result |= flag.getMask(); + } + return result; } /** diff --git a/src/main/java/org/lmdbjava/Txn.java b/src/main/java/org/lmdbjava/Txn.java index 05e8ce06..1d5d4860 100644 --- a/src/main/java/org/lmdbjava/Txn.java +++ b/src/main/java/org/lmdbjava/Txn.java @@ -49,7 +49,7 @@ public final class Txn implements AutoCloseable { Txn(final Env env, final Txn parent, final BufferProxy proxy, final TxnFlags... flags) { this.proxy = proxy; this.keyVal = proxy.keyVal(); - final int flagsMask = mask(true, flags); + final int flagsMask = mask(flags); this.readOnly = isSet(flagsMask, MDB_RDONLY_TXN); if (env.isReadOnly() && !this.readOnly) { throw new EnvIsReadOnly(); From bfbf223031191e0bb13c7c32a58930c85e880f5c Mon Sep 17 00:00:00 2001 From: at055612 <22818309+at055612@users.noreply.github.com> Date: Mon, 27 Oct 2025 15:23:46 +0000 Subject: [PATCH 08/21] Add FlagSet, DbiFlagSet, PutFlagSet Refactor DbiBuilder and Dbi ctor to use DbiFlagSet. --- src/main/java/org/lmdbjava/Dbi.java | 29 ++- src/main/java/org/lmdbjava/DbiBuilder.java | 42 ++-- src/main/java/org/lmdbjava/DbiFlagSet.java | 32 +++ src/main/java/org/lmdbjava/Env.java | 2 +- src/main/java/org/lmdbjava/FlagSet.java | 192 ++++++++++++++++++ src/main/java/org/lmdbjava/MaskedFlag.java | 38 +++- src/main/java/org/lmdbjava/PutFlagSet.java | 32 +++ .../java/org/lmdbjava/DbiFlagSetTest.java | 101 +++++++++ .../java/org/lmdbjava/PutFlagSetTest.java | 101 +++++++++ 9 files changed, 531 insertions(+), 38 deletions(-) create mode 100644 src/main/java/org/lmdbjava/DbiFlagSet.java create mode 100644 src/main/java/org/lmdbjava/FlagSet.java create mode 100644 src/main/java/org/lmdbjava/PutFlagSet.java create mode 100644 src/test/java/org/lmdbjava/DbiFlagSetTest.java create mode 100644 src/test/java/org/lmdbjava/PutFlagSetTest.java diff --git a/src/main/java/org/lmdbjava/Dbi.java b/src/main/java/org/lmdbjava/Dbi.java index ba20c1a4..17fa4c46 100644 --- a/src/main/java/org/lmdbjava/Dbi.java +++ b/src/main/java/org/lmdbjava/Dbi.java @@ -31,6 +31,7 @@ import static org.lmdbjava.PutFlags.MDB_RESERVE; import static org.lmdbjava.ResultCodeMapper.checkRc; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; @@ -55,6 +56,7 @@ public final class Dbi { private final byte[] name; private final Pointer ptr; private final BufferProxy proxy; + private final DbiFlagSet dbiFlagSet; Dbi( final Env env, @@ -63,7 +65,7 @@ public final class Dbi { final Comparator comparator, final boolean nativeCb, final BufferProxy proxy, - final DbiFlags... flags) { + final DbiFlagSet dbiFlagSet) { if (SHOULD_CHECK) { requireNonNull(txn); txn.checkReady(); @@ -72,9 +74,9 @@ public final class Dbi { this.name = name == null ? null : Arrays.copyOf(name, name.length); this.proxy = proxy; this.comparator = comparator; - final int flagsMask = mask(flags); + this.dbiFlagSet = dbiFlagSet; final Pointer dbiPtr = allocateDirect(RUNTIME, ADDRESS); - checkRc(LIB.mdb_dbi_open(txn.pointer(), name, flagsMask, dbiPtr)); + checkRc(LIB.mdb_dbi_open(txn.pointer(), name, dbiFlagSet.getMask(), dbiPtr)); ptr = dbiPtr.getPointer(0); if (nativeCb) { requireNonNull(comparator, "comparator cannot be null if nativeCb is set"); @@ -291,6 +293,7 @@ public CursorIterable iterate(final Txn txn, final KeyRange range) { * @return the list of flags this Dbi was created with */ public List listFlags(final Txn txn) { + // TODO we could just return what is in dbiFlagSet, rather than hitting LMDB. if (SHOULD_CHECK) { env.checkNotClosed(); } @@ -457,6 +460,26 @@ private void clean() { cleaned = true; } + private String getNameAsString() { + if (name == null) { + return ""; + } else { + try { + return new String(name, StandardCharsets.UTF_8); + } catch (Exception e) { + return "?"; + } + } + } + + @Override + public String toString() { + return "Dbi{" + + "name=" + getNameAsString() + + ", dbiFlagSet=" + dbiFlagSet + + '}'; + } + /** The specified DBI was changed unexpectedly. */ public static final class BadDbiException extends LmdbNativeException { diff --git a/src/main/java/org/lmdbjava/DbiBuilder.java b/src/main/java/org/lmdbjava/DbiBuilder.java index 5deb2e38..1803d142 100644 --- a/src/main/java/org/lmdbjava/DbiBuilder.java +++ b/src/main/java/org/lmdbjava/DbiBuilder.java @@ -19,9 +19,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Comparator; -import java.util.EnumSet; import java.util.Objects; -import java.util.Set; /** * Staged builder for building a {@link Dbi} @@ -224,19 +222,13 @@ public DbiBuilderStage3 withIteratorComparator(final Comparator comparator public static class DbiBuilderStage3 { private final DbiBuilderStage2 dbiBuilderStage2; - private Set dbiFlags = null; + private final FlagSet.Builder flagSetBuilder = DbiFlagSet.builder(); private Txn txn = null; private DbiBuilderStage3(DbiBuilderStage2 dbiBuilderStage2) { this.dbiBuilderStage2 = dbiBuilderStage2; } - private void initDbiFlags() { - if (dbiFlags == null) { - dbiFlags = EnumSet.noneOf(DbiFlags.class); - } - } - /** *

* Apply all the dbi flags supplied in dbiFlags. @@ -244,15 +236,14 @@ private void initDbiFlags() { *

* Replaces any flags applies in previous calls to * {@link DbiBuilderStage3#withDbiFlags(Collection)}, {@link DbiBuilderStage3#withDbiFlags(DbiFlags...)} - * or {@link DbiBuilderStage3#addDbiFlag(DbiFlags)}. + * or {@link DbiBuilderStage3#setDbiFlag(DbiFlags)}. *

* * @param dbiFlags to open the database with. */ public DbiBuilderStage3 withDbiFlags(final Collection dbiFlags) { - initDbiFlags(); if (dbiFlags != null) { - this.dbiFlags.stream() + dbiFlags.stream() .filter(Objects::nonNull) .forEach(dbiFlags::add); } @@ -265,36 +256,35 @@ public DbiBuilderStage3 withDbiFlags(final Collection dbiFlags) { *

*

* Replaces any flags applies in previous calls to - * {@link DbiBuilderStage3#withDbiFlags(Collection)}, {@link DbiBuilderStage3#withDbiFlags(DbiFlags...)} - * or {@link DbiBuilderStage3#addDbiFlag(DbiFlags)}. + * {@link DbiBuilderStage3#withDbiFlags(Collection)}, + * {@link DbiBuilderStage3#withDbiFlags(DbiFlags...)} + * or {@link DbiBuilderStage3#setDbiFlag(DbiFlags)}. *

* * @param dbiFlags to open the database with. * A null array is a no-op. Null items are ignored. */ public DbiBuilderStage3 withDbiFlags(final DbiFlags... dbiFlags) { - initDbiFlags(); + flagSetBuilder.clear(); if (dbiFlags != null) { Arrays.stream(dbiFlags) .filter(Objects::nonNull) - .forEach(this.dbiFlags::add); + .forEach(this.flagSetBuilder::setFlag); } return this; } /** * Adds dbiFlag to those flags already added to this builder by - * {@link DbiBuilderStage3#withDbiFlags(DbiFlags...)}, {@link DbiBuilderStage3#withDbiFlags(Collection)} - * or {@link DbiBuilderStage3#addDbiFlag(DbiFlags)}. + * {@link DbiBuilderStage3#withDbiFlags(DbiFlags...)}, + * {@link DbiBuilderStage3#withDbiFlags(Collection)} + * or {@link DbiBuilderStage3#setDbiFlag(DbiFlags)}. * * @param dbiFlag to open the database with. A null value is a no-op. * @return this builder instance. */ - public DbiBuilderStage3 addDbiFlag(final DbiFlags dbiFlag) { - initDbiFlags(); - if (dbiFlags != null) { - this.dbiFlags.add(dbiFlag); - } + public DbiBuilderStage3 setDbiFlag(final DbiFlags dbiFlag) { + this.flagSetBuilder.setFlag(dbiFlag); return this; } @@ -341,9 +331,7 @@ private Txn getTxn(final DbiBuilder dbiBuilder) { private Dbi open(final Txn txn, final DbiBuilder dbiBuilder) { - final DbiFlags[] dbiFlagsArr = dbiFlags != null && !dbiFlags.isEmpty() - ? this.dbiFlags.toArray(new DbiFlags[0]) - : null; + final DbiFlagSet dbiFlagSet = flagSetBuilder.build(); return new Dbi<>( dbiBuilder.env, @@ -352,7 +340,7 @@ private Dbi open(final Txn txn, dbiBuilderStage2.comparator, dbiBuilderStage2.useNativeCallback, dbiBuilder.proxy, - dbiFlagsArr); + dbiFlagSet); } } } diff --git a/src/main/java/org/lmdbjava/DbiFlagSet.java b/src/main/java/org/lmdbjava/DbiFlagSet.java new file mode 100644 index 00000000..2c6c5ac7 --- /dev/null +++ b/src/main/java/org/lmdbjava/DbiFlagSet.java @@ -0,0 +1,32 @@ +package org.lmdbjava; + +import java.util.EnumSet; +import java.util.Objects; + +public class DbiFlagSet extends FlagSet { + + public static final DbiFlagSet EMPTY = new DbiFlagSet(EnumSet.noneOf(DbiFlags.class)); + + private DbiFlagSet(final EnumSet flags) { + super(flags); + } + + public static DbiFlagSet empty() { + return EMPTY; + } + + public static DbiFlagSet of(final DbiFlags putFlag) { + Objects.requireNonNull(putFlag); + return new DbiFlagSet(EnumSet.of(putFlag)); + } + + public static DbiFlagSet of(final DbiFlags... DbiFlags) { + return builder() + .withFlags(DbiFlags) + .build(); + } + + public static Builder builder() { + return new Builder<>(DbiFlags.class, DbiFlagSet::new); + } +} diff --git a/src/main/java/org/lmdbjava/Env.java b/src/main/java/org/lmdbjava/Env.java index d7c7b269..5de0a7fd 100644 --- a/src/main/java/org/lmdbjava/Env.java +++ b/src/main/java/org/lmdbjava/Env.java @@ -405,7 +405,7 @@ public Dbi openDbi( final Comparator comparator, final boolean nativeCb, final DbiFlags... flags) { - return new Dbi<>(this, txn, name, comparator, nativeCb, proxy, flags); + return new Dbi<>(this, txn, name, comparator, nativeCb, proxy, DbiFlagSet.of(flags)); } /** diff --git a/src/main/java/org/lmdbjava/FlagSet.java b/src/main/java/org/lmdbjava/FlagSet.java new file mode 100644 index 00000000..21132f8c --- /dev/null +++ b/src/main/java/org/lmdbjava/FlagSet.java @@ -0,0 +1,192 @@ +package org.lmdbjava; + +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.EnumSet; +import java.util.Iterator; +import java.util.Objects; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * Encapsulates an immutable set of flags and the associated bit mask for the flags in the set. + * + * @param + */ +public abstract class FlagSet & MaskedFlag> implements Iterable { + + private final Set flags; + private final int mask; + + protected FlagSet(final EnumSet flags) { + Objects.requireNonNull(flags); + this.mask = MaskedFlag.mask(flags); + this.flags = Collections.unmodifiableSet(Objects.requireNonNull(flags)); + } + + /** + * @return THe combined bit mask for all flags in the set. + */ + int getMask() { + return mask; + } + + /** + * @return All flags in the set. + */ + public Set getFlags() { + return flags; + } + + /** + * @return True if flag has been set, i.e. is contained in this set. + */ + public boolean isSet(final T flag) { + return flag != null + && flags.contains(flag); + } + + /** + * @return The number of flags in this set. + */ + public int size() { + return flags.size(); + } + + /** + * @return True if this set is empty. + */ + public boolean isEmpty() { + return flags.isEmpty(); + } + + /** + * @return The {@link Iterator} for this set. + */ + @Override + public Iterator iterator() { + return flags.iterator(); + } + + @Override + public boolean equals(Object object) { + if (this == object) return true; + if (object == null || getClass() != object.getClass()) return false; + FlagSet flagSet = (FlagSet) object; + return mask == flagSet.mask && Objects.equals(flags, flagSet.flags); + } + + @Override + public int hashCode() { + return Objects.hash(flags, mask); + } + + @Override + public String toString() { + final String flagsStr = flags.stream() + .sorted(Comparator.comparing(MaskedFlag::getMask)) + .map(MaskedFlag::name) + .collect(Collectors.joining(", ")); + return "FlagSet{" + + "flags=[" + flagsStr + + "], mask=" + mask + + '}'; + } + + + // -------------------------------------------------------------------------------- + + + /** + * A builder for creating a {@link FlagSet}. + * + * @param The type of flag to be held in the {@link FlagSet} + * @param The type of the {@link FlagSet} implementation. + */ + public static class Builder & MaskedFlag, S extends FlagSet> { + + final Class type; + final EnumSet enumSet; + final Function, S> constructor; + + protected Builder(final Class type, + final Function, S> constructor) { + this.type = type; + this.enumSet = EnumSet.noneOf(type); + this.constructor = constructor; + } + + /** + * Replaces any flags already set in the builder with the contents of the passed flags {@link Collection} + * + * @param flags The flags to set in the builder. + * @return this builder instance. + */ + public Builder withFlags(final Collection flags) { + enumSet.clear(); + if (flags != null) { + for (E flag : flags) { + if (flag != null) { + enumSet.add(flag); + } + } + } + return this; + } + + /** + * @param flags The flags to set in the builder. + * @return this builder instance. + */ + @SafeVarargs + public final Builder withFlags(final E... flags) { + enumSet.clear(); + if (flags != null) { + for (E flag : flags) { + if (flag != null) { + if (!type.equals(flag.getClass())) { + throw new IllegalArgumentException("Unexpected type " + flag.getClass()); + } + enumSet.add(flag); + } + } + } + return this; + } + + /** + * Sets a single flag in the builder. + * + * @param flag The flag to set in the builder. + * @return this builder instance. + */ + public Builder setFlag(final E flag) { + if (flag != null) { + enumSet.add(flag); + } + return this; + } + + /** + * Clears any flags already set in this {@link Builder} + * + * @return this builder instance. + */ + public Builder clear() { + enumSet.clear(); + return this; + } + + /** + * Build the {@link DbiFlagSet} + * + * @return A + */ + public S build() { + return constructor.apply(enumSet); + } + } +} + diff --git a/src/main/java/org/lmdbjava/MaskedFlag.java b/src/main/java/org/lmdbjava/MaskedFlag.java index 4dc47b20..f2f08274 100644 --- a/src/main/java/org/lmdbjava/MaskedFlag.java +++ b/src/main/java/org/lmdbjava/MaskedFlag.java @@ -17,9 +17,13 @@ import static java.util.Objects.requireNonNull; +import java.util.Collection; + /** Indicates an enum that can provide integers for each of its values. */ public interface MaskedFlag { + int EMPTY_MASK = 0; + /** * Obtains the integer value for this enum which can be included in a mask. * @@ -27,6 +31,11 @@ public interface MaskedFlag { */ int getMask(); + /** + * @return The name of the flag. + */ + String name(); + /** * Fetch the integer mask for all presented flags. * @@ -37,17 +46,32 @@ public interface MaskedFlag { @SafeVarargs static int mask(final M... flags) { if (flags == null || flags.length == 0) { - return 0; + return EMPTY_MASK; + } else { + int result = EMPTY_MASK; + for (MaskedFlag flag : flags) { + if (flag == null) { + continue; + } + result |= flag.getMask(); + } + return result; } + } - int result = 0; - for (MaskedFlag flag : flags) { - if (flag == null) { - continue; + static int mask(final Collection flags) { + if (flags == null || flags.isEmpty()) { + return EMPTY_MASK; + } else { + int result = EMPTY_MASK; + for (MaskedFlag flag : flags) { + if (flag == null) { + continue; + } + result |= flag.getMask(); } - result |= flag.getMask(); + return result; } - return result; } /** diff --git a/src/main/java/org/lmdbjava/PutFlagSet.java b/src/main/java/org/lmdbjava/PutFlagSet.java new file mode 100644 index 00000000..290f9729 --- /dev/null +++ b/src/main/java/org/lmdbjava/PutFlagSet.java @@ -0,0 +1,32 @@ +package org.lmdbjava; + +import java.util.EnumSet; +import java.util.Objects; + +public class PutFlagSet extends FlagSet { + + public static final PutFlagSet EMPTY = new PutFlagSet(EnumSet.noneOf(PutFlags.class)); + + private PutFlagSet(final EnumSet flags) { + super(flags); + } + + public static PutFlagSet empty() { + return EMPTY; + } + + public static PutFlagSet of(final PutFlags putFlag) { + Objects.requireNonNull(putFlag); + return new org.lmdbjava.PutFlagSet(EnumSet.of(putFlag)); + } + + public static PutFlagSet of(final PutFlags... putFlags) { + return builder() + .withFlags(putFlags) + .build(); + } + + public static Builder builder() { + return new Builder<>(PutFlags.class, PutFlagSet::new); + } +} diff --git a/src/test/java/org/lmdbjava/DbiFlagSetTest.java b/src/test/java/org/lmdbjava/DbiFlagSetTest.java new file mode 100644 index 00000000..cfbda600 --- /dev/null +++ b/src/test/java/org/lmdbjava/DbiFlagSetTest.java @@ -0,0 +1,101 @@ +package org.lmdbjava; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +import java.util.Arrays; +import java.util.HashSet; +import org.junit.Test; + +public class DbiFlagSetTest { + + @Test + public void testEmpty() { + final DbiFlagSet putFlagSet = DbiFlagSet.empty(); + assertThat( + putFlagSet.getMask(), + is(0)); + assertThat( + putFlagSet.size(), + is(0)); + assertThat( + putFlagSet.isEmpty(), + is(true)); + assertThat( + putFlagSet.isSet(DbiFlags.MDB_REVERSEDUP), + is(false)); + } + + @Test + public void testOf() { + final DbiFlags putFlag = DbiFlags.MDB_CREATE; + final DbiFlagSet putFlagSet = DbiFlagSet.of(putFlag); + assertThat( + putFlagSet.getMask(), + is(MaskedFlag.mask(putFlag))); + assertThat( + putFlagSet.size(), + is(1)); + assertThat( + putFlagSet.isSet(DbiFlags.MDB_REVERSEDUP), + is(false)); + for (DbiFlags flag : putFlagSet) { + assertThat( + putFlagSet.isSet(flag), + is(true)); + } + } + + @Test + public void testOf2() { + final DbiFlags putFlag1 = DbiFlags.MDB_CREATE; + final DbiFlags putFlag2 = DbiFlags.MDB_INTEGERKEY; + final DbiFlagSet putFlagSet = DbiFlagSet.of(putFlag1, putFlag2); + assertThat( + putFlagSet.getMask(), + is(MaskedFlag.mask(putFlag1, putFlag2))); + assertThat( + putFlagSet.size(), + is(2)); + assertThat( + putFlagSet.isSet(DbiFlags.MDB_REVERSEDUP), + is(false)); + for (DbiFlags flag : putFlagSet) { + assertThat( + putFlagSet.isSet(flag), + is(true)); + } + } + + @Test + public void testBuilder() { + final DbiFlags putFlag1 = DbiFlags.MDB_CREATE; + final DbiFlags putFlag2 = DbiFlags.MDB_INTEGERKEY; + final DbiFlagSet putFlagSet = DbiFlagSet.builder() + .setFlag(putFlag1) + .setFlag(putFlag2) + .build(); + assertThat( + putFlagSet.getMask(), + is(MaskedFlag.mask(putFlag1, putFlag2))); + assertThat( + putFlagSet.size(), + is(2)); + assertThat( + putFlagSet.isSet(DbiFlags.MDB_REVERSEDUP), + is(false)); + for (DbiFlags flag : putFlagSet) { + assertThat( + putFlagSet.isSet(flag), + is(true)); + } + final DbiFlagSet putFlagSet2 = DbiFlagSet.builder() + .withFlags(putFlag1, putFlag2) + .build(); + final DbiFlagSet putFlagSet3 = DbiFlagSet.builder() + .withFlags(new HashSet<>(Arrays.asList(putFlag1, putFlag2))) + .build(); + assertThat(putFlagSet, is(putFlagSet2)); + assertThat(putFlagSet, is(putFlagSet3)); + } +} diff --git a/src/test/java/org/lmdbjava/PutFlagSetTest.java b/src/test/java/org/lmdbjava/PutFlagSetTest.java new file mode 100644 index 00000000..4826f436 --- /dev/null +++ b/src/test/java/org/lmdbjava/PutFlagSetTest.java @@ -0,0 +1,101 @@ +package org.lmdbjava; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +import java.util.Arrays; +import java.util.HashSet; +import org.junit.Test; + +public class PutFlagSetTest { + + @Test + public void testEmpty() { + final PutFlagSet putFlagSet = PutFlagSet.empty(); + assertThat( + putFlagSet.getMask(), + is(0)); + assertThat( + putFlagSet.size(), + is(0)); + assertThat( + putFlagSet.isEmpty(), + is(true)); + assertThat( + putFlagSet.isSet(PutFlags.MDB_MULTIPLE), + is(false)); + } + + @Test + public void testOf() { + final PutFlags putFlag = PutFlags.MDB_APPEND; + final PutFlagSet putFlagSet = PutFlagSet.of(putFlag); + assertThat( + putFlagSet.getMask(), + is(MaskedFlag.mask(putFlag))); + assertThat( + putFlagSet.size(), + is(1)); + assertThat( + putFlagSet.isSet(PutFlags.MDB_MULTIPLE), + is(false)); + for (PutFlags flag : putFlagSet) { + assertThat( + putFlagSet.isSet(flag), + is(true)); + } + } + + @Test + public void testOf2() { + final PutFlags putFlag1 = PutFlags.MDB_APPEND; + final PutFlags putFlag2 = PutFlags.MDB_NOOVERWRITE; + final PutFlagSet putFlagSet = PutFlagSet.of(putFlag1, putFlag2); + assertThat( + putFlagSet.getMask(), + is(MaskedFlag.mask(putFlag1, putFlag2))); + assertThat( + putFlagSet.size(), + is(2)); + assertThat( + putFlagSet.isSet(PutFlags.MDB_MULTIPLE), + is(false)); + for (PutFlags flag : putFlagSet) { + assertThat( + putFlagSet.isSet(flag), + is(true)); + } + } + + @Test + public void testBuilder() { + final PutFlags putFlag1 = PutFlags.MDB_APPEND; + final PutFlags putFlag2 = PutFlags.MDB_NOOVERWRITE; + final PutFlagSet putFlagSet = PutFlagSet.builder() + .setFlag(putFlag1) + .setFlag(putFlag2) + .build(); + assertThat( + putFlagSet.getMask(), + is(MaskedFlag.mask(putFlag1, putFlag2))); + assertThat( + putFlagSet.size(), + is(2)); + assertThat( + putFlagSet.isSet(PutFlags.MDB_MULTIPLE), + is(false)); + for (PutFlags flag : putFlagSet) { + assertThat( + putFlagSet.isSet(flag), + is(true)); + } + final PutFlagSet putFlagSet2 = PutFlagSet.builder() + .withFlags(putFlag1, putFlag2) + .build(); + final PutFlagSet putFlagSet3 = PutFlagSet.builder() + .withFlags(new HashSet<>(Arrays.asList(putFlag1, putFlag2))) + .build(); + assertThat(putFlagSet, is(putFlagSet2)); + assertThat(putFlagSet, is(putFlagSet3)); + } +} From 0f66aaf7021ebb25a295dc95ba56a8c78c6370d5 Mon Sep 17 00:00:00 2001 From: at055612 <22818309+at055612@users.noreply.github.com> Date: Mon, 27 Oct 2025 15:27:35 +0000 Subject: [PATCH 09/21] Rename FlagSet to AbstractFlagSet --- .../{FlagSet.java => AbstractFlagSet.java} | 14 +++++++------- src/main/java/org/lmdbjava/DbiBuilder.java | 2 +- src/main/java/org/lmdbjava/DbiFlagSet.java | 2 +- src/main/java/org/lmdbjava/PutFlagSet.java | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) rename src/main/java/org/lmdbjava/{FlagSet.java => AbstractFlagSet.java} (90%) diff --git a/src/main/java/org/lmdbjava/FlagSet.java b/src/main/java/org/lmdbjava/AbstractFlagSet.java similarity index 90% rename from src/main/java/org/lmdbjava/FlagSet.java rename to src/main/java/org/lmdbjava/AbstractFlagSet.java index 21132f8c..3c21fb15 100644 --- a/src/main/java/org/lmdbjava/FlagSet.java +++ b/src/main/java/org/lmdbjava/AbstractFlagSet.java @@ -15,12 +15,12 @@ * * @param */ -public abstract class FlagSet & MaskedFlag> implements Iterable { +public abstract class AbstractFlagSet & MaskedFlag> implements Iterable { private final Set flags; private final int mask; - protected FlagSet(final EnumSet flags) { + protected AbstractFlagSet(final EnumSet flags) { Objects.requireNonNull(flags); this.mask = MaskedFlag.mask(flags); this.flags = Collections.unmodifiableSet(Objects.requireNonNull(flags)); @@ -74,7 +74,7 @@ public Iterator iterator() { public boolean equals(Object object) { if (this == object) return true; if (object == null || getClass() != object.getClass()) return false; - FlagSet flagSet = (FlagSet) object; + AbstractFlagSet flagSet = (AbstractFlagSet) object; return mask == flagSet.mask && Objects.equals(flags, flagSet.flags); } @@ -100,12 +100,12 @@ public String toString() { /** - * A builder for creating a {@link FlagSet}. + * A builder for creating a {@link AbstractFlagSet}. * - * @param The type of flag to be held in the {@link FlagSet} - * @param The type of the {@link FlagSet} implementation. + * @param The type of flag to be held in the {@link AbstractFlagSet} + * @param The type of the {@link AbstractFlagSet} implementation. */ - public static class Builder & MaskedFlag, S extends FlagSet> { + public static class Builder & MaskedFlag, S extends AbstractFlagSet> { final Class type; final EnumSet enumSet; diff --git a/src/main/java/org/lmdbjava/DbiBuilder.java b/src/main/java/org/lmdbjava/DbiBuilder.java index 1803d142..7d05a51e 100644 --- a/src/main/java/org/lmdbjava/DbiBuilder.java +++ b/src/main/java/org/lmdbjava/DbiBuilder.java @@ -222,7 +222,7 @@ public DbiBuilderStage3 withIteratorComparator(final Comparator comparator public static class DbiBuilderStage3 { private final DbiBuilderStage2 dbiBuilderStage2; - private final FlagSet.Builder flagSetBuilder = DbiFlagSet.builder(); + private final AbstractFlagSet.Builder flagSetBuilder = DbiFlagSet.builder(); private Txn txn = null; private DbiBuilderStage3(DbiBuilderStage2 dbiBuilderStage2) { diff --git a/src/main/java/org/lmdbjava/DbiFlagSet.java b/src/main/java/org/lmdbjava/DbiFlagSet.java index 2c6c5ac7..cd1db934 100644 --- a/src/main/java/org/lmdbjava/DbiFlagSet.java +++ b/src/main/java/org/lmdbjava/DbiFlagSet.java @@ -3,7 +3,7 @@ import java.util.EnumSet; import java.util.Objects; -public class DbiFlagSet extends FlagSet { +public class DbiFlagSet extends AbstractFlagSet { public static final DbiFlagSet EMPTY = new DbiFlagSet(EnumSet.noneOf(DbiFlags.class)); diff --git a/src/main/java/org/lmdbjava/PutFlagSet.java b/src/main/java/org/lmdbjava/PutFlagSet.java index 290f9729..8820fe92 100644 --- a/src/main/java/org/lmdbjava/PutFlagSet.java +++ b/src/main/java/org/lmdbjava/PutFlagSet.java @@ -3,7 +3,7 @@ import java.util.EnumSet; import java.util.Objects; -public class PutFlagSet extends FlagSet { +public class PutFlagSet extends AbstractFlagSet { public static final PutFlagSet EMPTY = new PutFlagSet(EnumSet.noneOf(PutFlags.class)); From aa000a1389755dc8f2de4152d149fa6da09b6d22 Mon Sep 17 00:00:00 2001 From: at055612 <22818309+at055612@users.noreply.github.com> Date: Mon, 27 Oct 2025 21:19:27 +0000 Subject: [PATCH 10/21] Add remaining FlagSet impls Replace Env#copy(File, CopyFlags...) with copy(File, CopyFlagSet). As there is only one flag this should not be a breaking change. Deprecate Env#txn(Txn, TxnFlags...) as there is now Env#txn(Txn) Env#txn(Txn, TxnFlags) Env#txn(Txn, TxnFlagSet) --- .../java/org/lmdbjava/AbstractFlagSet.java | 186 +++++++++++++++--- src/main/java/org/lmdbjava/CopyFlagSet.java | 59 ++++++ src/main/java/org/lmdbjava/CopyFlags.java | 30 ++- src/main/java/org/lmdbjava/DbiFlagSet.java | 63 ++++-- src/main/java/org/lmdbjava/DbiFlags.java | 30 ++- src/main/java/org/lmdbjava/Env.java | 77 +++++++- src/main/java/org/lmdbjava/EnvFlagSet.java | 57 ++++++ src/main/java/org/lmdbjava/EnvFlags.java | 30 ++- src/main/java/org/lmdbjava/FlagSet.java | 61 ++++++ src/main/java/org/lmdbjava/PutFlagSet.java | 53 +++-- src/main/java/org/lmdbjava/PutFlags.java | 30 ++- src/main/java/org/lmdbjava/Txn.java | 12 +- src/main/java/org/lmdbjava/TxnFlagSet.java | 63 ++++++ src/main/java/org/lmdbjava/TxnFlags.java | 30 ++- .../java/org/lmdbjava/CopyFlagSetTest.java | 88 +++++++++ .../java/org/lmdbjava/DbiFlagSetTest.java | 93 +++++---- .../java/org/lmdbjava/EnvFlagSetTest.java | 116 +++++++++++ .../java/org/lmdbjava/PutFlagSetTest.java | 15 ++ .../java/org/lmdbjava/TxnFlagSetTest.java | 88 +++++++++ 19 files changed, 1066 insertions(+), 115 deletions(-) create mode 100644 src/main/java/org/lmdbjava/CopyFlagSet.java create mode 100644 src/main/java/org/lmdbjava/EnvFlagSet.java create mode 100644 src/main/java/org/lmdbjava/FlagSet.java create mode 100644 src/main/java/org/lmdbjava/TxnFlagSet.java create mode 100644 src/test/java/org/lmdbjava/CopyFlagSetTest.java create mode 100644 src/test/java/org/lmdbjava/EnvFlagSetTest.java create mode 100644 src/test/java/org/lmdbjava/TxnFlagSetTest.java diff --git a/src/main/java/org/lmdbjava/AbstractFlagSet.java b/src/main/java/org/lmdbjava/AbstractFlagSet.java index 3c21fb15..7ea413fb 100644 --- a/src/main/java/org/lmdbjava/AbstractFlagSet.java +++ b/src/main/java/org/lmdbjava/AbstractFlagSet.java @@ -2,20 +2,19 @@ import java.util.Collection; import java.util.Collections; -import java.util.Comparator; import java.util.EnumSet; import java.util.Iterator; import java.util.Objects; import java.util.Set; import java.util.function.Function; -import java.util.stream.Collectors; +import java.util.function.Supplier; /** * Encapsulates an immutable set of flags and the associated bit mask for the flags in the set. * * @param */ -public abstract class AbstractFlagSet & MaskedFlag> implements Iterable { +public abstract class AbstractFlagSet & MaskedFlag> implements FlagSet { private final Set flags; private final int mask; @@ -29,13 +28,15 @@ protected AbstractFlagSet(final EnumSet flags) { /** * @return THe combined bit mask for all flags in the set. */ - int getMask() { + @Override + public int getMask() { return mask; } /** * @return All flags in the set. */ + @Override public Set getFlags() { return flags; } @@ -43,14 +44,18 @@ public Set getFlags() { /** * @return True if flag has been set, i.e. is contained in this set. */ + @Override public boolean isSet(final T flag) { + // Probably cheaper to compare the masks than to use EnumSet.contains() return flag != null - && flags.contains(flag); + && MaskedFlag.isSet(mask, flag); + } /** * @return The number of flags in this set. */ + @Override public int size() { return flags.size(); } @@ -58,6 +63,7 @@ public int size() { /** * @return True if this set is empty. */ + @Override public boolean isEmpty() { return flags.isEmpty(); } @@ -73,9 +79,8 @@ public Iterator iterator() { @Override public boolean equals(Object object) { if (this == object) return true; - if (object == null || getClass() != object.getClass()) return false; - AbstractFlagSet flagSet = (AbstractFlagSet) object; - return mask == flagSet.mask && Objects.equals(flags, flagSet.flags); +// if (object == null || getClass() != object.getClass()) return false; + return FlagSet.equals(this, (FlagSet) object); } @Override @@ -85,14 +90,137 @@ public int hashCode() { @Override public String toString() { - final String flagsStr = flags.stream() - .sorted(Comparator.comparing(MaskedFlag::getMask)) - .map(MaskedFlag::name) - .collect(Collectors.joining(", ")); - return "FlagSet{" + - "flags=[" + flagsStr + - "], mask=" + mask + - '}'; + return FlagSet.asString(this); + } + + + // -------------------------------------------------------------------------------- + + + static abstract class AbstractSingleFlagSet & MaskedFlag> implements FlagSet { + + private final T flag; + // Only holding this for iterator() and getFlags() so make it lazy. + private EnumSet enumSet; + + public AbstractSingleFlagSet(final T flag) { + this.flag = Objects.requireNonNull(flag); + } + + @Override + public int getMask() { + return flag.getMask(); + } + + @Override + public Set getFlags() { + if (enumSet == null) { + return initSet(); + } else { + return this.enumSet; + } + } + + @Override + public boolean isSet(final T flag) { + return this.flag == flag; + } + + @Override + public int size() { + return 1; + } + + @Override + public boolean isEmpty() { + return false; + } + + @Override + public Iterator iterator() { + if (enumSet == null) { + return initSet().iterator(); + } else { + return this.enumSet.iterator(); + } + } + + @Override + public String toString() { + return FlagSet.asString(this); + } + + @Override + public boolean equals(Object object) { + if (this == object) return true; +// if (object == null || getClass() != object.getClass()) return false; + return FlagSet.equals(this, (FlagSet) object); + } + + @Override + public int hashCode() { + return Objects.hash(flag, getFlags()); + } + + private Set initSet() { + final EnumSet set = EnumSet.of(this.flag); + this.enumSet = set; + return set; + } + } + + + // -------------------------------------------------------------------------------- + + + static class AbstractEmptyFlagSet implements FlagSet { + + @Override + public int getMask() { + return MaskedFlag.EMPTY_MASK; + } + + @Override + public Set getFlags() { + return Collections.emptySet(); + } + + @Override + public boolean isSet(final T flag) { + return false; + } + + @Override + public int size() { + return 0; + } + + @Override + public boolean isEmpty() { + return true; + } + + @Override + public Iterator iterator() { + return Collections.emptyIterator(); + } + + @Override + public String toString() { + return FlagSet.asString(this); + } + + @Override + public boolean equals(Object object) { + if (this == object) return true; +// if (object == null || getClass() != object.getClass()) return false; + return FlagSet.equals(this, (FlagSet) object); + } + + @Override + public int hashCode() { + return Objects.hash(getMask(), getFlags()); + } } @@ -105,17 +233,23 @@ public String toString() { * @param The type of flag to be held in the {@link AbstractFlagSet} * @param The type of the {@link AbstractFlagSet} implementation. */ - public static class Builder & MaskedFlag, S extends AbstractFlagSet> { + public static class Builder & MaskedFlag, S extends FlagSet> { final Class type; final EnumSet enumSet; final Function, S> constructor; + final Function singletonSetConstructor; + final Supplier emptySetSupplier; protected Builder(final Class type, - final Function, S> constructor) { + final Function, S> constructor, + final Function singletonSetConstructor, + final Supplier emptySetSupplier) { this.type = type; this.enumSet = EnumSet.noneOf(type); - this.constructor = constructor; + this.constructor = Objects.requireNonNull(constructor); + this.singletonSetConstructor = Objects.requireNonNull(singletonSetConstructor); + this.emptySetSupplier = Objects.requireNonNull(emptySetSupplier); } /** @@ -125,7 +259,7 @@ protected Builder(final Class type, * @return this builder instance. */ public Builder withFlags(final Collection flags) { - enumSet.clear(); + clear(); if (flags != null) { for (E flag : flags) { if (flag != null) { @@ -142,7 +276,7 @@ public Builder withFlags(final Collection flags) { */ @SafeVarargs public final Builder withFlags(final E... flags) { - enumSet.clear(); + clear(); if (flags != null) { for (E flag : flags) { if (flag != null) { @@ -185,8 +319,14 @@ public Builder clear() { * @return A */ public S build() { - return constructor.apply(enumSet); + final int size = enumSet.size(); + if (size == 0) { + return emptySetSupplier.get(); + } else if (size == 1) { + return singletonSetConstructor.apply(enumSet.stream().findFirst().get()); + } else { + return constructor.apply(enumSet); + } } } } - diff --git a/src/main/java/org/lmdbjava/CopyFlagSet.java b/src/main/java/org/lmdbjava/CopyFlagSet.java new file mode 100644 index 00000000..62c73c8d --- /dev/null +++ b/src/main/java/org/lmdbjava/CopyFlagSet.java @@ -0,0 +1,59 @@ +package org.lmdbjava; + +import java.util.Collection; +import java.util.EnumSet; +import java.util.Objects; + +public interface CopyFlagSet extends FlagSet { + + static CopyFlagSet EMPTY = CopyFlagSetImpl.EMPTY; + + static CopyFlagSet empty() { + return CopyFlagSetImpl.EMPTY; + } + + static CopyFlagSet of(final CopyFlags dbiFlag) { + Objects.requireNonNull(dbiFlag); + return dbiFlag; + } + + static CopyFlagSet of(final CopyFlags... CopyFlags) { + return builder() + .withFlags(CopyFlags) + .build(); + } + + static CopyFlagSet of(final Collection CopyFlags) { + return builder() + .withFlags(CopyFlags) + .build(); + } + + static AbstractFlagSet.Builder builder() { + return new AbstractFlagSet.Builder<>( + CopyFlags.class, + CopyFlagSetImpl::new, + copyFlag -> copyFlag, + () -> CopyFlagSetImpl.EMPTY); + } + + + // -------------------------------------------------------------------------------- + + + class CopyFlagSetImpl extends AbstractFlagSet implements CopyFlagSet { + + static final CopyFlagSet EMPTY = new EmptyCopyFlagSet(); + + private CopyFlagSetImpl(final EnumSet flags) { + super(flags); + } + } + + + // -------------------------------------------------------------------------------- + + + class EmptyCopyFlagSet extends AbstractFlagSet.AbstractEmptyFlagSet implements CopyFlagSet { + } +} diff --git a/src/main/java/org/lmdbjava/CopyFlags.java b/src/main/java/org/lmdbjava/CopyFlags.java index 4365563c..fbd4d171 100644 --- a/src/main/java/org/lmdbjava/CopyFlags.java +++ b/src/main/java/org/lmdbjava/CopyFlags.java @@ -15,8 +15,11 @@ */ package org.lmdbjava; +import java.util.EnumSet; +import java.util.Set; + /** Flags for use when performing a {@link Env#copy(java.io.File, org.lmdbjava.CopyFlags...)}. */ -public enum CopyFlags implements MaskedFlag { +public enum CopyFlags implements MaskedFlag, CopyFlagSet { /** Compacting copy: Omit free space from copy, and renumber all pages sequentially. */ MDB_CP_COMPACT(0x01); @@ -31,4 +34,29 @@ public enum CopyFlags implements MaskedFlag { public int getMask() { return mask; } + + @Override + public Set getFlags() { + return EnumSet.of(this); + } + + @Override + public boolean isSet(final CopyFlags flag) { + return flag != null && mask == flag.getMask(); + } + + @Override + public int size() { + return 1; + } + + @Override + public boolean isEmpty() { + return false; + } + + @Override + public String toString() { + return FlagSet.asString(this); + } } diff --git a/src/main/java/org/lmdbjava/DbiFlagSet.java b/src/main/java/org/lmdbjava/DbiFlagSet.java index cd1db934..e5c97544 100644 --- a/src/main/java/org/lmdbjava/DbiFlagSet.java +++ b/src/main/java/org/lmdbjava/DbiFlagSet.java @@ -1,32 +1,57 @@ package org.lmdbjava; +import java.util.Collection; import java.util.EnumSet; import java.util.Objects; -public class DbiFlagSet extends AbstractFlagSet { +public interface DbiFlagSet extends FlagSet { - public static final DbiFlagSet EMPTY = new DbiFlagSet(EnumSet.noneOf(DbiFlags.class)); + static DbiFlagSet empty() { + return DbiFlagSetImpl.EMPTY; + } - private DbiFlagSet(final EnumSet flags) { - super(flags); - } + static DbiFlagSet of(final DbiFlags dbiFlag) { + Objects.requireNonNull(dbiFlag); + return dbiFlag; + } - public static DbiFlagSet empty() { - return EMPTY; - } + static DbiFlagSet of(final DbiFlags... DbiFlags) { + return builder() + .withFlags(DbiFlags) + .build(); + } - public static DbiFlagSet of(final DbiFlags putFlag) { - Objects.requireNonNull(putFlag); - return new DbiFlagSet(EnumSet.of(putFlag)); - } + static DbiFlagSet of(final Collection DbiFlags) { + return builder() + .withFlags(DbiFlags) + .build(); + } + + static AbstractFlagSet.Builder builder() { + return new AbstractFlagSet.Builder<>( + DbiFlags.class, + DbiFlagSetImpl::new, + dbiFlag -> dbiFlag, + () -> DbiFlagSetImpl.EMPTY); + } + + + // -------------------------------------------------------------------------------- - public static DbiFlagSet of(final DbiFlags... DbiFlags) { - return builder() - .withFlags(DbiFlags) - .build(); - } - public static Builder builder() { - return new Builder<>(DbiFlags.class, DbiFlagSet::new); + class DbiFlagSetImpl extends AbstractFlagSet implements DbiFlagSet { + + static final DbiFlagSet EMPTY = new EmptyDbiFlagSet(); + + private DbiFlagSetImpl(final EnumSet flags) { + super(flags); } + } + + + // -------------------------------------------------------------------------------- + + + class EmptyDbiFlagSet extends AbstractFlagSet.AbstractEmptyFlagSet implements DbiFlagSet { + } } diff --git a/src/main/java/org/lmdbjava/DbiFlags.java b/src/main/java/org/lmdbjava/DbiFlags.java index af6eaeaa..6e4b723d 100644 --- a/src/main/java/org/lmdbjava/DbiFlags.java +++ b/src/main/java/org/lmdbjava/DbiFlags.java @@ -15,8 +15,11 @@ */ package org.lmdbjava; +import java.util.EnumSet; +import java.util.Set; + /** Flags for use when opening a {@link Dbi}. */ -public enum DbiFlags implements MaskedFlag { +public enum DbiFlags implements MaskedFlag, DbiFlagSet { /** * Use reverse string keys. @@ -82,4 +85,29 @@ public enum DbiFlags implements MaskedFlag { public int getMask() { return mask; } + + @Override + public Set getFlags() { + return EnumSet.of(this); + } + + @Override + public boolean isSet(final DbiFlags flag) { + return flag != null && mask == flag.getMask(); + } + + @Override + public int size() { + return 1; + } + + @Override + public boolean isEmpty() { + return false; + } + + @Override + public String toString() { + return FlagSet.asString(this); + } } diff --git a/src/main/java/org/lmdbjava/Env.java b/src/main/java/org/lmdbjava/Env.java index 5de0a7fd..2ad929ad 100644 --- a/src/main/java/org/lmdbjava/Env.java +++ b/src/main/java/org/lmdbjava/Env.java @@ -26,7 +26,6 @@ import static org.lmdbjava.MaskedFlag.isSet; import static org.lmdbjava.MaskedFlag.mask; import static org.lmdbjava.ResultCodeMapper.checkRc; -import static org.lmdbjava.TxnFlags.MDB_RDONLY_TXN; import java.io.File; import java.nio.ByteBuffer; @@ -124,6 +123,27 @@ public void close() { LIB.mdb_env_close(ptr); } + /** + * Copies an LMDB environment to the specified destination path. + * + *

This function may be used to make a backup of an existing environment. No lockfile is + * created, since it gets recreated at need. + * + *

If this environment was created using {@link EnvFlags#MDB_NOSUBDIR}, the destination path + * must be a directory that exists but contains no files. If {@link EnvFlags#MDB_NOSUBDIR} was + * used, the destination path must not exist, but it must be possible to create a file at the + * provided path. + * + *

Note: This call can trigger significant file size growth if run in parallel with write + * transactions, because it employs a read-only transaction. See long-lived transactions under + * "Caveats" in the LMDB native documentation. + * + * @param path writable destination path as described above + */ + public void copy(final File path) { + copy(path, CopyFlagSet.EMPTY); + } + /** * Copies an LMDB environment to the specified destination path. * @@ -142,11 +162,10 @@ public void close() { * @param path writable destination path as described above * @param flags special options for this copy */ - public void copy(final File path, final CopyFlags... flags) { + public void copy(final File path, final CopyFlagSet flags) { requireNonNull(path); validatePath(path); - final int flagsMask = mask(flags); - checkRc(LIB.mdb_env_copy2(ptr, path.getAbsolutePath(), flagsMask)); + checkRc(LIB.mdb_env_copy2(ptr, path.getAbsolutePath(), flags.getMask())); } /** @@ -443,16 +462,54 @@ public void sync(final boolean force) { } /** + * @deprecated Instead use {@link Env#txn(Txn, TxnFlagSet)} + * * Obtain a transaction with the requested parent and flags. * * @param parent parent transaction (may be null if no parent) * @param flags applicable flags (eg for a reusable, read-only transaction) * @return a transaction (never null) */ + @Deprecated public Txn txn(final Txn parent, final TxnFlags... flags) { - if (closed) { - throw new AlreadyClosedException(); - } + checkNotClosed(); + return new Txn<>(this, parent, proxy, TxnFlagSet.of(flags)); + } + + /** + * Obtain a transaction with the requested parent and flags. + * + * @param parent parent transaction (may be null if no parent) + * @return a transaction (never null) + */ + public Txn txn(final Txn parent) { + checkNotClosed(); + return new Txn<>(this, parent, proxy, TxnFlagSet.EMPTY); + } + + /** + * Obtain a transaction with the requested parent and flags. + * + * @param parent parent transaction (may be null if no parent) + * @param flag applicable flag (eg for a reusable, read-only transaction) + * @return a transaction (never null) + */ + public Txn txn(final Txn parent, final TxnFlags flag) { + checkNotClosed(); + return new Txn<>(this, parent, proxy, flag); + } + + /** + * Obtain a transaction with the requested parent and flags. + * + * @param parent parent transaction (may be null if no parent) + * @param flags applicable flags (e.g. for a reusable, read-only transaction). + * If the set of flags is used frequently it is recommended to hold + * a static instance of the {@link TxnFlagSet} for re-use. + * @return a transaction (never null) + */ + public Txn txn(final Txn parent, final TxnFlagSet flags) { + checkNotClosed(); return new Txn<>(this, parent, proxy, flags); } @@ -462,7 +519,8 @@ public Txn txn(final Txn parent, final TxnFlags... flags) { * @return a read-only transaction */ public Txn txnRead() { - return txn(null, MDB_RDONLY_TXN); + checkNotClosed(); + return new Txn<>(this, null, proxy, TxnFlags.MDB_RDONLY_TXN); } /** @@ -471,7 +529,8 @@ public Txn txnRead() { * @return a read-write transaction */ public Txn txnWrite() { - return txn(null); + checkNotClosed(); + return new Txn<>(this, null, proxy, TxnFlagSet.EMPTY); } Pointer pointer() { diff --git a/src/main/java/org/lmdbjava/EnvFlagSet.java b/src/main/java/org/lmdbjava/EnvFlagSet.java new file mode 100644 index 00000000..944496e6 --- /dev/null +++ b/src/main/java/org/lmdbjava/EnvFlagSet.java @@ -0,0 +1,57 @@ +package org.lmdbjava; + +import java.util.Collection; +import java.util.EnumSet; +import java.util.Objects; + +public interface EnvFlagSet extends FlagSet { + + static EnvFlagSet empty() { + return EnvFlagSetImpl.EMPTY; + } + + static EnvFlagSet of(final EnvFlags envFlag) { + Objects.requireNonNull(envFlag); + return envFlag; + } + + static EnvFlagSet of(final EnvFlags... EnvFlags) { + return builder() + .withFlags(EnvFlags) + .build(); + } + + static EnvFlagSet of(final Collection EnvFlags) { + return builder() + .withFlags(EnvFlags) + .build(); + } + + static AbstractFlagSet.Builder builder() { + return new AbstractFlagSet.Builder<>( + EnvFlags.class, + EnvFlagSetImpl::new, + envFlag -> envFlag, + () -> EnvFlagSetImpl.EMPTY); + } + + + // -------------------------------------------------------------------------------- + + + class EnvFlagSetImpl extends AbstractFlagSet implements EnvFlagSet { + + static final EnvFlagSet EMPTY = new EmptyEnvFlagSet(); + + private EnvFlagSetImpl(final EnumSet flags) { + super(flags); + } + } + + + // -------------------------------------------------------------------------------- + + + class EmptyEnvFlagSet extends AbstractFlagSet.AbstractEmptyFlagSet implements EnvFlagSet { + } +} diff --git a/src/main/java/org/lmdbjava/EnvFlags.java b/src/main/java/org/lmdbjava/EnvFlags.java index 4ce555a8..7fb4a29b 100644 --- a/src/main/java/org/lmdbjava/EnvFlags.java +++ b/src/main/java/org/lmdbjava/EnvFlags.java @@ -15,8 +15,11 @@ */ package org.lmdbjava; +import java.util.EnumSet; +import java.util.Set; + /** Flags for use when opening the {@link Env}. */ -public enum EnvFlags implements MaskedFlag { +public enum EnvFlags implements MaskedFlag, EnvFlagSet { /** * Mmap at a fixed address (experimental). @@ -144,4 +147,29 @@ public enum EnvFlags implements MaskedFlag { public int getMask() { return mask; } + + @Override + public Set getFlags() { + return EnumSet.of(this); + } + + @Override + public boolean isSet(final EnvFlags flag) { + return flag != null && mask == flag.getMask(); + } + + @Override + public int size() { + return 1; + } + + @Override + public boolean isEmpty() { + return false; + } + + @Override + public String toString() { + return FlagSet.asString(this); + } } diff --git a/src/main/java/org/lmdbjava/FlagSet.java b/src/main/java/org/lmdbjava/FlagSet.java new file mode 100644 index 00000000..80a4c19e --- /dev/null +++ b/src/main/java/org/lmdbjava/FlagSet.java @@ -0,0 +1,61 @@ +package org.lmdbjava; + +import java.util.Comparator; +import java.util.Iterator; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * A set of flags, each with a bit mask value. + * Flags can be combined in a set such that the set has a combined bit mask value. + * @param + */ +public interface FlagSet extends Iterable { + + int getMask(); + + Set getFlags(); + + boolean isSet(T flag); + + default int size() { + return getFlags().size(); + } + + default boolean isEmpty() { + return getFlags().isEmpty(); + } + + default Iterator iterator() { + return getFlags().iterator(); + } + + static String asString(final FlagSet flagSet) { + Objects.requireNonNull(flagSet); + final String flagsStr = flagSet.getFlags() + .stream() + .sorted(Comparator.comparing(MaskedFlag::getMask)) + .map(MaskedFlag::name) + .collect(Collectors.joining(", ")); + return "FlagSet{" + + "flags=[" + flagsStr + + "], mask=" + flagSet.getMask() + + '}'; + } + + static boolean equals(final FlagSet flagSet1, + final FlagSet flagSet2) { + if (flagSet1 == flagSet2) { + return true; + } else if (flagSet1 != null && flagSet2 == null) { + return false; + } else if (flagSet1 == null) { + return false; + } else { + return flagSet1.getMask() == flagSet2.getMask() + && Objects.equals(flagSet1.getFlags(), flagSet2.getFlags()); + } + } + +} diff --git a/src/main/java/org/lmdbjava/PutFlagSet.java b/src/main/java/org/lmdbjava/PutFlagSet.java index 8820fe92..1eedaf10 100644 --- a/src/main/java/org/lmdbjava/PutFlagSet.java +++ b/src/main/java/org/lmdbjava/PutFlagSet.java @@ -1,32 +1,57 @@ package org.lmdbjava; +import java.util.Collection; import java.util.EnumSet; import java.util.Objects; -public class PutFlagSet extends AbstractFlagSet { +public interface PutFlagSet extends FlagSet { - public static final PutFlagSet EMPTY = new PutFlagSet(EnumSet.noneOf(PutFlags.class)); - - private PutFlagSet(final EnumSet flags) { - super(flags); - } - - public static PutFlagSet empty() { - return EMPTY; + static PutFlagSet empty() { + return PutFlagSetImpl.EMPTY; } - public static PutFlagSet of(final PutFlags putFlag) { + static PutFlagSet of(final PutFlags putFlag) { Objects.requireNonNull(putFlag); - return new org.lmdbjava.PutFlagSet(EnumSet.of(putFlag)); + return putFlag; } - public static PutFlagSet of(final PutFlags... putFlags) { + static PutFlagSet of(final PutFlags... putFlags) { return builder() .withFlags(putFlags) .build(); } - public static Builder builder() { - return new Builder<>(PutFlags.class, PutFlagSet::new); + static PutFlagSet of(final Collection putFlags) { + return builder() + .withFlags(putFlags) + .build(); + } + + static AbstractFlagSet.Builder builder() { + return new AbstractFlagSet.Builder<>( + PutFlags.class, + PutFlagSetImpl::new, + putFlag -> putFlag, + EmptyPutFlagSet::new); + } + + + // -------------------------------------------------------------------------------- + + + class PutFlagSetImpl extends AbstractFlagSet implements PutFlagSet { + + public static final PutFlagSet EMPTY = new EmptyPutFlagSet(); + + private PutFlagSetImpl(final EnumSet flags) { + super(flags); + } + } + + + // -------------------------------------------------------------------------------- + + + class EmptyPutFlagSet extends AbstractFlagSet.AbstractEmptyFlagSet implements PutFlagSet { } } diff --git a/src/main/java/org/lmdbjava/PutFlags.java b/src/main/java/org/lmdbjava/PutFlags.java index 809103de..03fa916a 100644 --- a/src/main/java/org/lmdbjava/PutFlags.java +++ b/src/main/java/org/lmdbjava/PutFlags.java @@ -15,8 +15,11 @@ */ package org.lmdbjava; +import java.util.EnumSet; +import java.util.Set; + /** Flags for use when performing a "put". */ -public enum PutFlags implements MaskedFlag { +public enum PutFlags implements MaskedFlag, PutFlagSet { /** For put: Don't write if the key already exists. */ MDB_NOOVERWRITE(0x10), @@ -49,4 +52,29 @@ public enum PutFlags implements MaskedFlag { public int getMask() { return mask; } + + @Override + public Set getFlags() { + return EnumSet.of(this); + } + + @Override + public boolean isSet(PutFlags flag) { + return flag != null && mask == flag.getMask(); + } + + @Override + public int size() { + return 1; + } + + @Override + public boolean isEmpty() { + return false; + } + + @Override + public String toString() { + return FlagSet.asString(this); + } } diff --git a/src/main/java/org/lmdbjava/Txn.java b/src/main/java/org/lmdbjava/Txn.java index 1d5d4860..5c8440fd 100644 --- a/src/main/java/org/lmdbjava/Txn.java +++ b/src/main/java/org/lmdbjava/Txn.java @@ -20,8 +20,6 @@ import static org.lmdbjava.Env.SHOULD_CHECK; import static org.lmdbjava.Library.LIB; import static org.lmdbjava.Library.RUNTIME; -import static org.lmdbjava.MaskedFlag.isSet; -import static org.lmdbjava.MaskedFlag.mask; import static org.lmdbjava.ResultCodeMapper.checkRc; import static org.lmdbjava.Txn.State.DONE; import static org.lmdbjava.Txn.State.READY; @@ -46,11 +44,13 @@ public final class Txn implements AutoCloseable { private final Env env; private State state; - Txn(final Env env, final Txn parent, final BufferProxy proxy, final TxnFlags... flags) { + Txn(final Env env, final Txn parent, final BufferProxy proxy, final TxnFlagSet flags) { + final TxnFlagSet flagSet = flags != null + ? flags + : TxnFlagSet.EMPTY; this.proxy = proxy; this.keyVal = proxy.keyVal(); - final int flagsMask = mask(flags); - this.readOnly = isSet(flagsMask, MDB_RDONLY_TXN); + this.readOnly = flagSet.isSet(MDB_RDONLY_TXN); if (env.isReadOnly() && !this.readOnly) { throw new EnvIsReadOnly(); } @@ -61,7 +61,7 @@ public final class Txn implements AutoCloseable { } final Pointer txnPtr = allocateDirect(RUNTIME, ADDRESS); final Pointer txnParentPtr = parent == null ? null : parent.ptr; - checkRc(LIB.mdb_txn_begin(env.pointer(), txnParentPtr, flagsMask, txnPtr)); + checkRc(LIB.mdb_txn_begin(env.pointer(), txnParentPtr, flagSet.getMask(), txnPtr)); ptr = txnPtr.getPointer(0); state = READY; diff --git a/src/main/java/org/lmdbjava/TxnFlagSet.java b/src/main/java/org/lmdbjava/TxnFlagSet.java new file mode 100644 index 00000000..6320eece --- /dev/null +++ b/src/main/java/org/lmdbjava/TxnFlagSet.java @@ -0,0 +1,63 @@ +package org.lmdbjava; + +import java.util.EnumSet; +import java.util.Objects; + +public interface TxnFlagSet extends FlagSet { + + TxnFlagSet EMPTY = TxnFlagSetImpl.EMPTY; + + static TxnFlagSet empty() { + return TxnFlagSetImpl.EMPTY; + } + + static TxnFlagSet of(final TxnFlags putFlag) { + Objects.requireNonNull(putFlag); + return new SingleTxnFlagSet(putFlag); + } + + static TxnFlagSet of(final TxnFlags... TxnFlags) { + return builder() + .withFlags(TxnFlags) + .build(); + } + + static AbstractFlagSet.Builder builder() { + return new AbstractFlagSet.Builder<>( + TxnFlags.class, + TxnFlagSetImpl::new, + SingleTxnFlagSet::new, + () -> TxnFlagSetImpl.EMPTY); + } + + + // -------------------------------------------------------------------------------- + + + class TxnFlagSetImpl extends AbstractFlagSet implements TxnFlagSet { + + static final TxnFlagSet EMPTY = new EmptyTxnFlagSet(); + + private TxnFlagSetImpl(final EnumSet flags) { + super(flags); + } + } + + + // -------------------------------------------------------------------------------- + + + class SingleTxnFlagSet extends AbstractFlagSet.AbstractSingleFlagSet implements TxnFlagSet { + + SingleTxnFlagSet(final TxnFlags flag) { + super(flag); + } + } + + + // -------------------------------------------------------------------------------- + + + class EmptyTxnFlagSet extends AbstractFlagSet.AbstractEmptyFlagSet implements TxnFlagSet { + } +} diff --git a/src/main/java/org/lmdbjava/TxnFlags.java b/src/main/java/org/lmdbjava/TxnFlags.java index 26caf6f1..8e4ea757 100644 --- a/src/main/java/org/lmdbjava/TxnFlags.java +++ b/src/main/java/org/lmdbjava/TxnFlags.java @@ -15,8 +15,11 @@ */ package org.lmdbjava; +import java.util.EnumSet; +import java.util.Set; + /** Flags for use when creating a {@link Txn}. */ -public enum TxnFlags implements MaskedFlag { +public enum TxnFlags implements MaskedFlag, TxnFlagSet { /** Read only. */ MDB_RDONLY_TXN(0x2_0000); @@ -30,4 +33,29 @@ public enum TxnFlags implements MaskedFlag { public int getMask() { return mask; } + + @Override + public Set getFlags() { + return EnumSet.of(this); + } + + @Override + public boolean isSet(final TxnFlags flag) { + return flag != null && mask == flag.getMask(); + } + + @Override + public int size() { + return 1; + } + + @Override + public boolean isEmpty() { + return false; + } + + @Override + public String toString() { + return FlagSet.asString(this); + } } diff --git a/src/test/java/org/lmdbjava/CopyFlagSetTest.java b/src/test/java/org/lmdbjava/CopyFlagSetTest.java new file mode 100644 index 00000000..1ea44b7e --- /dev/null +++ b/src/test/java/org/lmdbjava/CopyFlagSetTest.java @@ -0,0 +1,88 @@ +package org.lmdbjava; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.MatcherAssert.assertThat; + +import java.util.Collections; +import java.util.HashSet; +import org.junit.Test; + +public class CopyFlagSetTest { + + @Test + public void testEmpty() { + final CopyFlagSet copyFlagSet = CopyFlagSet.empty(); + assertThat( + copyFlagSet.getMask(), + is(0)); + assertThat( + copyFlagSet.size(), + is(0)); + assertThat( + copyFlagSet.isEmpty(), + is(true)); + assertThat( + copyFlagSet.isSet(CopyFlags.MDB_CP_COMPACT), + is(false)); + final CopyFlagSet copyFlagSet2 = CopyFlagSet.builder() + .build(); + assertThat(copyFlagSet, is(copyFlagSet2)); + assertThat(copyFlagSet, not(CopyFlagSet.of(CopyFlags.MDB_CP_COMPACT))); + assertThat(copyFlagSet, not(CopyFlagSet.builder() + .setFlag(CopyFlags.MDB_CP_COMPACT) + .build())); + } + + @Test + public void testOf() { + final CopyFlags copyFlag = CopyFlags.MDB_CP_COMPACT; + final CopyFlagSet copyFlagSet = CopyFlagSet.of(copyFlag); + assertThat( + copyFlagSet.getMask(), + is(MaskedFlag.mask(copyFlag))); + assertThat( + copyFlagSet.size(), + is(1)); + for (CopyFlags flag : copyFlagSet) { + assertThat( + copyFlagSet.isSet(flag), + is(true)); + } + + final CopyFlagSet copyFlagSet2 = CopyFlagSet.builder() + .setFlag(copyFlag) + .build(); + assertThat(copyFlagSet, is(copyFlagSet2)); + } + + @Test + public void testBuilder() { + final CopyFlags copyFlag1 = CopyFlags.MDB_CP_COMPACT; + final CopyFlagSet copyFlagSet = CopyFlagSet.builder() + .setFlag(copyFlag1) + .build(); + assertThat( + copyFlagSet.getMask(), + is(MaskedFlag.mask(copyFlag1))); + assertThat( + copyFlagSet.size(), + is(1)); + assertThat( + copyFlagSet.isSet(CopyFlags.MDB_CP_COMPACT), + is(true)); + for (CopyFlags flag : copyFlagSet) { + assertThat( + copyFlagSet.isSet(flag), + is(true)); + } + final CopyFlagSet copyFlagSet2 = CopyFlagSet.builder() + .withFlags(copyFlag1) + .build(); + final CopyFlagSet copyFlagSet3 = CopyFlagSet.builder() + .withFlags(new HashSet<>(Collections.singletonList(copyFlag1))) + .build(); + assertThat(copyFlagSet, is(copyFlagSet2)); + assertThat(copyFlagSet, is(copyFlagSet3)); + } +} diff --git a/src/test/java/org/lmdbjava/DbiFlagSetTest.java b/src/test/java/org/lmdbjava/DbiFlagSetTest.java index cfbda600..323a2ed4 100644 --- a/src/test/java/org/lmdbjava/DbiFlagSetTest.java +++ b/src/test/java/org/lmdbjava/DbiFlagSetTest.java @@ -1,6 +1,7 @@ package org.lmdbjava; import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.MatcherAssert.assertThat; import java.util.Arrays; @@ -11,91 +12,105 @@ public class DbiFlagSetTest { @Test public void testEmpty() { - final DbiFlagSet putFlagSet = DbiFlagSet.empty(); + final DbiFlagSet dbiFlagSet = DbiFlagSet.empty(); assertThat( - putFlagSet.getMask(), + dbiFlagSet.getMask(), is(0)); assertThat( - putFlagSet.size(), + dbiFlagSet.size(), is(0)); assertThat( - putFlagSet.isEmpty(), + dbiFlagSet.isEmpty(), is(true)); assertThat( - putFlagSet.isSet(DbiFlags.MDB_REVERSEDUP), + dbiFlagSet.isSet(DbiFlags.MDB_REVERSEDUP), is(false)); + final DbiFlagSet dbiFlagSet2 = DbiFlagSet.builder() + .build(); + assertThat(dbiFlagSet, is(dbiFlagSet2)); + assertThat(dbiFlagSet, not(DbiFlagSet.of(DbiFlags.MDB_CREATE))); + assertThat(dbiFlagSet, not(DbiFlagSet.of(DbiFlags.MDB_CREATE, DbiFlags.MDB_DUPSORT))); + assertThat(dbiFlagSet, not(DbiFlagSet.builder() + .setFlag(DbiFlags.MDB_CREATE) + .setFlag(DbiFlags.MDB_DUPFIXED) + .build())); } @Test public void testOf() { - final DbiFlags putFlag = DbiFlags.MDB_CREATE; - final DbiFlagSet putFlagSet = DbiFlagSet.of(putFlag); + final DbiFlags dbiFlag = DbiFlags.MDB_CREATE; + final DbiFlagSet dbiFlagSet = DbiFlagSet.of(dbiFlag); assertThat( - putFlagSet.getMask(), - is(MaskedFlag.mask(putFlag))); + dbiFlagSet.getMask(), + is(MaskedFlag.mask(dbiFlag))); assertThat( - putFlagSet.size(), + dbiFlagSet.size(), is(1)); assertThat( - putFlagSet.isSet(DbiFlags.MDB_REVERSEDUP), + dbiFlagSet.isSet(DbiFlags.MDB_REVERSEDUP), is(false)); - for (DbiFlags flag : putFlagSet) { + for (DbiFlags flag : dbiFlagSet) { assertThat( - putFlagSet.isSet(flag), + dbiFlagSet.isSet(flag), is(true)); } + + final DbiFlagSet dbiFlagSet2 = DbiFlagSet.builder() + .setFlag(dbiFlag) + .build(); + assertThat(dbiFlagSet, is(dbiFlagSet2)); } @Test public void testOf2() { - final DbiFlags putFlag1 = DbiFlags.MDB_CREATE; - final DbiFlags putFlag2 = DbiFlags.MDB_INTEGERKEY; - final DbiFlagSet putFlagSet = DbiFlagSet.of(putFlag1, putFlag2); + final DbiFlags dbiFlag1 = DbiFlags.MDB_CREATE; + final DbiFlags dbiFlag2 = DbiFlags.MDB_INTEGERKEY; + final DbiFlagSet dbiFlagSet = DbiFlagSet.of(dbiFlag1, dbiFlag2); assertThat( - putFlagSet.getMask(), - is(MaskedFlag.mask(putFlag1, putFlag2))); + dbiFlagSet.getMask(), + is(MaskedFlag.mask(dbiFlag1, dbiFlag2))); assertThat( - putFlagSet.size(), + dbiFlagSet.size(), is(2)); assertThat( - putFlagSet.isSet(DbiFlags.MDB_REVERSEDUP), + dbiFlagSet.isSet(DbiFlags.MDB_REVERSEDUP), is(false)); - for (DbiFlags flag : putFlagSet) { + for (DbiFlags flag : dbiFlagSet) { assertThat( - putFlagSet.isSet(flag), + dbiFlagSet.isSet(flag), is(true)); } } @Test public void testBuilder() { - final DbiFlags putFlag1 = DbiFlags.MDB_CREATE; - final DbiFlags putFlag2 = DbiFlags.MDB_INTEGERKEY; - final DbiFlagSet putFlagSet = DbiFlagSet.builder() - .setFlag(putFlag1) - .setFlag(putFlag2) + final DbiFlags dbiFlag1 = DbiFlags.MDB_CREATE; + final DbiFlags dbiFlag2 = DbiFlags.MDB_INTEGERKEY; + final DbiFlagSet dbiFlagSet = DbiFlagSet.builder() + .setFlag(dbiFlag1) + .setFlag(dbiFlag2) .build(); assertThat( - putFlagSet.getMask(), - is(MaskedFlag.mask(putFlag1, putFlag2))); + dbiFlagSet.getMask(), + is(MaskedFlag.mask(dbiFlag1, dbiFlag2))); assertThat( - putFlagSet.size(), + dbiFlagSet.size(), is(2)); assertThat( - putFlagSet.isSet(DbiFlags.MDB_REVERSEDUP), + dbiFlagSet.isSet(DbiFlags.MDB_REVERSEDUP), is(false)); - for (DbiFlags flag : putFlagSet) { + for (DbiFlags flag : dbiFlagSet) { assertThat( - putFlagSet.isSet(flag), + dbiFlagSet.isSet(flag), is(true)); } - final DbiFlagSet putFlagSet2 = DbiFlagSet.builder() - .withFlags(putFlag1, putFlag2) + final DbiFlagSet dbiFlagSet2 = DbiFlagSet.builder() + .withFlags(dbiFlag1, dbiFlag2) .build(); - final DbiFlagSet putFlagSet3 = DbiFlagSet.builder() - .withFlags(new HashSet<>(Arrays.asList(putFlag1, putFlag2))) + final DbiFlagSet dbiFlagSet3 = DbiFlagSet.builder() + .withFlags(new HashSet<>(Arrays.asList(dbiFlag1, dbiFlag2))) .build(); - assertThat(putFlagSet, is(putFlagSet2)); - assertThat(putFlagSet, is(putFlagSet3)); + assertThat(dbiFlagSet, is(dbiFlagSet2)); + assertThat(dbiFlagSet, is(dbiFlagSet3)); } } diff --git a/src/test/java/org/lmdbjava/EnvFlagSetTest.java b/src/test/java/org/lmdbjava/EnvFlagSetTest.java new file mode 100644 index 00000000..ed6a0fea --- /dev/null +++ b/src/test/java/org/lmdbjava/EnvFlagSetTest.java @@ -0,0 +1,116 @@ +package org.lmdbjava; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.MatcherAssert.assertThat; + +import java.util.Arrays; +import java.util.HashSet; +import org.junit.Test; + +public class EnvFlagSetTest { + + @Test + public void testEmpty() { + final EnvFlagSet envFlagSet = EnvFlagSet.empty(); + assertThat( + envFlagSet.getMask(), + is(0)); + assertThat( + envFlagSet.size(), + is(0)); + assertThat( + envFlagSet.isEmpty(), + is(true)); + assertThat( + envFlagSet.isSet(EnvFlags.MDB_NOSUBDIR), + is(false)); + final EnvFlagSet envFlagSet2 = EnvFlagSet.builder() + .build(); + assertThat(envFlagSet, is(envFlagSet2)); + assertThat(envFlagSet, not(EnvFlagSet.of(EnvFlags.MDB_FIXEDMAP))); + assertThat(envFlagSet, not(EnvFlagSet.of(EnvFlags.MDB_FIXEDMAP, EnvFlags.MDB_NORDAHEAD))); + assertThat(envFlagSet, not(EnvFlagSet.builder() + .setFlag(EnvFlags.MDB_FIXEDMAP) + .setFlag(EnvFlags.MDB_NORDAHEAD) + .build())); + } + + @Test + public void testOf() { + final EnvFlags envFlag = EnvFlags.MDB_FIXEDMAP; + final EnvFlagSet envFlagSet = EnvFlagSet.of(envFlag); + assertThat( + envFlagSet.getMask(), + is(MaskedFlag.mask(envFlag))); + assertThat( + envFlagSet.size(), + is(1)); + assertThat( + envFlagSet.isSet(EnvFlags.MDB_NOSUBDIR), + is(false)); + for (EnvFlags flag : envFlagSet) { + assertThat( + envFlagSet.isSet(flag), + is(true)); + } + + final EnvFlagSet envFlagSet2 = EnvFlagSet.builder() + .setFlag(envFlag) + .build(); + assertThat(envFlagSet, is(envFlagSet2)); + } + + @Test + public void testOf2() { + final EnvFlags envFlag1 = EnvFlags.MDB_FIXEDMAP; + final EnvFlags envFlag2 = EnvFlags.MDB_NORDAHEAD; + final EnvFlagSet envFlagSet = EnvFlagSet.of(envFlag1, envFlag2); + assertThat( + envFlagSet.getMask(), + is(MaskedFlag.mask(envFlag1, envFlag2))); + assertThat( + envFlagSet.size(), + is(2)); + assertThat( + envFlagSet.isSet(EnvFlags.MDB_WRITEMAP), + is(false)); + for (EnvFlags flag : envFlagSet) { + assertThat( + envFlagSet.isSet(flag), + is(true)); + } + } + + @Test + public void testBuilder() { + final EnvFlags envFlag1 = EnvFlags.MDB_FIXEDMAP; + final EnvFlags envFlag2 = EnvFlags.MDB_NORDAHEAD; + final EnvFlagSet envFlagSet = EnvFlagSet.builder() + .setFlag(envFlag1) + .setFlag(envFlag2) + .build(); + assertThat( + envFlagSet.getMask(), + is(MaskedFlag.mask(envFlag1, envFlag2))); + assertThat( + envFlagSet.size(), + is(2)); + assertThat( + envFlagSet.isSet(EnvFlags.MDB_NOTLS), + is(false)); + for (EnvFlags flag : envFlagSet) { + assertThat( + envFlagSet.isSet(flag), + is(true)); + } + final EnvFlagSet envFlagSet2 = EnvFlagSet.builder() + .withFlags(envFlag1, envFlag2) + .build(); + final EnvFlagSet envFlagSet3 = EnvFlagSet.builder() + .withFlags(new HashSet<>(Arrays.asList(envFlag1, envFlag2))) + .build(); + assertThat(envFlagSet, is(envFlagSet2)); + assertThat(envFlagSet, is(envFlagSet3)); + } +} diff --git a/src/test/java/org/lmdbjava/PutFlagSetTest.java b/src/test/java/org/lmdbjava/PutFlagSetTest.java index 4826f436..8cf1efe0 100644 --- a/src/test/java/org/lmdbjava/PutFlagSetTest.java +++ b/src/test/java/org/lmdbjava/PutFlagSetTest.java @@ -1,6 +1,7 @@ package org.lmdbjava; import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.MatcherAssert.assertThat; import java.util.Arrays; @@ -24,6 +25,15 @@ public void testEmpty() { assertThat( putFlagSet.isSet(PutFlags.MDB_MULTIPLE), is(false)); + final PutFlagSet putFlagSet2 = PutFlagSet.builder() + .build(); + assertThat(putFlagSet, is(putFlagSet2)); + assertThat(putFlagSet, not(PutFlagSet.of(PutFlags.MDB_APPEND))); + assertThat(putFlagSet, not(PutFlagSet.of(PutFlags.MDB_APPEND, PutFlags.MDB_RESERVE))); + assertThat(putFlagSet, not(PutFlagSet.builder() + .setFlag(PutFlags.MDB_CURRENT) + .setFlag(PutFlags.MDB_MULTIPLE) + .build())); } @Test @@ -44,6 +54,11 @@ public void testOf() { putFlagSet.isSet(flag), is(true)); } + + final PutFlagSet putFlagSet2 = PutFlagSet.builder() + .setFlag(putFlag) + .build(); + assertThat(putFlagSet, is(putFlagSet2)); } @Test diff --git a/src/test/java/org/lmdbjava/TxnFlagSetTest.java b/src/test/java/org/lmdbjava/TxnFlagSetTest.java new file mode 100644 index 00000000..b526ceeb --- /dev/null +++ b/src/test/java/org/lmdbjava/TxnFlagSetTest.java @@ -0,0 +1,88 @@ +package org.lmdbjava; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.MatcherAssert.assertThat; + +import java.util.Collections; +import java.util.HashSet; +import org.junit.Test; + +public class TxnFlagSetTest { + + @Test + public void testEmpty() { + final TxnFlagSet txnFlagSet = TxnFlagSet.empty(); + assertThat( + txnFlagSet.getMask(), + is(0)); + assertThat( + txnFlagSet.size(), + is(0)); + assertThat( + txnFlagSet.isEmpty(), + is(true)); + assertThat( + txnFlagSet.isSet(TxnFlags.MDB_RDONLY_TXN), + is(false)); + final TxnFlagSet txnFlagSet2 = TxnFlagSet.builder() + .build(); + assertThat(txnFlagSet, is(txnFlagSet2)); + assertThat(txnFlagSet, not(TxnFlagSet.of(TxnFlags.MDB_RDONLY_TXN))); + assertThat(txnFlagSet, not(TxnFlagSet.builder() + .setFlag(TxnFlags.MDB_RDONLY_TXN) + .build())); + } + + @Test + public void testOf() { + final TxnFlags txnFlag = TxnFlags.MDB_RDONLY_TXN; + final TxnFlagSet txnFlagSet = TxnFlagSet.of(txnFlag); + assertThat( + txnFlagSet.getMask(), + is(MaskedFlag.mask(txnFlag))); + assertThat( + txnFlagSet.size(), + is(1)); + for (TxnFlags flag : txnFlagSet) { + assertThat( + txnFlagSet.isSet(flag), + is(true)); + } + + final TxnFlagSet txnFlagSet2 = TxnFlagSet.builder() + .setFlag(txnFlag) + .build(); + assertThat(txnFlagSet, is(txnFlagSet2)); + } + + @Test + public void testBuilder() { + final TxnFlags txnFlag1 = TxnFlags.MDB_RDONLY_TXN; + final TxnFlagSet txnFlagSet = TxnFlagSet.builder() + .setFlag(txnFlag1) + .build(); + assertThat( + txnFlagSet.getMask(), + is(MaskedFlag.mask(txnFlag1))); + assertThat( + txnFlagSet.size(), + is(1)); + assertThat( + txnFlagSet.isSet(TxnFlags.MDB_RDONLY_TXN), + is(true)); + for (TxnFlags flag : txnFlagSet) { + assertThat( + txnFlagSet.isSet(flag), + is(true)); + } + final TxnFlagSet txnFlagSet2 = TxnFlagSet.builder() + .withFlags(txnFlag1) + .build(); + final TxnFlagSet txnFlagSet3 = TxnFlagSet.builder() + .withFlags(new HashSet<>(Collections.singletonList(txnFlag1))) + .build(); + assertThat(txnFlagSet, is(txnFlagSet2)); + assertThat(txnFlagSet, is(txnFlagSet3)); + } +} From 667dab37ccce1c8d79307c01ab1f1d130082cd84 Mon Sep 17 00:00:00 2001 From: at055612 <22818309+at055612@users.noreply.github.com> Date: Mon, 27 Oct 2025 21:25:11 +0000 Subject: [PATCH 11/21] Fix Javadoc --- src/main/java/org/lmdbjava/CopyFlags.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/lmdbjava/CopyFlags.java b/src/main/java/org/lmdbjava/CopyFlags.java index fbd4d171..b45dc87c 100644 --- a/src/main/java/org/lmdbjava/CopyFlags.java +++ b/src/main/java/org/lmdbjava/CopyFlags.java @@ -15,10 +15,11 @@ */ package org.lmdbjava; +import java.io.File; import java.util.EnumSet; import java.util.Set; -/** Flags for use when performing a {@link Env#copy(java.io.File, org.lmdbjava.CopyFlags...)}. */ +/** Flags for use when performing a {@link Env#copy(File, CopyFlagSet)}. */ public enum CopyFlags implements MaskedFlag, CopyFlagSet { /** Compacting copy: Omit free space from copy, and renumber all pages sequentially. */ From 0234d323244f874ccf12945949e6352879c5a3b3 Mon Sep 17 00:00:00 2001 From: at055612 <22818309+at055612@users.noreply.github.com> Date: Tue, 28 Oct 2025 16:34:03 +0000 Subject: [PATCH 12/21] Replace get(Uns|S)ignedComparator with getComparator(DbiFlagSet) Also improve javadoc and refactor some tests to use DbiBuilder. Some tests are failing. --- src/main/java/org/lmdbjava/BufferProxy.java | 45 +++- .../java/org/lmdbjava/ByteArrayProxy.java | 17 +- src/main/java/org/lmdbjava/ByteBufProxy.java | 15 +- .../java/org/lmdbjava/ByteBufferProxy.java | 46 +++- .../java/org/lmdbjava/CursorIterable.java | 4 +- src/main/java/org/lmdbjava/Dbi.java | 7 +- src/main/java/org/lmdbjava/DbiBuilder.java | 196 +++++++++++++----- src/main/java/org/lmdbjava/DbiFlags.java | 20 +- .../java/org/lmdbjava/DirectBufferProxy.java | 17 +- src/main/java/org/lmdbjava/Env.java | 20 +- src/main/java/org/lmdbjava/Txn.java | 2 +- src/main/java/org/lmdbjava/TxnFlags.java | 1 + .../java/org/lmdbjava/ComparatorTest.java | 10 +- .../CursorIterableIntegerKeyTest.java | 121 ++++++----- .../org/lmdbjava/CursorIterablePerfTest.java | 22 +- .../java/org/lmdbjava/CursorIterableTest.java | 111 +++++----- src/test/java/org/lmdbjava/DbiTest.java | 8 +- .../java/org/lmdbjava/TestDbiBuilder.java | 6 +- 18 files changed, 448 insertions(+), 220 deletions(-) diff --git a/src/main/java/org/lmdbjava/BufferProxy.java b/src/main/java/org/lmdbjava/BufferProxy.java index 2ff5a8fc..60272209 100644 --- a/src/main/java/org/lmdbjava/BufferProxy.java +++ b/src/main/java/org/lmdbjava/BufferProxy.java @@ -66,26 +66,49 @@ protected BufferProxy() {} protected abstract byte[] getBytes(T buffer); /** - * Get a suitable default {@link Comparator} to compare numeric key values as signed. + * Get a suitable default {@link Comparator} given the provided flags. * - *

Note: LMDB's default comparator is unsigned so if this is used only for the {@link - * CursorIterable} start/stop key comparisons then its behaviour will differ from the iteration - * order. Use with caution. + *

The provided comparator must strictly match the lexicographical order of keys in the native + * LMDB database. * + * @param dbiFlagSet The {@link DbiFlags} set for the database. * @return a comparator that can be used (never null) */ - public abstract Comparator getSignedComparator(); + public abstract Comparator getComparator(final DbiFlagSet dbiFlagSet); /** - * Get a suitable default {@link Comparator} to compare numeric key values as unsigned. - *

- * This should match the behaviour of the LMDB's mdb_cmp comparator as it may be used for - * {@link CursorIterable} start/stop keys comparisons, which must match LMDB's insertion order. - *

+ * Get a suitable default {@link Comparator} + * + *

The provided comparator must strictly match the lexicographical order of keys in the native + * LMDB database. * * @return a comparator that can be used (never null) */ - public abstract Comparator getUnsignedComparator(); + public Comparator getComparator() { + return getComparator(DbiFlagSet.empty()); + } + +// /** +// * Get a suitable default {@link Comparator} to compare numeric key values as signed. +// * +// *

Note: LMDB's default comparator is unsigned so if this is used only for the {@link +// * CursorIterable} start/stop key comparisons then its behaviour will differ from the iteration +// * order. Use with caution. +// * +// * @return a comparator that can be used (never null) +// */ +// public abstract Comparator getSignedComparator(); +// +// /** +// * Get a suitable default {@link Comparator} to compare numeric key values as unsigned. +// *

+// * This should match the behaviour of the LMDB's mdb_cmp comparator as it may be used for +// * {@link CursorIterable} start/stop keys comparisons, which must match LMDB's insertion order. +// *

+// * +// * @return a comparator that can be used (never null) +// */ +// public abstract Comparator getUnsignedComparator(final DbiFlagSet dbiFlagSet); /** * Called when the MDB_val should be set to reflect the passed buffer. This buffer diff --git a/src/main/java/org/lmdbjava/ByteArrayProxy.java b/src/main/java/org/lmdbjava/ByteArrayProxy.java index b4fb1e7b..5231ed51 100644 --- a/src/main/java/org/lmdbjava/ByteArrayProxy.java +++ b/src/main/java/org/lmdbjava/ByteArrayProxy.java @@ -104,15 +104,20 @@ protected byte[] getBytes(final byte[] buffer) { } @Override - public Comparator getSignedComparator() { - return signedComparator; - } - - @Override - public Comparator getUnsignedComparator() { + public Comparator getComparator(final DbiFlagSet dbiFlagSet) { return unsignedComparator; } + // @Override +// public Comparator getSignedComparator() { +// return signedComparator; +// } +// +// @Override +// public Comparator getUnsignedComparator() { +// return unsignedComparator; +// } + @Override protected Pointer in(final byte[] buffer, final Pointer ptr) { final Pointer pointer = MEM_MGR.allocateDirect(buffer.length); diff --git a/src/main/java/org/lmdbjava/ByteBufProxy.java b/src/main/java/org/lmdbjava/ByteBufProxy.java index e26bcb07..fc14b58f 100644 --- a/src/main/java/org/lmdbjava/ByteBufProxy.java +++ b/src/main/java/org/lmdbjava/ByteBufProxy.java @@ -114,14 +114,19 @@ protected ByteBuf allocate() { } @Override - public Comparator getSignedComparator() { + public Comparator getComparator(final DbiFlagSet dbiFlagSet) { return comparator; } - @Override - public Comparator getUnsignedComparator() { - return comparator; - } + // @Override +// public Comparator getSignedComparator() { +// return comparator; +// } +// +// @Override +// public Comparator getUnsignedComparator() { +// return comparator; +// } @Override protected void deallocate(final ByteBuf buff) { diff --git a/src/main/java/org/lmdbjava/ByteBufferProxy.java b/src/main/java/org/lmdbjava/ByteBufferProxy.java index d9edb6a8..4875572b 100644 --- a/src/main/java/org/lmdbjava/ByteBufferProxy.java +++ b/src/main/java/org/lmdbjava/ByteBufferProxy.java @@ -148,6 +148,35 @@ public static int compareBuff(final ByteBuffer o1, final ByteBuffer o2) { return o1.remaining() - o2.remaining(); } +// /** +// * Possible compareBuff method specifically for 4/8 byte keys when using MDB_INTEGER_KEY +// */ +// public static int compareBuff(final ByteBuffer o1, final ByteBuffer o2) { +// requireNonNull(o1); +// requireNonNull(o2); +// // Both buffers should be same len +// final int len1 = o1.limit(); +// final int len2 = o2.limit(); +// if (len1 != len2) { +// throw new RuntimeException("Length mismatch, len1: " + len1 + ", len2: " + len2 +// + ". Lengths must be identical and either 4 or 8 bytes."); +// } +// final boolean reverse1 = o1.order() == LITTLE_ENDIAN; +// final boolean reverse2 = o2.order() == LITTLE_ENDIAN; +// if (len1 == 8) { +// final long lw = reverse1 ? Long.reverseBytes(o1.getLong()) : o1.getLong(); +// final long rw = reverse2 ? Long.reverseBytes(o2.getLong()) : o2.getLong(); +// return Long.compareUnsigned(lw, rw); +// } else if (len1 == 4) { +// final int lw = reverse1 ? Integer.reverseBytes(o1.getInt()) : o1.getInt(); +// final int rw = reverse2 ? Integer.reverseBytes(o2.getInt()) : o2.getInt(); +// return Integer.compareUnsigned(lw, rw); +// } else { +// throw new RuntimeException("Unexpected length len1: " + len1 + ", len2: " + len2 +// + ". Lengths must be identical and either 4 or 8 bytes."); +// } +// } + static Field findField(final Class c, final String name) { Class clazz = c; do { @@ -182,15 +211,20 @@ protected final ByteBuffer allocate() { } @Override - public Comparator getSignedComparator() { - return signedComparator; - } - - @Override - public Comparator getUnsignedComparator() { + public Comparator getComparator(DbiFlagSet dbiFlagSet) { return unsignedComparator; } + // @Override +// public Comparator getSignedComparator() { +// return signedComparator; +// } +// +// @Override +// public Comparator getUnsignedComparator(final DbiFlagSet dbiFlagSet) { +// return unsignedComparator; +// } + @Override protected final void deallocate(final ByteBuffer buff) { buff.order(BIG_ENDIAN); diff --git a/src/main/java/org/lmdbjava/CursorIterable.java b/src/main/java/org/lmdbjava/CursorIterable.java index dd11a468..69d43fcd 100644 --- a/src/main/java/org/lmdbjava/CursorIterable.java +++ b/src/main/java/org/lmdbjava/CursorIterable.java @@ -60,10 +60,10 @@ public final class CursorIterable implements Iterable(); if (comparator != null) { - // User supplied java-side comparator so use that + // User supplied Java-side comparator so use that this.rangeComparator = new JavaRangeComparator<>(range, comparator, entry::key); } else { - // No java-side comparator so call down to LMDB to do the comparison + // No Java-side comparator, so call down to LMDB to do the comparison this.rangeComparator = new LmdbRangeComparator<>(txn, dbi, cursor, range, proxy); } } diff --git a/src/main/java/org/lmdbjava/Dbi.java b/src/main/java/org/lmdbjava/Dbi.java index 17fa4c46..9cdc03ee 100644 --- a/src/main/java/org/lmdbjava/Dbi.java +++ b/src/main/java/org/lmdbjava/Dbi.java @@ -51,6 +51,7 @@ public final class Dbi { private final ComparatorCallback ccb; private boolean cleaned; + // Used for CursorIterable KeyRange testing and/or native callbacks private final Comparator comparator; private final Env env; private final byte[] name; @@ -80,6 +81,7 @@ public final class Dbi { ptr = dbiPtr.getPointer(0); if (nativeCb) { requireNonNull(comparator, "comparator cannot be null if nativeCb is set"); + // LMDB will call back to this comparator for insertion/iteration order this.ccb = (keyA, keyB) -> { final T compKeyA = proxy.out(proxy.allocate(), keyA); @@ -465,6 +467,7 @@ private String getNameAsString() { return ""; } else { try { + // Assume a UTF8 encoding as we don't know, thus swallow if it fails return new String(name, StandardCharsets.UTF_8); } catch (Exception e) { return "?"; @@ -475,8 +478,8 @@ private String getNameAsString() { @Override public String toString() { return "Dbi{" + - "name=" + getNameAsString() + - ", dbiFlagSet=" + dbiFlagSet + + "name='" + getNameAsString() + + "', dbiFlagSet=" + dbiFlagSet + '}'; } diff --git a/src/main/java/org/lmdbjava/DbiBuilder.java b/src/main/java/org/lmdbjava/DbiBuilder.java index 7d05a51e..f2f925ce 100644 --- a/src/main/java/org/lmdbjava/DbiBuilder.java +++ b/src/main/java/org/lmdbjava/DbiBuilder.java @@ -98,8 +98,8 @@ public static class DbiBuilderStage2 { private final DbiBuilder dbiBuilder; - private Comparator comparator; - private boolean useNativeCallback; + private java.util.Comparator customComparator; + private ComparatorType comparatorType; private DbiBuilderStage2(final DbiBuilder dbiBuilder) { this.dbiBuilder = dbiBuilder; @@ -107,95 +107,110 @@ private DbiBuilderStage2(final DbiBuilder dbiBuilder) { /** *

- * {@link CursorIterable} will call down to LMDB's {@code mdb_cmp} method when - * comparing entries to start/stop keys. This ensures LmdbJava is comparing start/stop - * keys using the same comparator that is used for insert order into the db. + * This is the default choice when it comes to choosing a comparator. + * If you are not sure of the implications of the other methods then use this one as it + * is likely what you want and also probably the most performant. *

*

- * This option may be slightly less performant than when using - * {@link DbiBuilderStage2#withDefaultIteratorComparator()} as it need to call down - * to LMDB to perform the comparisons, however it guarantees that {@link CursorIterable} - * key comparison matches LMDB key comparison. + * With this option, {@link CursorIterable} will make use of the LmdbJava's default + * Java-side comparators when comparing iteration keys to the start/stop keys. + * LMDB will use its own comparator for controlling insertion order in the database. + * The two comparators are functionally identical. + *

+ *

+ * This option may be slightly more performant than when using + * {@link DbiBuilderStage2#withNativeComparator()} which calls down to LMDB for ALL + * comparison operations. *

*

* If you do not intend to use {@link CursorIterable} then it doesn't matter whether * you choose {@link DbiBuilderStage2#withNativeComparator()}, - * {@link DbiBuilderStage2#withDefaultIteratorComparator()} or + * {@link DbiBuilderStage2#withDefaultComparator()} or * {@link DbiBuilderStage2#withIteratorComparator(Comparator)} as these comparators will * never be used. *

* * @return The next builder stage. */ - public DbiBuilderStage3 withNativeComparator() { - this.comparator = null; - this.useNativeCallback = false; + public DbiBuilderStage3 withDefaultComparator() { + this.comparatorType = ComparatorType.DEFAULT; return new DbiBuilderStage3<>(this); } /** *

- * {@link CursorIterable} will make use of the default Java-side comparators when - * comparing entries to start/stop keys. + * With this option, {@link CursorIterable} will call down to LMDB's {@code mdb_cmp} method when + * comparing iteration keys to start/stop keys. This ensures LmdbJava is comparing start/stop + * keys using the same comparator that is used for insertion order into the db. *

*

- * This option may be slightly more performant than when using - * {@link DbiBuilderStage2#withNativeComparator()} but it relies on the default comparator - * in LmdbJava behaving identically to the comparator in LMDB. + * This option may be slightly less performant than when using + * {@link DbiBuilderStage2#withDefaultComparator()} as it needs to call down + * to LMDB to perform the comparisons, however it guarantees that {@link CursorIterable} + * key comparison matches LMDB key comparison. *

*

* If you do not intend to use {@link CursorIterable} then it doesn't matter whether * you choose {@link DbiBuilderStage2#withNativeComparator()}, - * {@link DbiBuilderStage2#withDefaultIteratorComparator()} or + * {@link DbiBuilderStage2#withDefaultComparator()} or * {@link DbiBuilderStage2#withIteratorComparator(Comparator)} as these comparators will * never be used. *

* * @return The next builder stage. */ - public DbiBuilderStage3 withDefaultIteratorComparator() { - this.comparator = dbiBuilder.proxy.getUnsignedComparator(); - this.useNativeCallback = false; + public DbiBuilderStage3 withNativeComparator() { + this.comparatorType = ComparatorType.NATIVE; return new DbiBuilderStage3<>(this); } + /** - * Provide a java-side {@link Comparator} that LMDB will call back to in order to - * manage database insertion/iteration order. It will also be used for {@link CursorIterable} - * start/stop key comparisons. + * Provide a java-side {@link Comparator} that LMDB will call back to for all + * comparison operations. + * Therefore, it will be called by LMDB to manage database insertion/iteration order. + * It will also be used for {@link CursorIterable} start/stop key comparisons. *

- * Due to calling back to java, this will be less performant than using LMDB's - * default comparator, but allows for total control over the order in which entries + * It can be useful if you need to sort your database using some other method, + * e.g. signed keys or case-insensitive order. + * Note, if you need keys stored in reverse order, see {@link DbiFlags#MDB_REVERSEKEY} + * and {@link DbiFlags#MDB_REVERSEDUP}. + *

+ *

+ * As this requires LMDB to call back to java, this will be less performant than using LMDB's + * default comparators, but allows for total control over the order in which entries * are stored in the database. *

* * @param comparator for all key comparison operations. * @return The next builder stage. */ - public DbiBuilderStage3 withCallbackIteratorComparator(final Comparator comparator) { - this.comparator = Objects.requireNonNull(comparator); - this.useNativeCallback = true; + public DbiBuilderStage3 withCallbackComparator(final Comparator comparator) { + this.customComparator = Objects.requireNonNull(comparator); + this.comparatorType = ComparatorType.CALLBACK; return new DbiBuilderStage3<>(this); } /** + *
*

- * {@link CursorIterable} will make use of the passed comparator for - * comparing entries to start/stop keys. It has NO bearing on the insert/iteration - * order of the db. + * WARNING: Only use this if you fully understand the risks and implications. *

+ *
*

- * WARNING: Only call this method if you fully understand the implications - * of using a comparator for the {@link CursorIterable} start/stop keys that behaves - * differently to the comparator in LMDB that controls the insert/iteration order. + * With this option, {@link CursorIterable} will make use of the passed comparator for + * comparing iteration keys to start/stop keys. It has NO bearing on the + * insert/iteration order of the database (which is controlled by LMDB's own comparators). *

*

- * The supplied {@link Comparator} should match the behaviour of LMDB's mdb_cmp comparator. + * It is vital that this comparator is functionally identical to the one + * used internally in LMDB for insertion/iteration order, else you will see unexpected behaviour + * when using {@link CursorIterable}. *

*

* If you do not intend to use {@link CursorIterable} then it doesn't matter whether * you choose {@link DbiBuilderStage2#withNativeComparator()}, - * {@link DbiBuilderStage2#withDefaultIteratorComparator()} or + * {@link DbiBuilderStage2#withDefaultComparator()} or * {@link DbiBuilderStage2#withIteratorComparator(Comparator)} as these comparators will * never be used. *

@@ -204,8 +219,8 @@ public DbiBuilderStage3 withCallbackIteratorComparator(final Comparator co * @return The next builder stage. */ public DbiBuilderStage3 withIteratorComparator(final Comparator comparator) { - this.comparator = Objects.requireNonNull(comparator); - this.useNativeCallback = false; + this.customComparator = Objects.requireNonNull(comparator); + this.comparatorType = ComparatorType.ITERATOR; return new DbiBuilderStage3<>(this); } } @@ -234,14 +249,18 @@ private DbiBuilderStage3(DbiBuilderStage2 dbiBuilderStage2) { * Apply all the dbi flags supplied in dbiFlags. *

*

- * Replaces any flags applies in previous calls to - * {@link DbiBuilderStage3#withDbiFlags(Collection)}, {@link DbiBuilderStage3#withDbiFlags(DbiFlags...)} + * Clears all flags currently set by previous calls to + * {@link DbiBuilderStage3#withDbiFlags(Collection)}, + * {@link DbiBuilderStage3#withDbiFlags(DbiFlags...)} * or {@link DbiBuilderStage3#setDbiFlag(DbiFlags)}. *

* * @param dbiFlags to open the database with. + * A null {@link Collection} will just clear all set flags. + * Null items are ignored. */ public DbiBuilderStage3 withDbiFlags(final Collection dbiFlags) { + flagSetBuilder.clear(); if (dbiFlags != null) { dbiFlags.stream() .filter(Objects::nonNull) @@ -255,14 +274,15 @@ public DbiBuilderStage3 withDbiFlags(final Collection dbiFlags) { * Apply all the dbi flags supplied in dbiFlags. *

*

- * Replaces any flags applies in previous calls to + * Clears all flags currently set by previous calls to * {@link DbiBuilderStage3#withDbiFlags(Collection)}, * {@link DbiBuilderStage3#withDbiFlags(DbiFlags...)} * or {@link DbiBuilderStage3#setDbiFlag(DbiFlags)}. *

* * @param dbiFlags to open the database with. - * A null array is a no-op. Null items are ignored. + * A null array will just clear all set flags. + * Null items are ignored. */ public DbiBuilderStage3 withDbiFlags(final DbiFlags... dbiFlags) { flagSetBuilder.clear(); @@ -275,7 +295,29 @@ public DbiBuilderStage3 withDbiFlags(final DbiFlags... dbiFlags) { } /** - * Adds dbiFlag to those flags already added to this builder by + *

+ * Apply all the dbi flags supplied in dbiFlags. + *

+ *

+ * Clears all flags currently set by previous calls to + * {@link DbiBuilderStage3#withDbiFlags(Collection)}, + * {@link DbiBuilderStage3#withDbiFlags(DbiFlags...)} + * or {@link DbiBuilderStage3#setDbiFlag(DbiFlags)}. + *

+ * + * @param dbiFlagSet to open the database with. + * A null value will just clear all set flags. + */ + public DbiBuilderStage3 withDbiFlags(final DbiFlagSet dbiFlagSet) { + flagSetBuilder.clear(); + if (dbiFlagSet != null) { + this.flagSetBuilder.withFlags(dbiFlagSet.getFlags()); + } + return this; + } + + /** + * Adds a dbiFlag to those flags already added to this builder by * {@link DbiBuilderStage3#withDbiFlags(DbiFlags...)}, * {@link DbiBuilderStage3#withDbiFlags(Collection)} * or {@link DbiBuilderStage3#setDbiFlag(DbiFlags)}. @@ -294,8 +336,15 @@ public DbiBuilderStage3 setDbiFlag(final DbiFlags dbiFlag) { * The caller MUST commit the transaction after calling {@link DbiBuilderStage3#open()}, * in order to retain the Dbi in the Env. *

+ *

+ * If you don't call this method to supply a {@link Txn}, a {@link Txn} will be opened for the purpose + * of creating and opening the {@link Dbi}, then closed. Therefore, if you already have a transaction + * open, you should supply that to avoid one blocking the other. + *

* - * @param txn transaction to use (required; not closed) + * @param txn transaction to use (required; not closed). If the {@link Env} was opened + * with the {@link EnvFlags#MDB_RDONLY_ENV} flag, the {@link Txn} can be read-only, + * else it needs to be a read/write {@link Txn}. * @return this builder instance. */ public DbiBuilderStage3 withTxn(final Txn txn) { @@ -329,18 +378,69 @@ private Txn getTxn(final DbiBuilder dbiBuilder) { : dbiBuilder.env.txnWrite(); } + private Comparator getComparator(final DbiBuilder dbiBuilder, + final ComparatorType comparatorType, + final DbiFlagSet dbiFlagSet) { + Comparator comparator = null; + switch (comparatorType) { + case DEFAULT: + // Get the appropriate default CursorIterable comparator based on the DbiFlags, + // e.g. MDB_INTEGERKEY may benefit from an optimised comparator. + comparator = dbiBuilder.proxy.getComparator(dbiFlagSet); + break; + case CALLBACK: + case ITERATOR: + comparator = dbiBuilderStage2.customComparator; + break; + case NATIVE: + break; + default: + throw new IllegalStateException("Unexpected comparatorType " + comparatorType); + } + return comparator; + } + private Dbi open(final Txn txn, final DbiBuilder dbiBuilder) { final DbiFlagSet dbiFlagSet = flagSetBuilder.build(); + final ComparatorType comparatorType = dbiBuilderStage2.comparatorType; + final Comparator comparator = getComparator(dbiBuilder, comparatorType, dbiFlagSet); + final boolean useNativeCallback = comparatorType == ComparatorType.CALLBACK; return new Dbi<>( dbiBuilder.env, txn, dbiBuilder.name, - dbiBuilderStage2.comparator, - dbiBuilderStage2.useNativeCallback, + comparator, + useNativeCallback, dbiBuilder.proxy, dbiFlagSet); } } + + + // -------------------------------------------------------------------------------- + + + private enum ComparatorType { + /** + * Default Java comparator for {@link CursorIterable} KeyRange testing, + * LMDB comparator for insertion/iteration order. + */ + DEFAULT, + /** + * Use LMDB native comparator for everything. + */ + NATIVE, + /** + * Use the supplied custom Java-side comparator for everything. + */ + CALLBACK, + /** + * Use the supplied custom Java-side comparator for {@link CursorIterable} KeyRange testing, + * LMDB comparator for insertion/iteration order. + */ + ITERATOR, + ; + } } diff --git a/src/main/java/org/lmdbjava/DbiFlags.java b/src/main/java/org/lmdbjava/DbiFlags.java index 6e4b723d..10952da9 100644 --- a/src/main/java/org/lmdbjava/DbiFlags.java +++ b/src/main/java/org/lmdbjava/DbiFlags.java @@ -32,7 +32,7 @@ public enum DbiFlags implements MaskedFlag, DbiFlagSet { * Use sorted duplicates. * *

Duplicate keys may be used in the database. Or, from another perspective, keys may have - * multiple data items, stored in sorted order. By default keys must be unique and may have only a + * multiple data items, stored in sorted order. By default, keys must be unique and may have only a * single data item. *

* @@ -40,8 +40,22 @@ public enum DbiFlags implements MaskedFlag, DbiFlagSet { */ MDB_DUPSORT(0x04), /** - * Numeric keys in native byte order: either unsigned int or size_t. The keys must all be of the - * same size. + * Numeric keys in native byte order: either unsigned int or size_t. + * The keys must all be of the same size. + *

+ * This is an optimisation that is available when your keys are 4 or 8 byte unsigned numeric values. + * There are performance benefits for both ordered and un-ordered puts as compared to not using + * this flag. + *

+ *

+ * When writing the key to the buffer you must write it in native order and subsequently read any + * keys retrieved from LMDB (via cursor or get method) also using native order. + *

+ *

+ * For more information, see + * Numeric Keys + * in the LmdbJava wiki. + *

*/ MDB_INTEGERKEY(0x08), /** diff --git a/src/main/java/org/lmdbjava/DirectBufferProxy.java b/src/main/java/org/lmdbjava/DirectBufferProxy.java index 1aab2573..514c04ab 100644 --- a/src/main/java/org/lmdbjava/DirectBufferProxy.java +++ b/src/main/java/org/lmdbjava/DirectBufferProxy.java @@ -111,15 +111,20 @@ protected DirectBuffer allocate() { } @Override - public Comparator getSignedComparator() { - return signedComparator; - } - - @Override - public Comparator getUnsignedComparator() { + public Comparator getComparator(final DbiFlagSet dbiFlagSet) { return unsignedComparator; } + // @Override +// public Comparator getSignedComparator() { +// return signedComparator; +// } +// +// @Override +// public Comparator getUnsignedComparator(final DbiFlagSet dbiFlagSet) { +// return unsignedComparator; +// } + @Override protected void deallocate(final DirectBuffer buff) { final ArrayDeque q = BUFFERS.get(); diff --git a/src/main/java/org/lmdbjava/Env.java b/src/main/java/org/lmdbjava/Env.java index 2ad929ad..efc4e240 100644 --- a/src/main/java/org/lmdbjava/Env.java +++ b/src/main/java/org/lmdbjava/Env.java @@ -270,13 +270,13 @@ public DbiBuilder buildDbi() { } /** + * @deprecated Instead use {@link Env#buildDbi()} * Convenience method that opens a {@link Dbi} with a UTF-8 database name and default {@link * Comparator} that is not invoked from native code. * * @param name name of the database (or null if no name is required) * @param flags to open the database with * @return a database that is ready to use - * @deprecated Instead use {@link Env#buildDbi()} */ @Deprecated() public Dbi openDbi(final String name, final DbiFlags... flags) { @@ -285,6 +285,7 @@ public Dbi openDbi(final String name, final DbiFlags... flags) { } /** + * @deprecated Instead use {@link Env#buildDbi()} * Convenience method that opens a {@link Dbi} with a UTF-8 database name and associated {@link * Comparator} for use by {@link CursorIterable} when comparing start/stop keys. * @@ -298,7 +299,6 @@ public Dbi openDbi(final String name, final DbiFlags... flags) { * comparator will be used. * @param flags to open the database with * @return a database that is ready to use - * @deprecated Instead use {@link Env#buildDbi()} */ @Deprecated() public Dbi openDbi( @@ -308,6 +308,7 @@ public Dbi openDbi( } /** + * @deprecated Instead use {@link Env#buildDbi()} * Convenience method that opens a {@link Dbi} with a UTF-8 database name and associated {@link * Comparator}. The comparator will be used by {@link CursorIterable} when comparing start/stop * keys as a minimum. If nativeCb is {@code true}, this comparator will also be called by LMDB to @@ -320,7 +321,6 @@ public Dbi openDbi( * @param nativeCb whether LMDB native code calls back to the Java comparator * @param flags to open the database with * @return a database that is ready to use - * @deprecated Instead use {@link Env#buildDbi()} */ @Deprecated() public Dbi openDbi( @@ -333,13 +333,13 @@ public Dbi openDbi( } /** + * @deprecated Instead use {@link Env#buildDbi()} * Convenience method that opens a {@link Dbi} with a default {@link Comparator} that is not * invoked from native code. * * @param name name of the database (or null if no name is required) * @param flags to open the database with * @return a database that is ready to use - * @deprecated Instead use {@link Env#buildDbi()} */ @Deprecated() public Dbi openDbi(final byte[] name, final DbiFlags... flags) { @@ -347,6 +347,7 @@ public Dbi openDbi(final byte[] name, final DbiFlags... flags) { } /** + * @deprecated Instead use {@link Env#buildDbi()} * Convenience method that opens a {@link Dbi} with an associated {@link Comparator} that is not * invoked from native code. * @@ -354,7 +355,6 @@ public Dbi openDbi(final byte[] name, final DbiFlags... flags) { * @param comparator custom comparator callback (or null to use LMDB default) * @param flags to open the database with * @return a database that is ready to use - * @deprecated Instead use {@link Env#buildDbi()} */ @Deprecated() public Dbi openDbi( @@ -363,6 +363,7 @@ public Dbi openDbi( } /** + * @deprecated Instead use {@link Env#buildDbi()} * Convenience method that opens a {@link Dbi} with an associated {@link Comparator} that may be * invoked from native code if specified. * @@ -374,7 +375,6 @@ public Dbi openDbi( * @param nativeCb whether native code calls back to the Java comparator * @param flags to open the database with * @return a database that is ready to use - * @deprecated Instead use {@link Env#buildDbi()} */ @Deprecated() public Dbi openDbi( @@ -390,6 +390,7 @@ public Dbi openDbi( } /** + * @deprecated Instead use {@link Env#buildDbi()} * Open the {@link Dbi} using the passed {@link Txn}. * *

The caller must commit the transaction after this method returns in order to retain the @@ -412,10 +413,9 @@ public Dbi openDbi( * @param txn transaction to use (required; not closed) * @param name name of the database (or null if no name is required) * @param comparator custom comparator callback (or null to use LMDB default) - * @param nativeCb whether native code should call back to the comparator + * @param nativeCb whether native LMDB code should call back to the Java comparator * @param flags to open the database with * @return a database that is ready to use - * @deprecated Instead use {@link Env#buildDbi()} */ @Deprecated() public Dbi openDbi( @@ -424,6 +424,10 @@ public Dbi openDbi( final Comparator comparator, final boolean nativeCb, final DbiFlags... flags) { + + if (nativeCb && comparator == null) { + throw new IllegalArgumentException("Is nativeCb is true, you must supply a comparator"); + } return new Dbi<>(this, txn, name, comparator, nativeCb, proxy, DbiFlagSet.of(flags)); } diff --git a/src/main/java/org/lmdbjava/Txn.java b/src/main/java/org/lmdbjava/Txn.java index 5c8440fd..dc57fc66 100644 --- a/src/main/java/org/lmdbjava/Txn.java +++ b/src/main/java/org/lmdbjava/Txn.java @@ -164,7 +164,7 @@ public void renew() { } /** - * Aborts this read-only transaction and resets the transaction handle so it can be reused upon + * Aborts this read-only transaction and resets the transaction handle, so it can be reused upon * calling {@link #renew()}. */ public void reset() { diff --git a/src/main/java/org/lmdbjava/TxnFlags.java b/src/main/java/org/lmdbjava/TxnFlags.java index 8e4ea757..94112957 100644 --- a/src/main/java/org/lmdbjava/TxnFlags.java +++ b/src/main/java/org/lmdbjava/TxnFlags.java @@ -20,6 +20,7 @@ /** Flags for use when creating a {@link Txn}. */ public enum TxnFlags implements MaskedFlag, TxnFlagSet { + /** Read only. */ MDB_RDONLY_TXN(0x2_0000); diff --git a/src/test/java/org/lmdbjava/ComparatorTest.java b/src/test/java/org/lmdbjava/ComparatorTest.java index 446b8545..1b00c71c 100644 --- a/src/test/java/org/lmdbjava/ComparatorTest.java +++ b/src/test/java/org/lmdbjava/ComparatorTest.java @@ -136,7 +136,7 @@ private static final class ByteArrayRunner implements ComparatorRunner { @Override public int compare(final byte[] o1, final byte[] o2) { - final Comparator c = PROXY_BA.getUnsignedComparator(); + final Comparator c = PROXY_BA.getComparator(); return c.compare(o1, o2); } } @@ -146,7 +146,7 @@ private static final class UnsignedByteArrayRunner implements ComparatorRunner { @Override public int compare(final byte[] o1, final byte[] o2) { - final Comparator c = PROXY_BA.getUnsignedComparator(); + final Comparator c = PROXY_BA.getComparator(); return c.compare(o1, o2); } } @@ -156,7 +156,7 @@ private static final class ByteBufferRunner implements ComparatorRunner { @Override public int compare(final byte[] o1, final byte[] o2) { - final Comparator c = PROXY_OPTIMAL.getUnsignedComparator(); + final Comparator c = PROXY_OPTIMAL.getComparator(); // Convert arrays to buffers that are larger than the array, with // limit set at the array length. One buffer bigger than the other. @@ -200,7 +200,7 @@ private static final class DirectBufferRunner implements ComparatorRunner { public int compare(final byte[] o1, final byte[] o2) { final DirectBuffer o1b = new UnsafeBuffer(o1); final DirectBuffer o2b = new UnsafeBuffer(o2); - final Comparator c = PROXY_DB.getUnsignedComparator(); + final Comparator c = PROXY_DB.getComparator(); return c.compare(o1b, o2b); } } @@ -234,7 +234,7 @@ public int compare(final byte[] o1, final byte[] o2) { final ByteBuf o2b = DEFAULT.directBuffer(o2.length); o1b.writeBytes(o1); o2b.writeBytes(o2); - final Comparator c = PROXY_NETTY.getUnsignedComparator(); + final Comparator c = PROXY_NETTY.getComparator(); return c.compare(o1b, o2b); } } diff --git a/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java b/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java index 431c6e51..aefe9d43 100644 --- a/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java +++ b/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java @@ -135,18 +135,27 @@ public void before() throws IOException { .open(path, POSIX_MODE, MDB_NOSUBDIR); // Use a java comparator for start/stop keys only - dbJavaComparator = env.openDbi(DB_1, - bufferProxy.getUnsignedComparator(), - MDB_CREATE, - MDB_INTEGERKEY); + DbiFlagSet dbiFlagSet = DbiFlagSet.of(MDB_CREATE, MDB_INTEGERKEY); + + dbJavaComparator = env.buildDbi() + .withDbName(DB_1) + .withIteratorComparator(bufferProxy.getComparator(dbiFlagSet)) + .withDbiFlags(dbiFlagSet) + .open(); + // Use LMDB comparator for start/stop keys - dbLmdbComparator = env.openDbi(DB_2, MDB_CREATE, MDB_INTEGERKEY); + dbLmdbComparator = env.buildDbi() + .withDbName(DB_2) + .withDefaultComparator() + .withDbiFlags(dbiFlagSet) + .open(); + // Use a java comparator for start/stop keys and as a callback comparaotr - dbCallbackComparator = env.openDbi(DB_3, - bufferProxy.getUnsignedComparator(), - true, - MDB_CREATE, - MDB_INTEGERKEY); + dbCallbackComparator = env.buildDbi() + .withDbName(DB_3) + .withCallbackComparator(bufferProxy.getComparator(dbiFlagSet)) + .withDbiFlags(dbiFlagSet) + .open(); populateList(); @@ -411,52 +420,52 @@ public void forEachRemainingWithClosedEnvTest() { } } - @Test - public void testSignedVsUnsigned() { - final ByteBuffer val1 = bbNative(1); - final ByteBuffer val2 = bbNative(2); - final ByteBuffer val110 = bbNative(110); - final ByteBuffer val111 = bbNative(111); - final ByteBuffer val150 = bbNative(150); - - final BufferProxy bufferProxy = ByteBufferProxy.PROXY_OPTIMAL; - final Comparator unsignedComparator = bufferProxy.getUnsignedComparator(); - final Comparator signedComparator = bufferProxy.getSignedComparator(); - - // Compare the same - assertThat( - unsignedComparator.compare(val1, val2), Matchers.is(signedComparator.compare(val1, val2))); - - // Compare differently - assertThat( - unsignedComparator.compare(val110, val150), - Matchers.not(signedComparator.compare(val110, val150))); - - // Compare differently - assertThat( - unsignedComparator.compare(val111, val150), - Matchers.not(signedComparator.compare(val111, val150))); - - // This will fail if the db is using a signed comparator for the start/stop keys - for (final Dbi db : dbs) { - db.put(val110, val110); - db.put(val150, val150); - - final ByteBuffer startKeyBuf = val111; - KeyRange keyRange = KeyRange.atLeastBackward(startKeyBuf); - - try (Txn txn = env.txnRead(); - CursorIterable c = db.iterate(txn, keyRange)) { - for (final KeyVal kv : c) { - final int key = getNativeInt(kv.key()); - final int val = kv.val().getInt(); - // System.out.println("key: " + key + " val: " + val); - assertThat(key, is(110)); - break; - } - } - } - } +// @Test +// public void testSignedVsUnsigned() { +// final ByteBuffer val1 = bbNative(1); +// final ByteBuffer val2 = bbNative(2); +// final ByteBuffer val110 = bbNative(110); +// final ByteBuffer val111 = bbNative(111); +// final ByteBuffer val150 = bbNative(150); +// +// final BufferProxy bufferProxy = ByteBufferProxy.PROXY_OPTIMAL; +// final Comparator unsignedComparator = bufferProxy.getUnsignedComparator(); +// final Comparator signedComparator = bufferProxy.getSignedComparator(); +// +// // Compare the same +// assertThat( +// unsignedComparator.compare(val1, val2), Matchers.is(signedComparator.compare(val1, val2))); +// +// // Compare differently +// assertThat( +// unsignedComparator.compare(val110, val150), +// Matchers.not(signedComparator.compare(val110, val150))); +// +// // Compare differently +// assertThat( +// unsignedComparator.compare(val111, val150), +// Matchers.not(signedComparator.compare(val111, val150))); +// +// // This will fail if the db is using a signed comparator for the start/stop keys +// for (final Dbi db : dbs) { +// db.put(val110, val110); +// db.put(val150, val150); +// +// final ByteBuffer startKeyBuf = val111; +// KeyRange keyRange = KeyRange.atLeastBackward(startKeyBuf); +// +// try (Txn txn = env.txnRead(); +// CursorIterable c = db.iterate(txn, keyRange)) { +// for (final KeyVal kv : c) { +// final int key = getNativeInt(kv.key()); +// final int val = kv.val().getInt(); +// // System.out.println("key: " + key + " val: " + val); +// assertThat(key, is(110)); +// break; +// } +// } +// } +// } private void verify(final KeyRange range, final int... expected) { // Verify using all comparator types diff --git a/src/test/java/org/lmdbjava/CursorIterablePerfTest.java b/src/test/java/org/lmdbjava/CursorIterablePerfTest.java index f77ac4d6..e2c54346 100644 --- a/src/test/java/org/lmdbjava/CursorIterablePerfTest.java +++ b/src/test/java/org/lmdbjava/CursorIterablePerfTest.java @@ -65,14 +65,26 @@ public void before() throws IOException { .setMaxDbs(3) .open(path, POSIX_MODE, MDB_NOSUBDIR); + final DbiFlagSet dbiFlagSet = MDB_CREATE; // Use a java comparator for start/stop keys only - dbJavaComparator = - env.openDbi("JavaComparator", bufferProxy.getUnsignedComparator(), MDB_CREATE); + dbJavaComparator = env.buildDbi() + .withDbName("JavaComparator") + .withDefaultComparator() + .withDbiFlags(dbiFlagSet) + .open(); // Use LMDB comparator for start/stop keys - dbLmdbComparator = env.openDbi("LmdbComparator", MDB_CREATE); + dbLmdbComparator = env.buildDbi() + .withDbName("LmdbComparator") + .withNativeComparator() + .withDbiFlags(dbiFlagSet) + .open(); + // Use a java comparator for start/stop keys and as a callback comparator - dbCallbackComparator = - env.openDbi("CallBackComparator", bufferProxy.getUnsignedComparator(), true, MDB_CREATE); + dbCallbackComparator = env.buildDbi() + .withDbName("CallBackComparator") + .withCallbackComparator(bufferProxy.getComparator(dbiFlagSet)) + .withDbiFlags(dbiFlagSet) + .open(); dbs.add(dbJavaComparator); dbs.add(dbLmdbComparator); diff --git a/src/test/java/org/lmdbjava/CursorIterableTest.java b/src/test/java/org/lmdbjava/CursorIterableTest.java index 22cf7361..79a9a34c 100644 --- a/src/test/java/org/lmdbjava/CursorIterableTest.java +++ b/src/test/java/org/lmdbjava/CursorIterableTest.java @@ -129,12 +129,25 @@ public void before() throws IOException { .setMaxDbs(3) .open(path, POSIX_MODE, MDB_NOSUBDIR); + final DbiFlagSet dbiFlagSet = MDB_CREATE; // Use a java comparator for start/stop keys only - dbJavaComparator = env.openDbi(DB_1, bufferProxy.getUnsignedComparator(), MDB_CREATE); + dbJavaComparator = env.buildDbi() + .withDbName(DB_1) + .withDefaultComparator() + .withDbiFlags(dbiFlagSet) + .open(); // Use LMDB comparator for start/stop keys - dbLmdbComparator = env.openDbi(DB_2, MDB_CREATE); + dbLmdbComparator = env.buildDbi() + .withDbName(DB_2) + .withNativeComparator() + .withDbiFlags(dbiFlagSet) + .open(); // Use a java comparator for start/stop keys and as a callback comparaotr - dbCallbackComparator = env.openDbi(DB_3, bufferProxy.getUnsignedComparator(), true, MDB_CREATE); + dbCallbackComparator = env.buildDbi() + .withDbName(DB_3) + .withCallbackComparator(bufferProxy.getComparator(dbiFlagSet)) + .withDbiFlags(dbiFlagSet) + .open(); populateList(); @@ -407,52 +420,52 @@ public void forEachRemainingWithClosedEnvTest() { } } - @Test - public void testSignedVsUnsigned() { - final ByteBuffer val1 = bb(1); - final ByteBuffer val2 = bb(2); - final ByteBuffer val110 = bb(110); - final ByteBuffer val111 = bb(111); - final ByteBuffer val150 = bb(150); - - final BufferProxy bufferProxy = ByteBufferProxy.PROXY_OPTIMAL; - final Comparator unsignedComparator = bufferProxy.getUnsignedComparator(); - final Comparator signedComparator = bufferProxy.getSignedComparator(); - - // Compare the same - assertThat( - unsignedComparator.compare(val1, val2), Matchers.is(signedComparator.compare(val1, val2))); - - // Compare differently - assertThat( - unsignedComparator.compare(val110, val150), - Matchers.not(signedComparator.compare(val110, val150))); - - // Compare differently - assertThat( - unsignedComparator.compare(val111, val150), - Matchers.not(signedComparator.compare(val111, val150))); - - // This will fail if the db is using a signed comparator for the start/stop keys - for (final Dbi db : dbs) { - db.put(val110, val110); - db.put(val150, val150); - - final ByteBuffer startKeyBuf = val111; - KeyRange keyRange = KeyRange.atLeastBackward(startKeyBuf); - - try (Txn txn = env.txnRead(); - CursorIterable c = db.iterate(txn, keyRange)) { - for (final CursorIterable.KeyVal kv : c) { - final int key = kv.key().getInt(); - final int val = kv.val().getInt(); - // System.out.println("key: " + key + " val: " + val); - assertThat(key, is(110)); - break; - } - } - } - } +// @Test +// public void testSignedVsUnsigned() { +// final ByteBuffer val1 = bb(1); +// final ByteBuffer val2 = bb(2); +// final ByteBuffer val110 = bb(110); +// final ByteBuffer val111 = bb(111); +// final ByteBuffer val150 = bb(150); +// +// final BufferProxy bufferProxy = ByteBufferProxy.PROXY_OPTIMAL; +// final Comparator unsignedComparator = bufferProxy.getUnsignedComparator(); +// final Comparator signedComparator = bufferProxy.getSignedComparator(); +// +// // Compare the same +// assertThat( +// unsignedComparator.compare(val1, val2), Matchers.is(signedComparator.compare(val1, val2))); +// +// // Compare differently +// assertThat( +// unsignedComparator.compare(val110, val150), +// Matchers.not(signedComparator.compare(val110, val150))); +// +// // Compare differently +// assertThat( +// unsignedComparator.compare(val111, val150), +// Matchers.not(signedComparator.compare(val111, val150))); +// +// // This will fail if the db is using a signed comparator for the start/stop keys +// for (final Dbi db : dbs) { +// db.put(val110, val110); +// db.put(val150, val150); +// +// final ByteBuffer startKeyBuf = val111; +// KeyRange keyRange = KeyRange.atLeastBackward(startKeyBuf); +// +// try (Txn txn = env.txnRead(); +// CursorIterable c = db.iterate(txn, keyRange)) { +// for (final CursorIterable.KeyVal kv : c) { +// final int key = kv.key().getInt(); +// final int val = kv.val().getInt(); +// // System.out.println("key: " + key + " val: " + val); +// assertThat(key, is(110)); +// break; +// } +// } +// } +// } private void verify(final KeyRange range, final int... expected) { // Verify using all comparator types diff --git a/src/test/java/org/lmdbjava/DbiTest.java b/src/test/java/org/lmdbjava/DbiTest.java index 37f141e9..2209e614 100644 --- a/src/test/java/org/lmdbjava/DbiTest.java +++ b/src/test/java/org/lmdbjava/DbiTest.java @@ -118,7 +118,7 @@ public void close() { public void customComparator() { final Comparator reverseOrder = (o1, o2) -> { - final int lexical = PROXY_OPTIMAL.getUnsignedComparator().compare(o1, o2); + final int lexical = PROXY_OPTIMAL.getComparator().compare(o1, o2); if (lexical == 0) { return 0; } @@ -131,7 +131,7 @@ public void customComparator() { public void customComparatorByteArray() { final Comparator reverseOrder = (o1, o2) -> { - final int lexical = PROXY_BA.getUnsignedComparator().compare(o1, o2); + final int lexical = PROXY_BA.getComparator().compare(o1, o2); if (lexical == 0) { return 0; } @@ -172,13 +172,13 @@ public void dbOpenMaxDatabases() { @Test public void dbiWithComparatorThreadSafety() { doDbiWithComparatorThreadSafety( - env, PROXY_OPTIMAL::getUnsignedComparator, TestUtils::bb, ByteBuffer::getInt); + env, PROXY_OPTIMAL::getComparator, TestUtils::bb, ByteBuffer::getInt); } @Test public void dbiWithComparatorThreadSafetyByteArray() { doDbiWithComparatorThreadSafety( - envBa, PROXY_BA::getUnsignedComparator, TestUtils::ba, TestUtils::fromBa); + envBa, PROXY_BA::getComparator, TestUtils::ba, TestUtils::fromBa); } public void doDbiWithComparatorThreadSafety( diff --git a/src/test/java/org/lmdbjava/TestDbiBuilder.java b/src/test/java/org/lmdbjava/TestDbiBuilder.java index 15b70812..7a1e8947 100644 --- a/src/test/java/org/lmdbjava/TestDbiBuilder.java +++ b/src/test/java/org/lmdbjava/TestDbiBuilder.java @@ -10,7 +10,6 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; -import junit.framework.TestCase; import org.hamcrest.Matchers; import org.junit.After; import org.junit.Before; @@ -31,6 +30,7 @@ public void after() { @Before public void before() throws IOException { + System.out.println("before"); final File path = tmp.newFile(); env = create() @@ -44,7 +44,7 @@ public void before() throws IOException { public void unnamed() { final Dbi dbi = env.buildDbi() .withoutDbName() - .withDefaultIteratorComparator() + .withDefaultComparator() .withDbiFlags(DbiFlags.MDB_CREATE) .open(); @@ -58,7 +58,7 @@ public void unnamed() { public void named() { final Dbi dbi = env.buildDbi() .withDbName("foo") - .withDefaultIteratorComparator() + .withDefaultComparator() .withDbiFlags(DbiFlags.MDB_CREATE) .open(); From ef0c852ad9a631ae0f194f4d63dce62e3d1b549f Mon Sep 17 00:00:00 2001 From: at055612 <22818309+at055612@users.noreply.github.com> Date: Tue, 28 Oct 2025 17:31:32 +0000 Subject: [PATCH 13/21] Add missing txn commit in DbiBuilder --- .../java/org/lmdbjava/AbstractFlagSet.java | 15 +++ src/main/java/org/lmdbjava/CopyFlagSet.java | 15 +++ src/main/java/org/lmdbjava/Dbi.java | 11 +-- src/main/java/org/lmdbjava/DbiBuilder.java | 15 +-- src/main/java/org/lmdbjava/DbiFlagSet.java | 15 +++ src/main/java/org/lmdbjava/EnvFlagSet.java | 15 +++ src/main/java/org/lmdbjava/FlagSet.java | 15 +++ src/main/java/org/lmdbjava/PutFlagSet.java | 15 +++ src/main/java/org/lmdbjava/ReferenceUtil.java | 1 + src/main/java/org/lmdbjava/Txn.java | 7 +- src/main/java/org/lmdbjava/TxnFlagSet.java | 15 +++ .../java/org/lmdbjava/CopyFlagSetTest.java | 15 +++ .../java/org/lmdbjava/DbiBuilderTest.java | 97 +++++++++++++++++++ .../java/org/lmdbjava/DbiFlagSetTest.java | 15 +++ .../java/org/lmdbjava/EnvFlagSetTest.java | 15 +++ .../java/org/lmdbjava/PutFlagSetTest.java | 15 +++ .../java/org/lmdbjava/TestDbiBuilder.java | 83 ---------------- .../java/org/lmdbjava/TxnFlagSetTest.java | 15 +++ 18 files changed, 296 insertions(+), 98 deletions(-) create mode 100644 src/test/java/org/lmdbjava/DbiBuilderTest.java delete mode 100644 src/test/java/org/lmdbjava/TestDbiBuilder.java diff --git a/src/main/java/org/lmdbjava/AbstractFlagSet.java b/src/main/java/org/lmdbjava/AbstractFlagSet.java index 7ea413fb..5e62b437 100644 --- a/src/main/java/org/lmdbjava/AbstractFlagSet.java +++ b/src/main/java/org/lmdbjava/AbstractFlagSet.java @@ -1,3 +1,18 @@ +/* + * Copyright © 2016-2025 The LmdbJava Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.lmdbjava; import java.util.Collection; diff --git a/src/main/java/org/lmdbjava/CopyFlagSet.java b/src/main/java/org/lmdbjava/CopyFlagSet.java index 62c73c8d..5f7901de 100644 --- a/src/main/java/org/lmdbjava/CopyFlagSet.java +++ b/src/main/java/org/lmdbjava/CopyFlagSet.java @@ -1,3 +1,18 @@ +/* + * Copyright © 2016-2025 The LmdbJava Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.lmdbjava; import java.util.Collection; diff --git a/src/main/java/org/lmdbjava/Dbi.java b/src/main/java/org/lmdbjava/Dbi.java index 9cdc03ee..c66dc780 100644 --- a/src/main/java/org/lmdbjava/Dbi.java +++ b/src/main/java/org/lmdbjava/Dbi.java @@ -49,7 +49,7 @@ */ public final class Dbi { - private final ComparatorCallback ccb; + private final ComparatorCallback callbackComparator; private boolean cleaned; // Used for CursorIterable KeyRange testing and/or native callbacks private final Comparator comparator; @@ -82,7 +82,7 @@ public final class Dbi { if (nativeCb) { requireNonNull(comparator, "comparator cannot be null if nativeCb is set"); // LMDB will call back to this comparator for insertion/iteration order - this.ccb = + this.callbackComparator = (keyA, keyB) -> { final T compKeyA = proxy.out(proxy.allocate(), keyA); final T compKeyB = proxy.out(proxy.allocate(), keyB); @@ -91,9 +91,9 @@ public final class Dbi { proxy.deallocate(compKeyB); return result; }; - LIB.mdb_set_compare(txn.pointer(), ptr, ccb); + LIB.mdb_set_compare(txn.pointer(), ptr, callbackComparator); } else { - ccb = null; + callbackComparator = null; } } @@ -380,8 +380,7 @@ public boolean put(final Txn txn, final T key, final T val, final PutFlags... final Pointer transientKey = txn.kv().keyIn(key); final Pointer transientVal = txn.kv().valIn(val); final int mask = mask(flags); - final int rc = - LIB.mdb_put(txn.pointer(), ptr, txn.kv().pointerKey(), txn.kv().pointerVal(), mask); + final int rc = LIB.mdb_put(txn.pointer(), ptr, txn.kv().pointerKey(), txn.kv().pointerVal(), mask); if (rc == MDB_KEYEXIST) { if (isSet(mask, MDB_NOOVERWRITE)) { txn.kv().valOut(); // marked as in,out in LMDB C docs diff --git a/src/main/java/org/lmdbjava/DbiBuilder.java b/src/main/java/org/lmdbjava/DbiBuilder.java index f2f925ce..dcf34d0a 100644 --- a/src/main/java/org/lmdbjava/DbiBuilder.java +++ b/src/main/java/org/lmdbjava/DbiBuilder.java @@ -355,20 +355,23 @@ public DbiBuilderStage3 withTxn(final Txn txn) { /** * Construct and open the {@link Dbi}. *

- * If a {@link Txn} was supplied to the builder, it should be committed upon return from - * this method. + * If a {@link Txn} was supplied to the builder, it is the callers responsibility to + * commit and close the txn upon return from this method, else the created DB won't be retained. *

* * @return A newly constructed and opened {@link Dbi}. */ public Dbi open() { final DbiBuilder dbiBuilder = dbiBuilderStage2.dbiBuilder; - if (txn == null) { + if (txn != null) { + return open(txn, dbiBuilder); + } else { try (final Txn txn = getTxn(dbiBuilder)) { - return open(txn, dbiBuilder); + final Dbi dbi = open(txn, dbiBuilder); + // even RO Txns require a commit to retain Dbi in Env + txn.commit(); + return dbi; } - } else { - return open(txn, dbiBuilder); } } diff --git a/src/main/java/org/lmdbjava/DbiFlagSet.java b/src/main/java/org/lmdbjava/DbiFlagSet.java index e5c97544..28f5e4f1 100644 --- a/src/main/java/org/lmdbjava/DbiFlagSet.java +++ b/src/main/java/org/lmdbjava/DbiFlagSet.java @@ -1,3 +1,18 @@ +/* + * Copyright © 2016-2025 The LmdbJava Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.lmdbjava; import java.util.Collection; diff --git a/src/main/java/org/lmdbjava/EnvFlagSet.java b/src/main/java/org/lmdbjava/EnvFlagSet.java index 944496e6..f1bab2d0 100644 --- a/src/main/java/org/lmdbjava/EnvFlagSet.java +++ b/src/main/java/org/lmdbjava/EnvFlagSet.java @@ -1,3 +1,18 @@ +/* + * Copyright © 2016-2025 The LmdbJava Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.lmdbjava; import java.util.Collection; diff --git a/src/main/java/org/lmdbjava/FlagSet.java b/src/main/java/org/lmdbjava/FlagSet.java index 80a4c19e..89b955a0 100644 --- a/src/main/java/org/lmdbjava/FlagSet.java +++ b/src/main/java/org/lmdbjava/FlagSet.java @@ -1,3 +1,18 @@ +/* + * Copyright © 2016-2025 The LmdbJava Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.lmdbjava; import java.util.Comparator; diff --git a/src/main/java/org/lmdbjava/PutFlagSet.java b/src/main/java/org/lmdbjava/PutFlagSet.java index 1eedaf10..85de014b 100644 --- a/src/main/java/org/lmdbjava/PutFlagSet.java +++ b/src/main/java/org/lmdbjava/PutFlagSet.java @@ -1,3 +1,18 @@ +/* + * Copyright © 2016-2025 The LmdbJava Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.lmdbjava; import java.util.Collection; diff --git a/src/main/java/org/lmdbjava/ReferenceUtil.java b/src/main/java/org/lmdbjava/ReferenceUtil.java index 2b9f211e..70b3c338 100644 --- a/src/main/java/org/lmdbjava/ReferenceUtil.java +++ b/src/main/java/org/lmdbjava/ReferenceUtil.java @@ -41,6 +41,7 @@ private ReferenceUtil() {} */ public static void reachabilityFence0(final Object ref) { if (ref != null) { + //noinspection EmptySynchronizedStatement synchronized (ref) { // Empty synchronized is ok: https://stackoverflow.com/a/31933260/1151521 } diff --git a/src/main/java/org/lmdbjava/Txn.java b/src/main/java/org/lmdbjava/Txn.java index dc57fc66..99439bf7 100644 --- a/src/main/java/org/lmdbjava/Txn.java +++ b/src/main/java/org/lmdbjava/Txn.java @@ -42,15 +42,16 @@ public final class Txn implements AutoCloseable { private final Pointer ptr; private final boolean readOnly; private final Env env; + private final TxnFlagSet flags; private State state; Txn(final Env env, final Txn parent, final BufferProxy proxy, final TxnFlagSet flags) { - final TxnFlagSet flagSet = flags != null + this.flags = flags != null ? flags : TxnFlagSet.EMPTY; this.proxy = proxy; this.keyVal = proxy.keyVal(); - this.readOnly = flagSet.isSet(MDB_RDONLY_TXN); + this.readOnly = this.flags.isSet(MDB_RDONLY_TXN); if (env.isReadOnly() && !this.readOnly) { throw new EnvIsReadOnly(); } @@ -61,7 +62,7 @@ public final class Txn implements AutoCloseable { } final Pointer txnPtr = allocateDirect(RUNTIME, ADDRESS); final Pointer txnParentPtr = parent == null ? null : parent.ptr; - checkRc(LIB.mdb_txn_begin(env.pointer(), txnParentPtr, flagSet.getMask(), txnPtr)); + checkRc(LIB.mdb_txn_begin(env.pointer(), txnParentPtr, this.flags.getMask(), txnPtr)); ptr = txnPtr.getPointer(0); state = READY; diff --git a/src/main/java/org/lmdbjava/TxnFlagSet.java b/src/main/java/org/lmdbjava/TxnFlagSet.java index 6320eece..8e6310b3 100644 --- a/src/main/java/org/lmdbjava/TxnFlagSet.java +++ b/src/main/java/org/lmdbjava/TxnFlagSet.java @@ -1,3 +1,18 @@ +/* + * Copyright © 2016-2025 The LmdbJava Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.lmdbjava; import java.util.EnumSet; diff --git a/src/test/java/org/lmdbjava/CopyFlagSetTest.java b/src/test/java/org/lmdbjava/CopyFlagSetTest.java index 1ea44b7e..66e89ccb 100644 --- a/src/test/java/org/lmdbjava/CopyFlagSetTest.java +++ b/src/test/java/org/lmdbjava/CopyFlagSetTest.java @@ -1,3 +1,18 @@ +/* + * Copyright © 2016-2025 The LmdbJava Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.lmdbjava; import static org.hamcrest.CoreMatchers.is; diff --git a/src/test/java/org/lmdbjava/DbiBuilderTest.java b/src/test/java/org/lmdbjava/DbiBuilderTest.java new file mode 100644 index 00000000..da9341c6 --- /dev/null +++ b/src/test/java/org/lmdbjava/DbiBuilderTest.java @@ -0,0 +1,97 @@ +/* + * Copyright © 2016-2025 The LmdbJava Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lmdbjava; + +import static com.jakewharton.byteunits.BinaryByteUnit.MEBIBYTES; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.lmdbjava.Env.create; +import static org.lmdbjava.EnvFlags.MDB_NOSUBDIR; +import static org.lmdbjava.TestUtils.bb; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import org.hamcrest.Matchers; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class DbiBuilderTest { + + @Rule + public final TemporaryFolder tmp = new TemporaryFolder(); + private Env env; + + @After + public void after() { + env.close(); + } + + @Before + public void before() throws IOException { + System.out.println("before"); + final File path = tmp.newFile(); + env = create() + .setMapSize(MEBIBYTES.toBytes(64)) + .setMaxReaders(2) + .setMaxDbs(2) + .open(path, MDB_NOSUBDIR); + } + + @Test + public void unnamed() { + final Dbi dbi = env.buildDbi() + .withoutDbName() + .withDefaultComparator() + .withDbiFlags(DbiFlags.MDB_CREATE) + .open(); + + assertThat(env.getDbiNames().size(), Matchers.is(0)); + + assertPutAndGet(dbi); + } + + + @Test + public void named() { + final Dbi dbi = env.buildDbi() + .withDbName("foo") + .withDefaultComparator() + .withDbiFlags(DbiFlags.MDB_CREATE) + .open(); + + assertPutAndGet(dbi); + + assertThat(env.getDbiNames().size(), Matchers.is(1)); + assertThat(new String(env.getDbiNames().get(0), StandardCharsets.UTF_8), Matchers.is("foo")); + } + + private void assertPutAndGet(Dbi dbi) { + try (Txn writeTxn = env.txnWrite()) { + dbi.put(writeTxn, bb(123), bb(123_000)); + writeTxn.commit(); + } + + try (Txn readTxn = env.txnRead()) { + final ByteBuffer byteBuffer = dbi.get(readTxn, bb(123)); + final int val = byteBuffer.getInt(); + assertThat(val, Matchers.is(123_000)); + } + } +} diff --git a/src/test/java/org/lmdbjava/DbiFlagSetTest.java b/src/test/java/org/lmdbjava/DbiFlagSetTest.java index 323a2ed4..457e4718 100644 --- a/src/test/java/org/lmdbjava/DbiFlagSetTest.java +++ b/src/test/java/org/lmdbjava/DbiFlagSetTest.java @@ -1,3 +1,18 @@ +/* + * Copyright © 2016-2025 The LmdbJava Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.lmdbjava; import static org.hamcrest.CoreMatchers.is; diff --git a/src/test/java/org/lmdbjava/EnvFlagSetTest.java b/src/test/java/org/lmdbjava/EnvFlagSetTest.java index ed6a0fea..2ecce3c2 100644 --- a/src/test/java/org/lmdbjava/EnvFlagSetTest.java +++ b/src/test/java/org/lmdbjava/EnvFlagSetTest.java @@ -1,3 +1,18 @@ +/* + * Copyright © 2016-2025 The LmdbJava Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.lmdbjava; import static org.hamcrest.CoreMatchers.is; diff --git a/src/test/java/org/lmdbjava/PutFlagSetTest.java b/src/test/java/org/lmdbjava/PutFlagSetTest.java index 8cf1efe0..3e402732 100644 --- a/src/test/java/org/lmdbjava/PutFlagSetTest.java +++ b/src/test/java/org/lmdbjava/PutFlagSetTest.java @@ -1,3 +1,18 @@ +/* + * Copyright © 2016-2025 The LmdbJava Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.lmdbjava; import static org.hamcrest.CoreMatchers.is; diff --git a/src/test/java/org/lmdbjava/TestDbiBuilder.java b/src/test/java/org/lmdbjava/TestDbiBuilder.java deleted file mode 100644 index 7a1e8947..00000000 --- a/src/test/java/org/lmdbjava/TestDbiBuilder.java +++ /dev/null @@ -1,83 +0,0 @@ -package org.lmdbjava; - -import static com.jakewharton.byteunits.BinaryByteUnit.MEBIBYTES; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.lmdbjava.Env.create; -import static org.lmdbjava.EnvFlags.MDB_NOSUBDIR; -import static org.lmdbjava.TestUtils.bb; - -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import org.hamcrest.Matchers; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -public class TestDbiBuilder { - - @Rule - public final TemporaryFolder tmp = new TemporaryFolder(); - private Env env; - - @After - public void after() { - env.close(); - } - - @Before - public void before() throws IOException { - System.out.println("before"); - final File path = tmp.newFile(); - env = - create() - .setMapSize(MEBIBYTES.toBytes(64)) - .setMaxReaders(2) - .setMaxDbs(2) - .open(path, MDB_NOSUBDIR); - } - - @Test - public void unnamed() { - final Dbi dbi = env.buildDbi() - .withoutDbName() - .withDefaultComparator() - .withDbiFlags(DbiFlags.MDB_CREATE) - .open(); - - assertThat(env.getDbiNames().size(), Matchers.is(0)); - - assertPutAndGet(dbi); - } - - - @Test - public void named() { - final Dbi dbi = env.buildDbi() - .withDbName("foo") - .withDefaultComparator() - .withDbiFlags(DbiFlags.MDB_CREATE) - .open(); - - assertPutAndGet(dbi); - - assertThat(env.getDbiNames().size(), Matchers.is(1)); - assertThat(new String(env.getDbiNames().get(0), StandardCharsets.UTF_8), Matchers.is("foo")); - } - - private void assertPutAndGet(Dbi dbi) { - try (Txn writeTxn = env.txnWrite()) { - dbi.put(writeTxn, bb(123), bb(123_000)); - writeTxn.commit(); - } - - try (Txn readTxn = env.txnRead()) { - final ByteBuffer byteBuffer = dbi.get(readTxn, bb(123)); - final int val = byteBuffer.getInt(); - assertThat(val, Matchers.is(123_000)); - } - } -} diff --git a/src/test/java/org/lmdbjava/TxnFlagSetTest.java b/src/test/java/org/lmdbjava/TxnFlagSetTest.java index b526ceeb..2bb3790a 100644 --- a/src/test/java/org/lmdbjava/TxnFlagSetTest.java +++ b/src/test/java/org/lmdbjava/TxnFlagSetTest.java @@ -1,3 +1,18 @@ +/* + * Copyright © 2016-2025 The LmdbJava Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.lmdbjava; import static org.hamcrest.CoreMatchers.is; From 1b3f94df197b0889e6612418d3a43a9f496ee452 Mon Sep 17 00:00:00 2001 From: at055612 <22818309+at055612@users.noreply.github.com> Date: Tue, 28 Oct 2025 18:21:58 +0000 Subject: [PATCH 14/21] Change CursorIterableTest to use Parameterized --- .../java/org/lmdbjava/CursorIterableTest.java | 205 ++++++++++-------- src/test/java/org/lmdbjava/TestUtils.java | 1 + 2 files changed, 116 insertions(+), 90 deletions(-) diff --git a/src/test/java/org/lmdbjava/CursorIterableTest.java b/src/test/java/org/lmdbjava/CursorIterableTest.java index 79a9a34c..bf2eb9eb 100644 --- a/src/test/java/org/lmdbjava/CursorIterableTest.java +++ b/src/test/java/org/lmdbjava/CursorIterableTest.java @@ -45,6 +45,7 @@ import static org.lmdbjava.TestUtils.DB_1; import static org.lmdbjava.TestUtils.DB_2; import static org.lmdbjava.TestUtils.DB_3; +import static org.lmdbjava.TestUtils.DB_4; import static org.lmdbjava.TestUtils.POSIX_MODE; import static org.lmdbjava.TestUtils.bb; @@ -59,25 +60,83 @@ import java.util.LinkedList; import java.util.List; import java.util.NoSuchElementException; +import java.util.function.Function; import org.hamcrest.Matchers; import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import org.lmdbjava.CursorIterable.KeyVal; -/** Test {@link CursorIterable}. */ +/** + * Test {@link CursorIterable}. + */ +@RunWith(Parameterized.class) public final class CursorIterableTest { - @Rule public final TemporaryFolder tmp = new TemporaryFolder(); - private Dbi dbJavaComparator; - private Dbi dbLmdbComparator; - private Dbi dbCallbackComparator; - private List> dbs = new ArrayList<>(); + private static final DbiFlagSet dbiFlagSet = MDB_CREATE; + private static final BufferProxy bufferProxy = ByteBufferProxy.PROXY_OPTIMAL; + + @Rule + public final TemporaryFolder tmp = new TemporaryFolder(); + private Env env; private Deque list; + /** Injected by {@link #data()} with appropriate runner. */ + @Parameterized.Parameter + public DbiFactory dbiFactory; + + @Before + public void before() throws IOException { + final File path = tmp.newFile(); + final BufferProxy bufferProxy = ByteBufferProxy.PROXY_OPTIMAL; + env = + create(bufferProxy) + .setMapSize(KIBIBYTES.toBytes(256)) + .setMaxReaders(1) + .setMaxDbs(3) + .open(path, POSIX_MODE, MDB_NOSUBDIR); + + populateTestDataList(); + } + + @Parameterized.Parameters(name = "{index}: dbi: {0}") + public static Object[] data() { + final DbiFactory defaultComparator = new DbiFactory("defaultComparator", env -> + env.buildDbi() + .withDbName(DB_1) + .withDefaultComparator() + .withDbiFlags(dbiFlagSet) + .open()); + final DbiFactory nativeComparator = new DbiFactory("nativeComparator", env -> + env.buildDbi() + .withDbName(DB_2) + .withNativeComparator() + .withDbiFlags(dbiFlagSet) + .open()); + final DbiFactory callbackComparator = new DbiFactory("callbackComparator", env -> + env.buildDbi() + .withDbName(DB_3) + .withCallbackComparator(bufferProxy.getComparator(dbiFlagSet)) + .withDbiFlags(dbiFlagSet) + .open()); + final DbiFactory iteratorComparator = new DbiFactory("iteratorComparator", env -> + env.buildDbi() + .withDbName(DB_4) + .withIteratorComparator(bufferProxy.getComparator(dbiFlagSet)) + .withDbiFlags(dbiFlagSet) + .open()); + return new Object[] { + defaultComparator, + nativeComparator, + callbackComparator, + iteratorComparator}; + } + @After public void after() { env.close(); @@ -118,49 +177,8 @@ public void atMostTest() { verify(atMost(bb(6)), 2, 4, 6); } - @Before - public void before() throws IOException { - final File path = tmp.newFile(); - final BufferProxy bufferProxy = ByteBufferProxy.PROXY_OPTIMAL; - env = - create(bufferProxy) - .setMapSize(KIBIBYTES.toBytes(256)) - .setMaxReaders(1) - .setMaxDbs(3) - .open(path, POSIX_MODE, MDB_NOSUBDIR); - final DbiFlagSet dbiFlagSet = MDB_CREATE; - // Use a java comparator for start/stop keys only - dbJavaComparator = env.buildDbi() - .withDbName(DB_1) - .withDefaultComparator() - .withDbiFlags(dbiFlagSet) - .open(); - // Use LMDB comparator for start/stop keys - dbLmdbComparator = env.buildDbi() - .withDbName(DB_2) - .withNativeComparator() - .withDbiFlags(dbiFlagSet) - .open(); - // Use a java comparator for start/stop keys and as a callback comparaotr - dbCallbackComparator = env.buildDbi() - .withDbName(DB_3) - .withCallbackComparator(bufferProxy.getComparator(dbiFlagSet)) - .withDbiFlags(dbiFlagSet) - .open(); - - populateList(); - - populateDatabase(dbJavaComparator); - populateDatabase(dbLmdbComparator); - populateDatabase(dbCallbackComparator); - - dbs.add(dbJavaComparator); - dbs.add(dbLmdbComparator); - dbs.add(dbCallbackComparator); - } - - private void populateList() { + private void populateTestDataList() { list = new LinkedList<>(); list.addAll(asList(2, 3, 4, 5, 6, 7, 8, 9)); } @@ -203,14 +221,6 @@ public void closedTest() { verify(closed(bb(1), bb(7)), 2, 4, 6); } - public void closedTest1() { - verify(dbLmdbComparator, closed(bb(3), bb(7)), 4, 6); - } - - public void closedTest2() { - verify(dbJavaComparator, closed(bb(3), bb(7)), 4, 6); - } - @Test public void greaterThanBackwardTest() { verify(greaterThanBackward(bb(6)), 4, 2); @@ -226,21 +236,19 @@ public void greaterThanTest() { @Test(expected = IllegalStateException.class) public void iterableOnlyReturnedOnce() { - for (final Dbi db : dbs) { - try (Txn txn = env.txnRead(); - CursorIterable c = db.iterate(txn)) { + final Dbi db = getDb(); + try (Txn txn = env.txnRead(); + CursorIterable c = db.iterate(txn)) { c.iterator(); // ok c.iterator(); // fails } - } } @Test public void iterate() { - for (final Dbi db : dbs) { - populateList(); + final Dbi db = getDb(); try (Txn txn = env.txnRead(); - CursorIterable c = db.iterate(txn)) { + CursorIterable c = db.iterate(txn)) { int cnt = 0; for (final KeyVal kv : c) { @@ -248,18 +256,16 @@ public void iterate() { assertThat(kv.val().getInt(), is(list.pollFirst())); } } - } } @Test(expected = IllegalStateException.class) public void iteratorOnlyReturnedOnce() { - for (final Dbi db : dbs) { + final Dbi db = getDb(); try (Txn txn = env.txnRead(); - CursorIterable c = db.iterate(txn)) { + CursorIterable c = db.iterate(txn)) { c.iterator(); // ok c.iterator(); // fails } - } } @Test @@ -276,10 +282,10 @@ public void lessThanTest() { @Test(expected = NoSuchElementException.class) public void nextThrowsNoSuchElementExceptionIfNoMoreElements() { - for (final Dbi db : dbs) { - populateList(); + final Dbi db = getDb(); + populateTestDataList(); try (Txn txn = env.txnRead(); - CursorIterable c = db.iterate(txn)) { + CursorIterable c = db.iterate(txn)) { final Iterator> i = c.iterator(); while (i.hasNext()) { final KeyVal kv = i.next(); @@ -289,7 +295,6 @@ public void nextThrowsNoSuchElementExceptionIfNoMoreElements() { assertThat(i.hasNext(), is(false)); i.next(); } - } } @Test @@ -341,8 +346,8 @@ public void openTest() { @Test public void removeOddElements() { - for (final Dbi db : dbs) { - verify(db, all(), 2, 4, 6, 8); + final Dbi db = getDb(); + verify(db, all(), 2, 4, 6, 8); int idx = -1; try (Txn txn = env.txnWrite()) { try (CursorIterable ci = db.iterate(txn)) { @@ -358,12 +363,11 @@ public void removeOddElements() { txn.commit(); } verify(db, all(), 4, 8); - } } @Test(expected = Env.AlreadyClosedException.class) public void nextWithClosedEnvTest() { - for (final Dbi db : dbs) { + final Dbi db = getDb(); try (Txn txn = env.txnRead()) { try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { final Iterator> c = ci.iterator(); @@ -372,12 +376,11 @@ public void nextWithClosedEnvTest() { c.next(); } } - } } @Test(expected = Env.AlreadyClosedException.class) public void removeWithClosedEnvTest() { - for (final Dbi db : dbs) { + final Dbi db = getDb(); try (Txn txn = env.txnWrite()) { try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { final Iterator> c = ci.iterator(); @@ -389,12 +392,11 @@ public void removeWithClosedEnvTest() { c.remove(); } } - } } @Test(expected = Env.AlreadyClosedException.class) public void hasNextWithClosedEnvTest() { - for (final Dbi db : dbs) { + final Dbi db = getDb(); try (Txn txn = env.txnRead()) { try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { final Iterator> c = ci.iterator(); @@ -403,21 +405,20 @@ public void hasNextWithClosedEnvTest() { c.hasNext(); } } - } } @Test(expected = Env.AlreadyClosedException.class) public void forEachRemainingWithClosedEnvTest() { - for (final Dbi db : dbs) { + final Dbi db = getDb(); try (Txn txn = env.txnRead()) { try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { final Iterator> c = ci.iterator(); env.close(); - c.forEachRemaining(keyVal -> {}); + c.forEachRemaining(keyVal -> { + }); } } - } } // @Test @@ -468,10 +469,8 @@ public void forEachRemainingWithClosedEnvTest() { // } private void verify(final KeyRange range, final int... expected) { - // Verify using all comparator types - for (final Dbi db : dbs) { - verify(range, db, expected); - } + final Dbi db = getDb(); + verify(range, db, expected); } private void verify( @@ -479,13 +478,14 @@ private void verify( verify(range, dbi, expected); } - private void verify( - final KeyRange range, final Dbi dbi, final int... expected) { + private void verify(final KeyRange range, + final Dbi dbi, + final int... expected) { final List results = new ArrayList<>(); try (Txn txn = env.txnRead(); - CursorIterable c = dbi.iterate(txn, range)) { + CursorIterable c = dbi.iterate(txn, range)) { for (final KeyVal kv : c) { final int key = kv.key().getInt(); final int val = kv.val().getInt(); @@ -499,4 +499,29 @@ private void verify( assertThat(results.get(idx), is(expected[idx])); } } + + private Dbi getDb() { + final Dbi dbi = dbiFactory.factory.apply(env); + populateDatabase(dbi); + return dbi; + } + + + // -------------------------------------------------------------------------------- + + + private static class DbiFactory { + private final String name; + private final Function, Dbi> factory; + + private DbiFactory(String name, Function, Dbi> factory) { + this.name = name; + this.factory = factory; + } + + @Override + public String toString() { + return name; + } + } } diff --git a/src/test/java/org/lmdbjava/TestUtils.java b/src/test/java/org/lmdbjava/TestUtils.java index bc9561ed..c26c1c52 100644 --- a/src/test/java/org/lmdbjava/TestUtils.java +++ b/src/test/java/org/lmdbjava/TestUtils.java @@ -33,6 +33,7 @@ final class TestUtils { public static final String DB_1 = "test-db-1"; public static final String DB_2 = "test-db-2"; public static final String DB_3 = "test-db-3"; + public static final String DB_4 = "test-db-2"; public static final int POSIX_MODE = 0664; From c0bbe73cb1d128ebb04baf842b9ed98b941a8e7f Mon Sep 17 00:00:00 2001 From: at055612 <22818309+at055612@users.noreply.github.com> Date: Tue, 28 Oct 2025 21:13:21 +0000 Subject: [PATCH 15/21] Deprecate methods using varargs flags --- src/main/java/org/lmdbjava/Cursor.java | 181 ++++++++++++++++-- src/main/java/org/lmdbjava/Dbi.java | 99 +++++++--- src/main/java/org/lmdbjava/DbiBuilder.java | 14 +- src/main/java/org/lmdbjava/DbiFlagSet.java | 2 + src/main/java/org/lmdbjava/Env.java | 179 +++++++++-------- src/main/java/org/lmdbjava/EnvFlagSet.java | 2 + src/main/java/org/lmdbjava/FlagSet.java | 33 ++++ src/main/java/org/lmdbjava/MaskedFlag.java | 4 + src/main/java/org/lmdbjava/PutFlagSet.java | 2 + .../org/lmdbjava/CursorIterablePerfTest.java | 6 +- 10 files changed, 392 insertions(+), 130 deletions(-) diff --git a/src/main/java/org/lmdbjava/Cursor.java b/src/main/java/org/lmdbjava/Cursor.java index 6d31a3d6..c1ac7374 100644 --- a/src/main/java/org/lmdbjava/Cursor.java +++ b/src/main/java/org/lmdbjava/Cursor.java @@ -20,8 +20,6 @@ import static org.lmdbjava.Dbi.KeyNotFoundException.MDB_NOTFOUND; import static org.lmdbjava.Env.SHOULD_CHECK; import static org.lmdbjava.Library.LIB; -import static org.lmdbjava.MaskedFlag.isSet; -import static org.lmdbjava.MaskedFlag.mask; import static org.lmdbjava.PutFlags.MDB_MULTIPLE; import static org.lmdbjava.PutFlags.MDB_NODUPDATA; import static org.lmdbjava.PutFlags.MDB_NOOVERWRITE; @@ -97,23 +95,49 @@ public long count() { checkRc(LIB.mdb_cursor_count(ptrCursor, longByReference)); return longByReference.longValue(); } + /** + * @deprecated Instead use {@link Cursor#delete(PutFlagSet)}. + *
+ * Delete current key/data pair. + * + *

This function deletes the key/data pair to which the cursor refers. + * + * @param flags flags (either null or {@link PutFlags#MDB_NODUPDATA} + */ + @Deprecated + public void delete(final PutFlags... flags) { + delete(PutFlagSet.of(flags)); + } + + /** + * @deprecated Instead use {@link Cursor#delete(PutFlagSet)}. + *

+ * Delete current key/data pair. + * + *

This function deletes the key/data pair to which the cursor refers. + */ + public void delete() { + delete(PutFlagSet.EMPTY); + } /** * Delete current key/data pair. * *

This function deletes the key/data pair to which the cursor refers. * - * @param f flags (either null or {@link PutFlags#MDB_NODUPDATA} + * @param flags flags (either null or {@link PutFlags#MDB_NODUPDATA} */ - public void delete(final PutFlags... f) { + public void delete(final PutFlagSet flags) { if (SHOULD_CHECK) { env.checkNotClosed(); checkNotClosed(); txn.checkReady(); txn.checkWritesAllowed(); } - final int flags = mask(f); - checkRc(LIB.mdb_cursor_del(ptrCursor, flags)); + final PutFlagSet putFlagSet = flags != null + ? flags + : PutFlagSet.EMPTY; + checkRc(LIB.mdb_cursor_del(ptrCursor, putFlagSet.getMask())); } /** @@ -235,17 +259,49 @@ public boolean prev() { } /** + * @deprecated Use {@link Cursor#put(Object, Object, PutFlagSet)} instead. + *

* Store by cursor. * *

This function stores key/data pairs into the database. * * @param key key to store * @param val data to store - * @param op options for this operation + * @param flags options for this operation + * @return true if the value was put, false if MDB_NOOVERWRITE or MDB_NODUPDATA were set and the + * key/value existed already. + */ + @Deprecated + public boolean put(final T key, final T val, final PutFlags... flags) { + return put(key, val, PutFlagSet.of(flags)); + } + + /** + * Store by cursor. + * + *

This function stores key/data pairs into the database. + * + * @param key key to store + * @param val data to store + * @return true if the value was put, false if MDB_NOOVERWRITE or MDB_NODUPDATA were set and the + * key/value existed already. + */ + public boolean put(final T key, final T val) { + return put(key, val, PutFlagSet.EMPTY); + } + + /** + * Store by cursor. + * + *

This function stores key/data pairs into the database. + * + * @param key key to store + * @param val data to store + * @param flags options for this operation * @return true if the value was put, false if MDB_NOOVERWRITE or MDB_NODUPDATA were set and the * key/value existed already. */ - public boolean put(final T key, final T val, final PutFlags... op) { + public boolean put(final T key, final T val, final PutFlagSet flags) { if (SHOULD_CHECK) { requireNonNull(key); requireNonNull(val); @@ -256,12 +312,14 @@ public boolean put(final T key, final T val, final PutFlags... op) { } final Pointer transientKey = kv.keyIn(key); final Pointer transientVal = kv.valIn(val); - final int mask = mask(op); - final int rc = LIB.mdb_cursor_put(ptrCursor, kv.pointerKey(), kv.pointerVal(), mask); + final PutFlagSet putFlagSet = flags != null + ? flags + : PutFlagSet.EMPTY; + final int rc = LIB.mdb_cursor_put(ptrCursor, kv.pointerKey(), kv.pointerVal(), putFlagSet.getMask()); if (rc == MDB_KEYEXIST) { - if (isSet(mask, MDB_NOOVERWRITE)) { + if (putFlagSet.isSet(MDB_NOOVERWRITE)) { kv.valOut(); // marked as in,out in LMDB C docs - } else if (!isSet(mask, MDB_NODUPDATA)) { + } else if (!putFlagSet.isSet(MDB_NODUPDATA)) { checkRc(rc); } return false; @@ -274,6 +332,42 @@ public boolean put(final T key, final T val, final PutFlags... op) { return true; } + /** + * @deprecated Use {@link Cursor#put(Object, Object, PutFlagSet)} instead. + *

+ * Put multiple values into the database in one MDB_MULTIPLE operation. + * + *

The database must have been opened with {@link DbiFlags#MDB_DUPFIXED}. The buffer must + * contain fixed-sized values to be inserted. The size of each element is calculated from the + * buffer's size divided by the given element count. For example, to populate 10 X 4 byte integers + * at once, present a buffer of 40 bytes and specify the element as 10. + * + * @param key key to store in the database (not null) + * @param val value to store in the database (not null) + * @param elements number of elements contained in the passed value buffer + * @param flags options for operation (must set MDB_MULTIPLE) + */ + @Deprecated + public void putMultiple(final T key, final T val, final int elements, final PutFlags... flags) { + putMultiple(key, val, elements, PutFlagSet.of(flags)); + } + + /** + * Put multiple values into the database in one MDB_MULTIPLE operation. + * + *

The database must have been opened with {@link DbiFlags#MDB_DUPFIXED}. The buffer must + * contain fixed-sized values to be inserted. The size of each element is calculated from the + * buffer's size divided by the given element count. For example, to populate 10 X 4 byte integers + * at once, present a buffer of 40 bytes and specify the element as 10. + * + * @param key key to store in the database (not null) + * @param val value to store in the database (not null) + * @param elements number of elements contained in the passed value buffer + */ + public void putMultiple(final T key, final T val, final int elements) { + putMultiple(key, val, elements, PutFlagSet.EMPTY); + } + /** * Put multiple values into the database in one MDB_MULTIPLE operation. * @@ -285,9 +379,10 @@ public boolean put(final T key, final T val, final PutFlags... op) { * @param key key to store in the database (not null) * @param val value to store in the database (not null) * @param elements number of elements contained in the passed value buffer - * @param op options for operation (must set MDB_MULTIPLE) + * @param flags options for operation (must set MDB_MULTIPLE) + * Either a {@link PutFlagSet} or a single {@link PutFlags}. */ - public void putMultiple(final T key, final T val, final int elements, final PutFlags... op) { + public void putMultiple(final T key, final T val, final int elements, final PutFlagSet flags) { if (SHOULD_CHECK) { requireNonNull(txn); requireNonNull(key); @@ -296,13 +391,15 @@ public void putMultiple(final T key, final T val, final int elements, final PutF txn.checkReady(); txn.checkWritesAllowed(); } - final int mask = mask(op); - if (SHOULD_CHECK && !isSet(mask, MDB_MULTIPLE)) { + final PutFlagSet putFlagSet = flags != null + ? flags + : PutFlagSet.EMPTY; + if (SHOULD_CHECK && !putFlagSet.isSet(MDB_MULTIPLE)) { throw new IllegalArgumentException("Must set " + MDB_MULTIPLE + " flag"); } final Pointer transientKey = txn.kv().keyIn(key); final Pointer dataPtr = txn.kv().valInMulti(val, elements); - final int rc = LIB.mdb_cursor_put(ptrCursor, txn.kv().pointerKey(), dataPtr, mask); + final int rc = LIB.mdb_cursor_put(ptrCursor, txn.kv().pointerKey(), dataPtr, putFlagSet.getMask()); checkRc(rc); ReferenceUtil.reachabilityFence0(transientKey); ReferenceUtil.reachabilityFence0(dataPtr); @@ -334,6 +431,8 @@ public void renew(final Txn newTxn) { } /** + * @deprecated Use {@link Cursor#reserve(Object, int, PutFlagSet)} instead. + *

* Reserve space for data of the given size, but don't copy the given val. Instead, return a * pointer to the reserved space, which the caller can fill in later - before the next update * operation or the transaction ends. This saves an extra memcpy if the data is being generated @@ -344,10 +443,46 @@ public void renew(final Txn newTxn) { * * @param key key to store in the database (not null) * @param size size of the value to be stored in the database (not null) - * @param op options for this operation + * @param flags options for this operation + * @return a buffer that can be used to modify the value + */ + @Deprecated + public T reserve(final T key, final int size, final PutFlags... flags) { + return reserve(key, size, PutFlagSet.of(flags)); + } + + /** + * Reserve space for data of the given size, but don't copy the given val. Instead, return a + * pointer to the reserved space, which the caller can fill in later - before the next update + * operation or the transaction ends. This saves an extra {@code memcpy} if the data is being generated + * later. LMDB does nothing else with this memory, the caller is expected to modify all the + * space requested. + * + *

This flag must not be specified if the database was opened with MDB_DUPSORT + * + * @param key key to store in the database (not null) + * @param size size of the value to be stored in the database (not null) + * @return a buffer that can be used to modify the value + */ + public T reserve(final T key, final int size) { + return reserve(key, size, PutFlagSet.EMPTY); + } + + /** + * Reserve space for data of the given size, but don't copy the given val. Instead, return a + * pointer to the reserved space, which the caller can fill in later - before the next update + * operation or the transaction ends. This saves an extra memcpy if the data is being generated + * later. LMDB does nothing else with this memory, the caller is expected to modify all of the + * space requested. + * + *

This flag must not be specified if the database was opened with MDB_DUPSORT + * + * @param key key to store in the database (not null) + * @param size size of the value to be stored in the database (not null) + * @param flags options for this operation * @return a buffer that can be used to modify the value */ - public T reserve(final T key, final int size, final PutFlags... op) { + public T reserve(final T key, final int size, final PutFlagSet flags) { if (SHOULD_CHECK) { requireNonNull(key); env.checkNotClosed(); @@ -357,8 +492,12 @@ public T reserve(final T key, final int size, final PutFlags... op) { } final Pointer transientKey = kv.keyIn(key); final Pointer transientVal = kv.valIn(size); - final int flags = mask(op) | MDB_RESERVE.getMask(); - checkRc(LIB.mdb_cursor_put(ptrCursor, kv.pointerKey(), kv.pointerVal(), flags)); + final PutFlagSet putFlagSet = flags != null + ? flags + : PutFlagSet.EMPTY; + // This is inconsistent with putMultiple which require MDB_MULTIPLE to be in the set. + final int flagsMask = putFlagSet.getMaskWith(MDB_RESERVE); + checkRc(LIB.mdb_cursor_put(ptrCursor, kv.pointerKey(), kv.pointerVal(), flagsMask)); kv.valOut(); ReferenceUtil.reachabilityFence0(transientKey); ReferenceUtil.reachabilityFence0(transientVal); diff --git a/src/main/java/org/lmdbjava/Dbi.java b/src/main/java/org/lmdbjava/Dbi.java index c66dc780..bcb1ccfc 100644 --- a/src/main/java/org/lmdbjava/Dbi.java +++ b/src/main/java/org/lmdbjava/Dbi.java @@ -31,7 +31,7 @@ import static org.lmdbjava.PutFlags.MDB_RESERVE; import static org.lmdbjava.ResultCodeMapper.checkRc; -import java.nio.charset.StandardCharsets; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; @@ -261,6 +261,31 @@ public byte[] getName() { return name == null ? null : Arrays.copyOf(name, name.length); } + public String getNameAsString() { + return getNameAsString(Env.DEFAULT_NAME_CHARSET); + } + + + /** + * Obtains the name of this database, using the supplied {@link Charset}. + * + * @return The name of the database. If this is the unnamed database an empty + * string will be returned. + * @throws RuntimeException if the name can't be decoded. + */ + public String getNameAsString(final Charset charset) { + if (name == null) { + return ""; + } else { + // Assume a UTF8 encoding as we don't know, thus swallow if it fails + try { + return new String(name, requireNonNull(charset)); + } catch (Exception e) { + throw new RuntimeException("Unable to decode database name using charset " + charset); + } + } + } + /** * Iterate the database from the first item and forwards. * @@ -345,18 +370,22 @@ public Cursor openCursor(final Txn txn) { * * @param key key to store in the database (not null) * @param val value to store in the database (not null) - * @see #put(org.lmdbjava.Txn, java.lang.Object, java.lang.Object, org.lmdbjava.PutFlags...) + * @see #put(Txn, Object, Object, PutFlagSet) */ public void put(final T key, final T val) { try (Txn txn = env.txnWrite()) { - put(txn, key, val); + put(txn, key, val, PutFlagSet.EMPTY); txn.commit(); } } /** + * @deprecated Use {@link Dbi#put(Txn, Object, Object, PutFlagSet)} instead, with a statically + * held {@link PutFlagSet}. + *

+ *

* Store a key/value pair in the database. - * + *

*

This function stores key/data pairs in the database. The default behavior is to enter the * new key/data pair, replacing any previously existing key if duplicates are disallowed, or * adding a duplicate data item if duplicates are allowed ({@link DbiFlags#MDB_DUPSORT}). @@ -368,7 +397,40 @@ public void put(final T key, final T val) { * @return true if the value was put, false if MDB_NOOVERWRITE or MDB_NODUPDATA were set and the * key/value existed already. */ + @Deprecated public boolean put(final Txn txn, final T key, final T val, final PutFlags... flags) { + return put(txn, key, val, PutFlagSet.of(flags)); + } + + /** + * Store a key/value pair in the database. + * + * @param txn transaction handle (not null; not committed; must be R-W) + * @param key key to store in the database (not null) + * @param val value to store in the database (not null) + * @return true if the value was put, false if MDB_NOOVERWRITE or MDB_NODUPDATA were set and the + * key/value existed already. + * @see #put(Txn, Object, Object, PutFlagSet) + */ + public boolean put(final Txn txn, final T key, final T val) { + return put(txn, key, val, PutFlagSet.EMPTY); + } + + /** + * Store a key/value pair in the database. + * + *

This function stores key/data pairs in the database. The default behavior is to enter the + * new key/data pair, replacing any previously existing key if duplicates are disallowed, or + * adding a duplicate data item if duplicates are allowed ({@link DbiFlags#MDB_DUPSORT}). + * + * @param txn transaction handle (not null; not committed; must be R-W) + * @param key key to store in the database (not null) + * @param val value to store in the database (not null) + * @param flags Special options for this operation. + * @return true if the value was put, false if MDB_NOOVERWRITE or MDB_NODUPDATA were set and the + * key/value existed already. + */ + public boolean put(final Txn txn, final T key, final T val, final PutFlagSet flags) { if (SHOULD_CHECK) { requireNonNull(txn); requireNonNull(key); @@ -377,14 +439,14 @@ public boolean put(final Txn txn, final T key, final T val, final PutFlags... txn.checkReady(); txn.checkWritesAllowed(); } + final PutFlagSet flagSet = flags != null ? flags : PutFlagSet.empty(); final Pointer transientKey = txn.kv().keyIn(key); final Pointer transientVal = txn.kv().valIn(val); - final int mask = mask(flags); - final int rc = LIB.mdb_put(txn.pointer(), ptr, txn.kv().pointerKey(), txn.kv().pointerVal(), mask); + final int rc = LIB.mdb_put(txn.pointer(), ptr, txn.kv().pointerKey(), txn.kv().pointerVal(), flagSet.getMask()); if (rc == MDB_KEYEXIST) { - if (isSet(mask, MDB_NOOVERWRITE)) { + if (flagSet.isSet(MDB_NOOVERWRITE)) { txn.kv().valOut(); // marked as in,out in LMDB C docs - } else if (!isSet(mask, MDB_NODUPDATA)) { + } else if (!flagSet.isSet(MDB_NODUPDATA)) { checkRc(rc); } return false; @@ -461,23 +523,16 @@ private void clean() { cleaned = true; } - private String getNameAsString() { - if (name == null) { - return ""; - } else { - try { - // Assume a UTF8 encoding as we don't know, thus swallow if it fails - return new String(name, StandardCharsets.UTF_8); - } catch (Exception e) { - return "?"; - } - } - } - @Override public String toString() { + String name; + try { + name = getNameAsString(); + } catch (Exception e) { + name = "?"; + } return "Dbi{" + - "name='" + getNameAsString() + + "name='" + name + "', dbiFlagSet=" + dbiFlagSet + '}'; } diff --git a/src/main/java/org/lmdbjava/DbiBuilder.java b/src/main/java/org/lmdbjava/DbiBuilder.java index dcf34d0a..2b4e6ad8 100644 --- a/src/main/java/org/lmdbjava/DbiBuilder.java +++ b/src/main/java/org/lmdbjava/DbiBuilder.java @@ -28,6 +28,7 @@ */ public class DbiBuilder { + private final Env env; private final BufferProxy proxy; private final boolean readOnly; @@ -56,7 +57,7 @@ public DbiBuilderStage2 withDbName(final String name) { // Null name is allowed so no null check final byte[] nameBytes = name == null ? null - : name.getBytes(StandardCharsets.UTF_8); + : name.getBytes(Env.DEFAULT_NAME_CHARSET); return withDbName(nameBytes); } @@ -252,7 +253,7 @@ private DbiBuilderStage3(DbiBuilderStage2 dbiBuilderStage2) { * Clears all flags currently set by previous calls to * {@link DbiBuilderStage3#withDbiFlags(Collection)}, * {@link DbiBuilderStage3#withDbiFlags(DbiFlags...)} - * or {@link DbiBuilderStage3#setDbiFlag(DbiFlags)}. + * or {@link DbiBuilderStage3#addDbiFlag(DbiFlags)}. *

* * @param dbiFlags to open the database with. @@ -277,7 +278,7 @@ public DbiBuilderStage3 withDbiFlags(final Collection dbiFlags) { * Clears all flags currently set by previous calls to * {@link DbiBuilderStage3#withDbiFlags(Collection)}, * {@link DbiBuilderStage3#withDbiFlags(DbiFlags...)} - * or {@link DbiBuilderStage3#setDbiFlag(DbiFlags)}. + * or {@link DbiBuilderStage3#addDbiFlag(DbiFlags)}. *

* * @param dbiFlags to open the database with. @@ -302,7 +303,7 @@ public DbiBuilderStage3 withDbiFlags(final DbiFlags... dbiFlags) { * Clears all flags currently set by previous calls to * {@link DbiBuilderStage3#withDbiFlags(Collection)}, * {@link DbiBuilderStage3#withDbiFlags(DbiFlags...)} - * or {@link DbiBuilderStage3#setDbiFlag(DbiFlags)}. + * or {@link DbiBuilderStage3#addDbiFlag(DbiFlags)}. *

* * @param dbiFlagSet to open the database with. @@ -320,12 +321,12 @@ public DbiBuilderStage3 withDbiFlags(final DbiFlagSet dbiFlagSet) { * Adds a dbiFlag to those flags already added to this builder by * {@link DbiBuilderStage3#withDbiFlags(DbiFlags...)}, * {@link DbiBuilderStage3#withDbiFlags(Collection)} - * or {@link DbiBuilderStage3#setDbiFlag(DbiFlags)}. + * or {@link DbiBuilderStage3#addDbiFlag(DbiFlags)}. * * @param dbiFlag to open the database with. A null value is a no-op. * @return this builder instance. */ - public DbiBuilderStage3 setDbiFlag(final DbiFlags dbiFlag) { + public DbiBuilderStage3 addDbiFlag(final DbiFlags dbiFlag) { this.flagSetBuilder.setFlag(dbiFlag); return this; } @@ -409,7 +410,6 @@ private Dbi open(final Txn txn, final ComparatorType comparatorType = dbiBuilderStage2.comparatorType; final Comparator comparator = getComparator(dbiBuilder, comparatorType, dbiFlagSet); final boolean useNativeCallback = comparatorType == ComparatorType.CALLBACK; - return new Dbi<>( dbiBuilder.env, txn, diff --git a/src/main/java/org/lmdbjava/DbiFlagSet.java b/src/main/java/org/lmdbjava/DbiFlagSet.java index 28f5e4f1..5edf10c7 100644 --- a/src/main/java/org/lmdbjava/DbiFlagSet.java +++ b/src/main/java/org/lmdbjava/DbiFlagSet.java @@ -21,6 +21,8 @@ public interface DbiFlagSet extends FlagSet { + DbiFlagSet EMPTY = DbiFlagSetImpl.EMPTY; + static DbiFlagSet empty() { return DbiFlagSetImpl.EMPTY; } diff --git a/src/main/java/org/lmdbjava/Env.java b/src/main/java/org/lmdbjava/Env.java index efc4e240..55c88ec1 100644 --- a/src/main/java/org/lmdbjava/Env.java +++ b/src/main/java/org/lmdbjava/Env.java @@ -29,6 +29,8 @@ import java.io.File; import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -46,8 +48,11 @@ */ public final class Env implements AutoCloseable { - /** Java system property name that can be set to disable optional checks. */ + /** + * Java system property name that can be set to disable optional checks. + */ public static final String DISABLE_CHECKS_PROP = "lmdbjava.disable.checks"; + public static final Charset DEFAULT_NAME_CHARSET = StandardCharsets.UTF_8; /** * Indicates whether optional checks should be applied in LmdbJava. Optional checks are only @@ -88,7 +93,7 @@ public static Builder create() { /** * Create an {@link Env} using the passed {@link BufferProxy}. * - * @param buffer type + * @param buffer type * @param proxy the proxy to use (required) * @return the environment (never null) */ @@ -100,13 +105,15 @@ public static Builder create(final BufferProxy proxy) { * Opens an environment with a single default database in 0664 mode using the {@link * ByteBufferProxy#PROXY_OPTIMAL}. * - * @param path file system destination - * @param size size in megabytes + * @param path file system destination + * @param size size in megabytes * @param flags the flags for this new environment * @return env the environment (never null) */ public static Env open(final File path, final int size, final EnvFlags... flags) { - return new Builder<>(PROXY_OPTIMAL).setMapSize(size * 1_024L * 1_024L).open(path, flags); + return new Builder<>(PROXY_OPTIMAL) + .setMapSize(size * 1_024L * 1_024L) + .open(path, flags); } /** @@ -159,7 +166,7 @@ public void copy(final File path) { * transactions, because it employs a read-only transaction. See long-lived transactions under * "Caveats" in the LMDB native documentation. * - * @param path writable destination path as described above + * @param path writable destination path as described above * @param flags special options for this copy */ public void copy(final File path, final CopyFlagSet flags) { @@ -183,7 +190,7 @@ public List getDbiNames() { final List result = new ArrayList<>(); final Dbi names = openDbi((byte[]) null); try (Txn txn = txnRead(); - Cursor cursor = names.openCursor(txn)) { + Cursor cursor = names.openCursor(txn)) { if (!cursor.first()) { return Collections.emptyList(); } @@ -263,6 +270,7 @@ public boolean isReadOnly() { /** * Open (and optionally creates, if {@link DbiFlags#MDB_CREATE} is set) * a {@link Dbi} using a builder. + * * @return A new builder instance for creating/opening a {@link Dbi}. */ public DbiBuilder buildDbi() { @@ -270,13 +278,12 @@ public DbiBuilder buildDbi() { } /** + * @param name name of the database (or null if no name is required) + * @param flags to open the database with + * @return a database that is ready to use * @deprecated Instead use {@link Env#buildDbi()} * Convenience method that opens a {@link Dbi} with a UTF-8 database name and default {@link * Comparator} that is not invoked from native code. - * - * @param name name of the database (or null if no name is required) - * @param flags to open the database with - * @return a database that is ready to use */ @Deprecated() public Dbi openDbi(final String name, final DbiFlags... flags) { @@ -285,6 +292,11 @@ public Dbi openDbi(final String name, final DbiFlags... flags) { } /** + * @param name name of the database (or null if no name is required) + * @param comparator custom comparator for cursor start/stop key comparisons. If null, LMDB's + * comparator will be used. + * @param flags to open the database with + * @return a database that is ready to use * @deprecated Instead use {@link Env#buildDbi()} * Convenience method that opens a {@link Dbi} with a UTF-8 database name and associated {@link * Comparator} for use by {@link CursorIterable} when comparing start/stop keys. @@ -293,88 +305,83 @@ public Dbi openDbi(final String name, final DbiFlags... flags) { * LMDB uses for its insertion order (for the type of data that will be stored in the database), * or you fully understand the implications of them behaving differently. LMDB's comparator is * unsigned lexicographical, unless {@link DbiFlags#MDB_INTEGERKEY} is used. - * - * @param name name of the database (or null if no name is required) - * @param comparator custom comparator for cursor start/stop key comparisons. If null, LMDB's - * comparator will be used. - * @param flags to open the database with - * @return a database that is ready to use */ @Deprecated() - public Dbi openDbi( - final String name, final Comparator comparator, final DbiFlags... flags) { + public Dbi openDbi(final String name, + final Comparator comparator, + final DbiFlags... flags) { final byte[] nameBytes = name == null ? null : name.getBytes(UTF_8); return openDbi(nameBytes, comparator, false, flags); } /** + * @param name name of the database (or null if no name is required) + * @param comparator custom comparator for cursor start/stop key comparisons and optionally for + * LMDB to call back to. If null, LMDB's comparator will be used. + * @param nativeCb whether LMDB native code calls back to the Java comparator + * @param flags to open the database with + * @return a database that is ready to use * @deprecated Instead use {@link Env#buildDbi()} * Convenience method that opens a {@link Dbi} with a UTF-8 database name and associated {@link * Comparator}. The comparator will be used by {@link CursorIterable} when comparing start/stop * keys as a minimum. If nativeCb is {@code true}, this comparator will also be called by LMDB to * determine insertion/iteration order. Calling back to a java comparator may significantly impact * performance. - * - * @param name name of the database (or null if no name is required) - * @param comparator custom comparator for cursor start/stop key comparisons and optionally for - * LMDB to call back to. If null, LMDB's comparator will be used. - * @param nativeCb whether LMDB native code calls back to the Java comparator - * @param flags to open the database with - * @return a database that is ready to use */ @Deprecated() - public Dbi openDbi( - final String name, - final Comparator comparator, - final boolean nativeCb, - final DbiFlags... flags) { + public Dbi openDbi(final String name, + final Comparator comparator, + final boolean nativeCb, + final DbiFlags... flags) { final byte[] nameBytes = name == null ? null : name.getBytes(UTF_8); return openDbi(nameBytes, comparator, nativeCb, flags); } /** + * @param name name of the database (or null if no name is required) + * @param flags to open the database with + * @return a database that is ready to use * @deprecated Instead use {@link Env#buildDbi()} + *
* Convenience method that opens a {@link Dbi} with a default {@link Comparator} that is not * invoked from native code. - * - * @param name name of the database (or null if no name is required) - * @param flags to open the database with - * @return a database that is ready to use */ @Deprecated() - public Dbi openDbi(final byte[] name, final DbiFlags... flags) { + public Dbi openDbi(final byte[] name, + final DbiFlags... flags) { return openDbi(name, null, false, flags); } /** + * @param name name of the database (or null if no name is required) + * @param comparator custom comparator callback (or null to use LMDB default) + * @param flags to open the database with + * @return a database that is ready to use * @deprecated Instead use {@link Env#buildDbi()} + *
* Convenience method that opens a {@link Dbi} with an associated {@link Comparator} that is not * invoked from native code. - * - * @param name name of the database (or null if no name is required) - * @param comparator custom comparator callback (or null to use LMDB default) - * @param flags to open the database with - * @return a database that is ready to use */ @Deprecated() - public Dbi openDbi( - final byte[] name, final Comparator comparator, final DbiFlags... flags) { + public Dbi openDbi(final byte[] name, + final Comparator comparator, + final DbiFlags... flags) { return openDbi(name, comparator, false, flags); } /** + * @param name name of the database (or null if no name is required) + * @param comparator custom comparator callback (or null to use LMDB default) + * @param nativeCb whether native code calls back to the Java comparator + * @param flags to open the database with + * @return a database that is ready to use * @deprecated Instead use {@link Env#buildDbi()} + *
* Convenience method that opens a {@link Dbi} with an associated {@link Comparator} that may be * invoked from native code if specified. * *

This method will automatically commit the private transaction before returning. This ensures * the Dbi is available in the Env. - * - * @param name name of the database (or null if no name is required) - * @param comparator custom comparator callback (or null to use LMDB default) - * @param nativeCb whether native code calls back to the Java comparator - * @param flags to open the database with - * @return a database that is ready to use */ @Deprecated() public Dbi openDbi( @@ -390,6 +397,12 @@ public Dbi openDbi( } /** + * @param txn transaction to use (required; not closed) + * @param name name of the database (or null if no name is required) + * @param comparator custom comparator callback (or null to use LMDB default) + * @param nativeCb whether native LMDB code should call back to the Java comparator + * @param flags to open the database with + * @return a database that is ready to use * @deprecated Instead use {@link Env#buildDbi()} * Open the {@link Dbi} using the passed {@link Txn}. * @@ -409,13 +422,6 @@ public Dbi openDbi( * *

This method (and its overloaded convenience variants) must not be called from concurrent * threads. - * - * @param txn transaction to use (required; not closed) - * @param name name of the database (or null if no name is required) - * @param comparator custom comparator callback (or null to use LMDB default) - * @param nativeCb whether native LMDB code should call back to the Java comparator - * @param flags to open the database with - * @return a database that is ready to use */ @Deprecated() public Dbi openDbi( @@ -455,7 +461,7 @@ public Stat stat() { * Flushes the data buffers to disk. * * @param force force a synchronous flush (otherwise if the environment has the MDB_NOSYNC flag - * set the flushes will be omitted, and with MDB_MAPASYNC they will be asynchronous) + * set the flushes will be omitted, and with MDB_MAPASYNC they will be asynchronous) */ public void sync(final boolean force) { if (closed) { @@ -466,13 +472,12 @@ public void sync(final boolean force) { } /** - * @deprecated Instead use {@link Env#txn(Txn, TxnFlagSet)} - * - * Obtain a transaction with the requested parent and flags. - * * @param parent parent transaction (may be null if no parent) - * @param flags applicable flags (eg for a reusable, read-only transaction) + * @param flags applicable flags (eg for a reusable, read-only transaction) * @return a transaction (never null) + * @deprecated Instead use {@link Env#txn(Txn, TxnFlagSet)} + *

+ * Obtain a transaction with the requested parent and flags. */ @Deprecated public Txn txn(final Txn parent, final TxnFlags... flags) { @@ -495,7 +500,7 @@ public Txn txn(final Txn parent) { * Obtain a transaction with the requested parent and flags. * * @param parent parent transaction (may be null if no parent) - * @param flag applicable flag (eg for a reusable, read-only transaction) + * @param flag applicable flag (eg for a reusable, read-only transaction) * @return a transaction (never null) */ public Txn txn(final Txn parent, final TxnFlags flag) { @@ -507,9 +512,9 @@ public Txn txn(final Txn parent, final TxnFlags flag) { * Obtain a transaction with the requested parent and flags. * * @param parent parent transaction (may be null if no parent) - * @param flags applicable flags (e.g. for a reusable, read-only transaction). - * If the set of flags is used frequently it is recommended to hold - * a static instance of the {@link TxnFlagSet} for re-use. + * @param flags applicable flags (e.g. for a reusable, read-only transaction). + * If the set of flags is used frequently it is recommended to hold + * a static instance of the {@link TxnFlagSet} for re-use. * @return a transaction (never null) */ public Txn txn(final Txn parent, final TxnFlagSet flags) { @@ -581,23 +586,31 @@ public int readerCheck() { return resultPtr.intValue(); } - /** Object has already been closed and the operation is therefore prohibited. */ + /** + * Object has already been closed and the operation is therefore prohibited. + */ public static final class AlreadyClosedException extends LmdbException { private static final long serialVersionUID = 1L; - /** Creates a new instance. */ + /** + * Creates a new instance. + */ public AlreadyClosedException() { super("Environment has already been closed"); } } - /** Object has already been opened and the operation is therefore prohibited. */ + /** + * Object has already been opened and the operation is therefore prohibited. + */ public static final class AlreadyOpenException extends LmdbException { private static final long serialVersionUID = 1L; - /** Creates a new instance. */ + /** + * Creates a new instance. + */ public AlreadyOpenException() { super("Environment has already been opened"); } @@ -625,8 +638,8 @@ public static final class Builder { /** * Opens the environment. * - * @param path file system destination - * @param mode Unix permissions to set on created files and semaphores + * @param path file system destination + * @param mode Unix permissions to set on created files and semaphores * @param flags the flags for this new environment * @return an environment ready for use */ @@ -657,7 +670,7 @@ public Env open(final File path, final int mode, final EnvFlags... flags) { /** * Opens the environment with 0664 mode. * - * @param path file system destination + * @param path file system destination * @param flags the flags for this new environment * @return an environment ready for use */ @@ -711,7 +724,9 @@ public Builder setMaxReaders(final int readers) { } } - /** File is not a valid LMDB file. */ + /** + * File is not a valid LMDB file. + */ public static final class FileInvalidException extends LmdbNativeException { static final int MDB_INVALID = -30_793; @@ -722,7 +737,9 @@ public static final class FileInvalidException extends LmdbNativeException { } } - /** The specified copy destination is invalid. */ + /** + * The specified copy destination is invalid. + */ public static final class InvalidCopyDestination extends LmdbException { private static final long serialVersionUID = 1L; @@ -737,7 +754,9 @@ public InvalidCopyDestination(final String message) { } } - /** Environment mapsize reached. */ + /** + * Environment mapsize reached. + */ public static final class MapFullException extends LmdbNativeException { static final int MDB_MAP_FULL = -30_792; @@ -748,7 +767,9 @@ public static final class MapFullException extends LmdbNativeException { } } - /** Environment maxreaders reached. */ + /** + * Environment maxreaders reached. + */ public static final class ReadersFullException extends LmdbNativeException { static final int MDB_READERS_FULL = -30_790; @@ -759,7 +780,9 @@ public static final class ReadersFullException extends LmdbNativeException { } } - /** Environment version mismatch. */ + /** + * Environment version mismatch. + */ public static final class VersionMismatchException extends LmdbNativeException { static final int MDB_VERSION_MISMATCH = -30_794; diff --git a/src/main/java/org/lmdbjava/EnvFlagSet.java b/src/main/java/org/lmdbjava/EnvFlagSet.java index f1bab2d0..a3c8d1fa 100644 --- a/src/main/java/org/lmdbjava/EnvFlagSet.java +++ b/src/main/java/org/lmdbjava/EnvFlagSet.java @@ -21,6 +21,8 @@ public interface EnvFlagSet extends FlagSet { + EnvFlagSet EMPTY = EnvFlagSetImpl.EMPTY; + static EnvFlagSet empty() { return EnvFlagSetImpl.EMPTY; } diff --git a/src/main/java/org/lmdbjava/FlagSet.java b/src/main/java/org/lmdbjava/FlagSet.java index 89b955a0..668e33ba 100644 --- a/src/main/java/org/lmdbjava/FlagSet.java +++ b/src/main/java/org/lmdbjava/FlagSet.java @@ -28,24 +28,57 @@ */ public interface FlagSet extends Iterable { + /** + * @return The combined mask for this flagSet. + */ int getMask(); + /** + * @return The result of combining the mask of this {@link FlagSet} + * with the mask of the other {@link FlagSet}. + */ + default int getMaskWith(final FlagSet other) { + if (other != null) { + return MaskedFlag.mask(getMask(), other.getMask()); + } else { + return getMask(); + } + } + + /** + * @return The set of flags in this {@link FlagSet}. + */ Set getFlags(); + /** + * @return True if flag is non-null and included in this {@link FlagSet}. + */ boolean isSet(T flag); + /** + * @return The size of this {@link FlagSet} + */ default int size() { return getFlags().size(); } + /** + * @return True if this {@link FlagSet} is empty. + */ default boolean isEmpty() { return getFlags().isEmpty(); } + /** + * @return The {@link Iterator} (in no particular order) for the flags in this {@link FlagSet}. + */ default Iterator iterator() { return getFlags().iterator(); } + /** + * Convert this {@link FlagSet} to a string for use in toString methods. + */ static String asString(final FlagSet flagSet) { Objects.requireNonNull(flagSet); final String flagsStr = flagSet.getFlags() diff --git a/src/main/java/org/lmdbjava/MaskedFlag.java b/src/main/java/org/lmdbjava/MaskedFlag.java index f2f08274..271bb122 100644 --- a/src/main/java/org/lmdbjava/MaskedFlag.java +++ b/src/main/java/org/lmdbjava/MaskedFlag.java @@ -59,6 +59,10 @@ static int mask(final M... flags) { } } + static int mask(final int mask1, final int mask2) { + return mask1 | mask2; + } + static int mask(final Collection flags) { if (flags == null || flags.isEmpty()) { return EMPTY_MASK; diff --git a/src/main/java/org/lmdbjava/PutFlagSet.java b/src/main/java/org/lmdbjava/PutFlagSet.java index 85de014b..9d3a7288 100644 --- a/src/main/java/org/lmdbjava/PutFlagSet.java +++ b/src/main/java/org/lmdbjava/PutFlagSet.java @@ -21,6 +21,8 @@ public interface PutFlagSet extends FlagSet { + PutFlagSet EMPTY = PutFlagSetImpl.EMPTY; + static PutFlagSet empty() { return PutFlagSetImpl.EMPTY; } diff --git a/src/test/java/org/lmdbjava/CursorIterablePerfTest.java b/src/test/java/org/lmdbjava/CursorIterablePerfTest.java index e2c54346..c5240215 100644 --- a/src/test/java/org/lmdbjava/CursorIterablePerfTest.java +++ b/src/test/java/org/lmdbjava/CursorIterablePerfTest.java @@ -110,6 +110,8 @@ private void populateDatabases(final boolean randomOrder) { data = this.data; } + final PutFlagSet noOverwriteAndAppendFlagSet = PutFlagSet.of(MDB_NOOVERWRITE, MDB_APPEND); + for (int round = 0; round < 3; round++) { System.out.println("round: " + round + " -----------------------------------------"); @@ -122,14 +124,14 @@ private void populateDatabases(final boolean randomOrder) { } } - final String dbName = new String(db.getName(), StandardCharsets.UTF_8); + final String dbName = db.getNameAsString(StandardCharsets.UTF_8); final Instant start = Instant.now(); try (Txn txn = env.txnWrite()) { for (final Integer i : data) { if (randomOrder) { db.put(txn, bb(i), bb(i + 1), MDB_NOOVERWRITE); } else { - db.put(txn, bb(i), bb(i + 1), MDB_NOOVERWRITE, MDB_APPEND); + db.put(txn, bb(i), bb(i + 1), noOverwriteAndAppendFlagSet); } } txn.commit(); From 4fd89fff1755e1572929da3097809dfdad5fa1bb Mon Sep 17 00:00:00 2001 From: at055612 <22818309+at055612@users.noreply.github.com> Date: Wed, 29 Oct 2025 11:32:38 +0000 Subject: [PATCH 16/21] Add int key compare method to (Direct|Byte)BufferProxy --- .../java/org/lmdbjava/AbstractFlagSet.java | 27 +- src/main/java/org/lmdbjava/BufferProxy.java | 27 +- .../java/org/lmdbjava/ByteArrayProxy.java | 31 -- src/main/java/org/lmdbjava/ByteBufProxy.java | 10 - .../java/org/lmdbjava/ByteBufferProxy.java | 122 ++++---- .../java/org/lmdbjava/DirectBufferProxy.java | 60 ++-- src/main/java/org/lmdbjava/FlagSet.java | 39 ++- .../org/lmdbjava/ByteBufferProxyTest.java | 90 +++++- .../CursorIterableIntegerKeyTest.java | 293 +++++++++++------- .../java/org/lmdbjava/CursorIterableTest.java | 45 +-- .../java/org/lmdbjava/DbiBuilderTest.java | 1 - 11 files changed, 453 insertions(+), 292 deletions(-) diff --git a/src/main/java/org/lmdbjava/AbstractFlagSet.java b/src/main/java/org/lmdbjava/AbstractFlagSet.java index 5e62b437..25aa328b 100644 --- a/src/main/java/org/lmdbjava/AbstractFlagSet.java +++ b/src/main/java/org/lmdbjava/AbstractFlagSet.java @@ -64,7 +64,6 @@ public boolean isSet(final T flag) { // Probably cheaper to compare the masks than to use EnumSet.contains() return flag != null && MaskedFlag.isSet(mask, flag); - } /** @@ -93,9 +92,7 @@ public Iterator iterator() { @Override public boolean equals(Object object) { - if (this == object) return true; -// if (object == null || getClass() != object.getClass()) return false; - return FlagSet.equals(this, (FlagSet) object); + return FlagSet.equals(this, object); } @Override @@ -141,6 +138,15 @@ public boolean isSet(final T flag) { return this.flag == flag; } + @Override + public boolean areAnySet(FlagSet flags) { + if (flags == null) { + return false; + } else { + return flags.isSet(this.flag); + } + } + @Override public int size() { return 1; @@ -167,9 +173,7 @@ public String toString() { @Override public boolean equals(Object object) { - if (this == object) return true; -// if (object == null || getClass() != object.getClass()) return false; - return FlagSet.equals(this, (FlagSet) object); + return FlagSet.equals(this, object); } @Override @@ -205,6 +209,11 @@ public boolean isSet(final T flag) { return false; } + @Override + public boolean areAnySet(final FlagSet flags) { + return false; + } + @Override public int size() { return 0; @@ -227,9 +236,7 @@ public String toString() { @Override public boolean equals(Object object) { - if (this == object) return true; -// if (object == null || getClass() != object.getClass()) return false; - return FlagSet.equals(this, (FlagSet) object); + return FlagSet.equals(this, object); } @Override diff --git a/src/main/java/org/lmdbjava/BufferProxy.java b/src/main/java/org/lmdbjava/BufferProxy.java index 60272209..af0c7f06 100644 --- a/src/main/java/org/lmdbjava/BufferProxy.java +++ b/src/main/java/org/lmdbjava/BufferProxy.java @@ -40,6 +40,11 @@ public abstract class BufferProxy { /** Offset from a pointer of the MDB_val.mv_size field. */ protected static final int STRUCT_FIELD_OFFSET_SIZE = 0; + /** The set of {@link DbiFlags} that indicate unsigned integer keys are being used. */ + protected static final DbiFlagSet INTEGER_KEY_FLAGS = DbiFlagSet.of( + DbiFlags.MDB_INTEGERKEY, + DbiFlags.MDB_INTEGERDUP); + /** Explicitly-defined default constructor to avoid warnings. */ protected BufferProxy() {} @@ -88,28 +93,6 @@ public Comparator getComparator() { return getComparator(DbiFlagSet.empty()); } -// /** -// * Get a suitable default {@link Comparator} to compare numeric key values as signed. -// * -// *

Note: LMDB's default comparator is unsigned so if this is used only for the {@link -// * CursorIterable} start/stop key comparisons then its behaviour will differ from the iteration -// * order. Use with caution. -// * -// * @return a comparator that can be used (never null) -// */ -// public abstract Comparator getSignedComparator(); -// -// /** -// * Get a suitable default {@link Comparator} to compare numeric key values as unsigned. -// *

-// * This should match the behaviour of the LMDB's mdb_cmp comparator as it may be used for -// * {@link CursorIterable} start/stop keys comparisons, which must match LMDB's insertion order. -// *

-// * -// * @return a comparator that can be used (never null) -// */ -// public abstract Comparator getUnsignedComparator(final DbiFlagSet dbiFlagSet); - /** * Called when the MDB_val should be set to reflect the passed buffer. This buffer * will have been created by end users, not {@link #allocate()}. diff --git a/src/main/java/org/lmdbjava/ByteArrayProxy.java b/src/main/java/org/lmdbjava/ByteArrayProxy.java index 5231ed51..d7c23919 100644 --- a/src/main/java/org/lmdbjava/ByteArrayProxy.java +++ b/src/main/java/org/lmdbjava/ByteArrayProxy.java @@ -36,7 +36,6 @@ public final class ByteArrayProxy extends BufferProxy { private static final MemoryManager MEM_MGR = RUNTIME.getMemoryManager(); - private static final Comparator signedComparator = ByteArrayProxy::compareArraysSigned; private static final Comparator unsignedComparator = ByteArrayProxy::compareArrays; private ByteArrayProxy() {} @@ -68,26 +67,6 @@ public static int compareArrays(final byte[] o1, final byte[] o2) { return o1.length - o2.length; } - /** - * Compare two byte arrays. - * - * @param b1 left operand (required) - * @param b2 right operand (required) - * @return as specified by {@link Comparable} interface - */ - public static int compareArraysSigned(final byte[] b1, final byte[] b2) { - requireNonNull(b1); - requireNonNull(b2); - - if (b1 == b2) return 0; - - for (int i = 0; i < min(b1.length, b2.length); ++i) { - if (b1[i] != b2[i]) return b1[i] - b2[i]; - } - - return b1.length - b2.length; - } - @Override protected byte[] allocate() { return new byte[0]; @@ -108,16 +87,6 @@ public Comparator getComparator(final DbiFlagSet dbiFlagSet) { return unsignedComparator; } - // @Override -// public Comparator getSignedComparator() { -// return signedComparator; -// } -// -// @Override -// public Comparator getUnsignedComparator() { -// return unsignedComparator; -// } - @Override protected Pointer in(final byte[] buffer, final Pointer ptr) { final Pointer pointer = MEM_MGR.allocateDirect(buffer.length); diff --git a/src/main/java/org/lmdbjava/ByteBufProxy.java b/src/main/java/org/lmdbjava/ByteBufProxy.java index fc14b58f..319256fb 100644 --- a/src/main/java/org/lmdbjava/ByteBufProxy.java +++ b/src/main/java/org/lmdbjava/ByteBufProxy.java @@ -118,16 +118,6 @@ public Comparator getComparator(final DbiFlagSet dbiFlagSet) { return comparator; } - // @Override -// public Comparator getSignedComparator() { -// return comparator; -// } -// -// @Override -// public Comparator getUnsignedComparator() { -// return comparator; -// } - @Override protected void deallocate(final ByteBuf buff) { buff.release(); diff --git a/src/main/java/org/lmdbjava/ByteBufferProxy.java b/src/main/java/org/lmdbjava/ByteBufferProxy.java index 4875572b..ca4deba3 100644 --- a/src/main/java/org/lmdbjava/ByteBufferProxy.java +++ b/src/main/java/org/lmdbjava/ByteBufferProxy.java @@ -54,15 +54,19 @@ public final class ByteBufferProxy { */ public static final BufferProxy PROXY_OPTIMAL; - /** The safe, reflective {@link ByteBuffer} proxy for this system. Guaranteed to never be null. */ + /** + * The safe, reflective {@link ByteBuffer} proxy for this system. Guaranteed to never be null. + */ public static final BufferProxy PROXY_SAFE; + static { PROXY_SAFE = new ReflectiveProxy(); PROXY_OPTIMAL = getProxyOptimal(); } - private ByteBufferProxy() {} + private ByteBufferProxy() { + } private static BufferProxy getProxyOptimal() { try { @@ -72,17 +76,25 @@ private static BufferProxy getProxyOptimal() { } } - /** The buffer must be a direct buffer (not heap allocated). */ + /** + * The buffer must be a direct buffer (not heap allocated). + */ public static final class BufferMustBeDirectException extends LmdbException { private static final long serialVersionUID = 1L; - /** Creates a new instance. */ + /** + * Creates a new instance. + */ public BufferMustBeDirectException() { super("The buffer must be a direct buffer (not heap allocated"); } } + + // -------------------------------------------------------------------------------- + + /** * Provides {@link ByteBuffer} pooling and address resolution for concrete {@link BufferProxy} * implementations. @@ -92,16 +104,6 @@ abstract static class AbstractByteBufferProxy extends BufferProxy { protected static final String FIELD_NAME_ADDRESS = "address"; protected static final String FIELD_NAME_CAPACITY = "capacity"; - private static final Comparator signedComparator = - (o1, o2) -> { - requireNonNull(o1); - requireNonNull(o2); - - return o1.compareTo(o2); - }; - private static final Comparator unsignedComparator = - AbstractByteBufferProxy::compareBuff; - /** * A thread-safe pool for a given length. If the buffer found is valid (ie not of a negative * length) then that buffer is used. If no valid buffer is found, a new buffer is created. @@ -116,7 +118,7 @@ abstract static class AbstractByteBufferProxy extends BufferProxy { * @param o2 right operand (required) * @return as specified by {@link Comparable} interface */ - public static int compareBuff(final ByteBuffer o1, final ByteBuffer o2) { + public static int compareLexicographically(final ByteBuffer o1, final ByteBuffer o2) { requireNonNull(o1); requireNonNull(o2); if (o1.equals(o2)) { @@ -148,34 +150,42 @@ public static int compareBuff(final ByteBuffer o1, final ByteBuffer o2) { return o1.remaining() - o2.remaining(); } -// /** -// * Possible compareBuff method specifically for 4/8 byte keys when using MDB_INTEGER_KEY -// */ -// public static int compareBuff(final ByteBuffer o1, final ByteBuffer o2) { -// requireNonNull(o1); -// requireNonNull(o2); -// // Both buffers should be same len -// final int len1 = o1.limit(); -// final int len2 = o2.limit(); -// if (len1 != len2) { -// throw new RuntimeException("Length mismatch, len1: " + len1 + ", len2: " + len2 -// + ". Lengths must be identical and either 4 or 8 bytes."); -// } -// final boolean reverse1 = o1.order() == LITTLE_ENDIAN; -// final boolean reverse2 = o2.order() == LITTLE_ENDIAN; -// if (len1 == 8) { -// final long lw = reverse1 ? Long.reverseBytes(o1.getLong()) : o1.getLong(); -// final long rw = reverse2 ? Long.reverseBytes(o2.getLong()) : o2.getLong(); -// return Long.compareUnsigned(lw, rw); -// } else if (len1 == 4) { -// final int lw = reverse1 ? Integer.reverseBytes(o1.getInt()) : o1.getInt(); -// final int rw = reverse2 ? Integer.reverseBytes(o2.getInt()) : o2.getInt(); -// return Integer.compareUnsigned(lw, rw); -// } else { -// throw new RuntimeException("Unexpected length len1: " + len1 + ", len2: " + len2 -// + ". Lengths must be identical and either 4 or 8 bytes."); -// } -// } + /** + * Buffer comparator specifically for 4/8 byte keys that are unsigned ints/longs, + * i.e. when using MDB_INTEGER_KEY/MDB_INTEGERDUP. Compares the buffers numerically. + *

+ * Both buffer must have 4 or 8 bytes remaining + *

+ * + * @param o1 left operand (required) + * @param o2 right operand (required) + * @return as specified by {@link Comparable} interface + */ + public static int compareAsIntegerKeys(final ByteBuffer o1, final ByteBuffer o2) { + requireNonNull(o1); + requireNonNull(o2); + // Both buffers should be same len + final int len1 = o1.limit(); + final int len2 = o2.limit(); + if (len1 != len2) { + throw new RuntimeException("Length mismatch, len1: " + len1 + ", len2: " + len2 + + ". Lengths must be identical and either 4 or 8 bytes."); + } + final boolean reverse1 = o1.order() == LITTLE_ENDIAN; + final boolean reverse2 = o2.order() == LITTLE_ENDIAN; + if (len1 == 8) { + final long lw = reverse1 ? Long.reverseBytes(o1.getLong(0)) : o1.getLong(0); + final long rw = reverse2 ? Long.reverseBytes(o2.getLong(0)) : o2.getLong(0); + return Long.compareUnsigned(lw, rw); + } else if (len1 == 4) { + final int lw = reverse1 ? Integer.reverseBytes(o1.getInt(0)) : o1.getInt(0); + final int rw = reverse2 ? Integer.reverseBytes(o2.getInt(0)) : o2.getInt(0); + return Integer.compareUnsigned(lw, rw); + } else { + throw new RuntimeException("Unexpected length1: " + len1 + + ". Lengths must be identical and either 4 or 8 bytes."); + } + } static Field findField(final Class c, final String name) { Class clazz = c; @@ -211,20 +221,14 @@ protected final ByteBuffer allocate() { } @Override - public Comparator getComparator(DbiFlagSet dbiFlagSet) { - return unsignedComparator; + public Comparator getComparator(final DbiFlagSet dbiFlagSet) { + if (dbiFlagSet.areAnySet(INTEGER_KEY_FLAGS)) { + return AbstractByteBufferProxy::compareAsIntegerKeys; + } else { + return AbstractByteBufferProxy::compareLexicographically; + } } - // @Override -// public Comparator getSignedComparator() { -// return signedComparator; -// } -// -// @Override -// public Comparator getUnsignedComparator(final DbiFlagSet dbiFlagSet) { -// return unsignedComparator; -// } - @Override protected final void deallocate(final ByteBuffer buff) { buff.order(BIG_ENDIAN); @@ -240,6 +244,10 @@ protected byte[] getBytes(final ByteBuffer buffer) { } } + + // -------------------------------------------------------------------------------- + + /** * A proxy that uses Java reflection to modify byte buffer fields, and official JNR-FFF methods to * manipulate native pointers. @@ -284,6 +292,10 @@ protected ByteBuffer out(final ByteBuffer buffer, final Pointer ptr) { } } + + // -------------------------------------------------------------------------------- + + /** * A proxy that uses Java's "unsafe" class to directly manipulate byte buffer fields and JNR-FFF * allocated memory pointers. diff --git a/src/main/java/org/lmdbjava/DirectBufferProxy.java b/src/main/java/org/lmdbjava/DirectBufferProxy.java index 514c04ab..9c90d98b 100644 --- a/src/main/java/org/lmdbjava/DirectBufferProxy.java +++ b/src/main/java/org/lmdbjava/DirectBufferProxy.java @@ -35,14 +35,6 @@ *

This class requires {@link UnsafeAccess} and Agrona must be in the classpath. */ public final class DirectBufferProxy extends BufferProxy { - private static final Comparator signedComparator = - (o1, o2) -> { - requireNonNull(o1); - requireNonNull(o2); - - return o1.compareTo(o2); - }; - private static final Comparator unsignedComparator = DirectBufferProxy::compareBuff; /** * The {@link MutableDirectBuffer} proxy. Guaranteed to never be null, although a class @@ -67,7 +59,7 @@ private DirectBufferProxy() {} * @param o2 right operand (required) * @return as specified by {@link Comparable} interface */ - public static int compareBuff(final DirectBuffer o1, final DirectBuffer o2) { + public static int compareLexicographically(final DirectBuffer o1, final DirectBuffer o2) { requireNonNull(o1); requireNonNull(o2); if (o1.equals(o2)) { @@ -97,6 +89,40 @@ public static int compareBuff(final DirectBuffer o1, final DirectBuffer o2) { return o1.capacity() - o2.capacity(); } + /** + * Buffer comparator specifically for 4/8 byte keys that are unsigned ints/longs, + * i.e. when using MDB_INTEGER_KEY/MDB_INTEGERDUP. Compares the buffers numerically. + *

+ * Both buffer must have 4 or 8 bytes remaining + *

+ * @param o1 left operand (required) + * @param o2 right operand (required) + * @return as specified by {@link Comparable} interface + */ + public static int compareAsIntegerKeys(final DirectBuffer o1, final DirectBuffer o2) { + requireNonNull(o1); + requireNonNull(o2); + // Both buffers should be same len + final int len1 = o1.capacity(); + final int len2 = o2.capacity(); + if (len1 != len2) { + throw new RuntimeException("Length mismatch, len1: " + len1 + ", len2: " + len2 + + ". Lengths must be identical and either 4 or 8 bytes."); + } + if (len1 == 8) { + final long lw = o1.getLong(0, BIG_ENDIAN); + final long rw = o2.getLong(0, BIG_ENDIAN); + return Long.compareUnsigned(lw, rw); + } else if (len1 == 4) { + final int lw = o1.getInt(0, BIG_ENDIAN); + final int rw = o2.getInt(0, BIG_ENDIAN); + return Integer.compareUnsigned(lw, rw); + } else { + throw new RuntimeException("Unexpected length len1: " + len1 + ", len2: " + len2 + + ". Lengths must be identical and either 4 or 8 bytes."); + } + } + @Override protected DirectBuffer allocate() { final ArrayDeque q = BUFFERS.get(); @@ -112,19 +138,13 @@ protected DirectBuffer allocate() { @Override public Comparator getComparator(final DbiFlagSet dbiFlagSet) { - return unsignedComparator; + if (dbiFlagSet.areAnySet(INTEGER_KEY_FLAGS)) { + return DirectBufferProxy::compareAsIntegerKeys; + } else { + return DirectBufferProxy::compareLexicographically; + } } - // @Override -// public Comparator getSignedComparator() { -// return signedComparator; -// } -// -// @Override -// public Comparator getUnsignedComparator(final DbiFlagSet dbiFlagSet) { -// return unsignedComparator; -// } - @Override protected void deallocate(final DirectBuffer buff) { final ArrayDeque q = BUFFERS.get(); diff --git a/src/main/java/org/lmdbjava/FlagSet.java b/src/main/java/org/lmdbjava/FlagSet.java index 668e33ba..27513fcd 100644 --- a/src/main/java/org/lmdbjava/FlagSet.java +++ b/src/main/java/org/lmdbjava/FlagSet.java @@ -55,6 +55,22 @@ default int getMaskWith(final FlagSet other) { */ boolean isSet(T flag); + /** + * @return True if at least one of flags are included in thie {@link FlagSet} + */ + default boolean areAnySet(final FlagSet flags) { + if (flags == null) { + return false; + } else { + for (final T flag : flags) { + if (isSet(flag)) { + return true; + } + } + } + return false; + } + /** * @return The size of this {@link FlagSet} */ @@ -92,17 +108,20 @@ static String asString(final FlagSet flagSet) { '}'; } - static boolean equals(final FlagSet flagSet1, - final FlagSet flagSet2) { - if (flagSet1 == flagSet2) { - return true; - } else if (flagSet1 != null && flagSet2 == null) { - return false; - } else if (flagSet1 == null) { - return false; + static boolean equals(final FlagSet flagSet, + final Object other) { + if (other instanceof FlagSet) { + final FlagSet flagSet2 = (FlagSet) other; + if (flagSet == flagSet2) { + return true; + } else if (flagSet == null) { + return false; + } else { + return flagSet.getMask() == flagSet2.getMask() + && Objects.equals(flagSet.getFlags(), flagSet2.getFlags()); + } } else { - return flagSet1.getMask() == flagSet2.getMask() - && Objects.equals(flagSet1.getFlags(), flagSet2.getFlags()); + return false; } } diff --git a/src/test/java/org/lmdbjava/ByteBufferProxyTest.java b/src/test/java/org/lmdbjava/ByteBufferProxyTest.java index b68f39ef..1372b74a 100644 --- a/src/test/java/org/lmdbjava/ByteBufferProxyTest.java +++ b/src/test/java/org/lmdbjava/ByteBufferProxyTest.java @@ -40,20 +40,31 @@ import java.io.IOException; import java.lang.reflect.Field; import java.nio.ByteBuffer; +import java.time.Duration; +import java.time.Instant; +import java.util.Comparator; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Random; +import java.util.Set; import jnr.ffi.Pointer; import jnr.ffi.provider.MemoryManager; +import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.lmdbjava.ByteBufferProxy.BufferMustBeDirectException; import org.lmdbjava.Env.ReadersFullException; -/** Test {@link ByteBufferProxy}. */ +/** + * Test {@link ByteBufferProxy}. + */ public final class ByteBufferProxyTest { static final MemoryManager MEM_MGR = RUNTIME.getMemoryManager(); - @Rule public final TemporaryFolder tmp = new TemporaryFolder(); + @Rule + public final TemporaryFolder tmp = new TemporaryFolder(); @Test(expected = BufferMustBeDirectException.class) public void buffersMustBeDirect() throws IOException { @@ -129,6 +140,81 @@ public void unsafeIsDefault() { assertThat(v.getClass().getSimpleName(), startsWith("Unsafe")); } + /** + * For 100 rounds of 1,000,000 comparisons + * compareAsIntegerKeys: PT0.267813487S + * compareLexicographically: PT0.644165235S + */ + @Test + public void comparatorPerformance() { + final Random random = new Random(); + final ByteBuffer buffer1 = ByteBuffer.allocateDirect(Long.BYTES); + final ByteBuffer buffer2 = ByteBuffer.allocateDirect(Long.BYTES); + buffer1.limit(Long.BYTES); + buffer2.limit(Long.BYTES); + final long[] values = random.longs(1_000_000).toArray(); + + Instant time = Instant.now(); + int x = 0; + for (int rounds = 0; rounds < 100; rounds++) { + for (int i = 1; i < values.length; i++) { + buffer1.putLong(0, values[i - 1]); + buffer2.putLong(0, values[i]); + final int result = ByteBufferProxy.AbstractByteBufferProxy.compareAsIntegerKeys(buffer1, buffer2); + x += result; + } + } + System.out.println("compareAsIntegerKeys: " + Duration.between(time, Instant.now())); + + time = Instant.now(); + x = 0; + for (int rounds = 0; rounds < 100; rounds++) { + for (int i = 1; i < values.length; i++) { + buffer1.putLong(0, values[i - 1]); + buffer2.putLong(0, values[i]); + final int result = ByteBufferProxy.AbstractByteBufferProxy.compareLexicographically(buffer1, buffer2); + x += result; + } + } + System.out.println("compareLexicographically: " + Duration.between(time, Instant.now())); + } + + @Test + public void verifyComparators() { + final Random random = new Random(203948); + final ByteBuffer buffer1 = ByteBuffer.allocateDirect(Long.BYTES); + final ByteBuffer buffer2 = ByteBuffer.allocateDirect(Long.BYTES); + buffer1.limit(Long.BYTES); + buffer2.limit(Long.BYTES); + final long[] values = random.longs(10_000_000).toArray(); + + final LinkedHashMap> comparators = new LinkedHashMap<>(); + comparators.put("compareAsIntegerKeys", ByteBufferProxy.AbstractByteBufferProxy::compareAsIntegerKeys); + comparators.put("compareLexicographically", ByteBufferProxy.AbstractByteBufferProxy::compareLexicographically); + + final LinkedHashMap results = new LinkedHashMap<>(comparators.size()); + final Set uniqueResults = new HashSet<>(comparators.size()); + + for (int i = 1; i < values.length; i++) { + final long val1 = values[i - 1]; + final long val2 = values[i]; + buffer1.putLong(0, val1); + buffer2.putLong(0, val2); + uniqueResults.clear(); + + // Make sure all comparators give the same result for the same inputs + comparators.forEach((name, comparator) -> { + final int result = comparator.compare(buffer1, buffer2); + results.put(name, result); + uniqueResults.add(result); + }); + + if (uniqueResults.size() != 1) { + Assert.fail("Comparator mismatch for values: " + val1 + " and " + val2 + ". Results: " + results); + } + } + } + private void checkInOut(final BufferProxy v) { // allocate a buffer larger than max key size final ByteBuffer b = allocateDirect(1_000); diff --git a/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java b/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java index aefe9d43..85c0a567 100644 --- a/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java +++ b/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java @@ -46,6 +46,7 @@ import static org.lmdbjava.TestUtils.DB_1; import static org.lmdbjava.TestUtils.DB_2; import static org.lmdbjava.TestUtils.DB_3; +import static org.lmdbjava.TestUtils.DB_4; import static org.lmdbjava.TestUtils.POSIX_MODE; import static org.lmdbjava.TestUtils.bb; import static org.lmdbjava.TestUtils.bbNative; @@ -63,26 +64,128 @@ import java.util.LinkedList; import java.util.List; import java.util.NoSuchElementException; +import java.util.function.Function; import org.hamcrest.Matchers; import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import org.lmdbjava.CursorIterable.KeyVal; -/** Test {@link CursorIterable} using {@link DbiFlags#MDB_INTEGERKEY} to ensure that - * comparators work with native order integer keys. */ +/** + * Test {@link CursorIterable} using {@link DbiFlags#MDB_INTEGERKEY} to ensure that + * comparators work with native order integer keys. + */ +@RunWith(Parameterized.class) public final class CursorIterableIntegerKeyTest { - @Rule public final TemporaryFolder tmp = new TemporaryFolder(); - private Dbi dbJavaComparator; - private Dbi dbLmdbComparator; - private Dbi dbCallbackComparator; - private List> dbs = new ArrayList<>(); + private static final DbiFlagSet DBI_FLAGS = DbiFlagSet.of(MDB_CREATE, MDB_INTEGERKEY); + private static final BufferProxy BUFFER_PROXY = ByteBufferProxy.PROXY_OPTIMAL; + + @Rule + public final TemporaryFolder tmp = new TemporaryFolder(); + private Env env; private Deque list; + /** + * Injected by {@link #data()} with appropriate runner. + */ + @SuppressWarnings("ClassEscapesDefinedScope") + @Parameterized.Parameter + public DbiFactory dbiFactory; + + @Parameterized.Parameters(name = "{index}: dbi: {0}") + public static Object[] data() { + final DbiFactory defaultComparator = new DbiFactory("defaultComparator", env -> + env.buildDbi() + .withDbName(DB_1) + .withDefaultComparator() + .withDbiFlags(DBI_FLAGS) + .open()); + final DbiFactory nativeComparator = new DbiFactory("nativeComparator", env -> + env.buildDbi() + .withDbName(DB_2) + .withNativeComparator() + .withDbiFlags(DBI_FLAGS) + .open()); + final DbiFactory callbackComparator = new DbiFactory("callbackComparator", env -> + env.buildDbi() + .withDbName(DB_3) + .withCallbackComparator(BUFFER_PROXY.getComparator(DBI_FLAGS)) + .withDbiFlags(DBI_FLAGS) + .open()); + final DbiFactory iteratorComparator = new DbiFactory("iteratorComparator", env -> + env.buildDbi() + .withDbName(DB_4) + .withIteratorComparator(BUFFER_PROXY.getComparator(DBI_FLAGS)) + .withDbiFlags(DBI_FLAGS) + .open()); + return new Object[]{ + defaultComparator, + nativeComparator, + callbackComparator, + iteratorComparator}; + } + + @Before + public void before() throws IOException { + final File path = tmp.newFile(); + final BufferProxy bufferProxy = ByteBufferProxy.PROXY_OPTIMAL; + env = + create(bufferProxy) + .setMapSize(KIBIBYTES.toBytes(256)) + .setMaxReaders(1) + .setMaxDbs(3) + .open(path, POSIX_MODE, MDB_NOSUBDIR); + + populateTestDataList(); +// final File path = tmp.newFile(); +// final BufferProxy bufferProxy = ByteBufferProxy.PROXY_OPTIMAL; +// env = +// create(bufferProxy) +// .setMapSize(KIBIBYTES.toBytes(256)) +// .setMaxReaders(1) +// .setMaxDbs(3) +// .open(path, POSIX_MODE, MDB_NOSUBDIR); +// +// // Use a java comparator for start/stop keys only +// DbiFlagSet dbiFlagSet = DbiFlagSet.of(MDB_CREATE, MDB_INTEGERKEY); +// +// dbJavaComparator = env.buildDbi() +// .withDbName(DB_1) +// .withIteratorComparator(bufferProxy.getComparator(dbiFlagSet)) +// .withDbiFlags(dbiFlagSet) +// .open(); +// +// // Use LMDB comparator for start/stop keys +// dbLmdbComparator = env.buildDbi() +// .withDbName(DB_2) +// .withDefaultComparator() +// .withDbiFlags(dbiFlagSet) +// .open(); +// +// // Use a java comparator for start/stop keys and as a callback comparaotr +// dbCallbackComparator = env.buildDbi() +// .withDbName(DB_3) +// .withCallbackComparator(bufferProxy.getComparator(dbiFlagSet)) +// .withDbiFlags(dbiFlagSet) +// .open(); +// +// populateTestDataList(); +// +// populateDatabase(dbJavaComparator); +// populateDatabase(dbLmdbComparator); +// populateDatabase(dbCallbackComparator); +// +// dbs.add(dbJavaComparator); +// dbs.add(dbLmdbComparator); +// dbs.add(dbCallbackComparator); + } + @After public void after() { env.close(); @@ -123,52 +226,8 @@ public void atMostTest() { verify(atMost(bbNative(6)), 2, 4, 6); } - @Before - public void before() throws IOException { - final File path = tmp.newFile(); - final BufferProxy bufferProxy = ByteBufferProxy.PROXY_OPTIMAL; - env = - create(bufferProxy) - .setMapSize(KIBIBYTES.toBytes(256)) - .setMaxReaders(1) - .setMaxDbs(3) - .open(path, POSIX_MODE, MDB_NOSUBDIR); - - // Use a java comparator for start/stop keys only - DbiFlagSet dbiFlagSet = DbiFlagSet.of(MDB_CREATE, MDB_INTEGERKEY); - - dbJavaComparator = env.buildDbi() - .withDbName(DB_1) - .withIteratorComparator(bufferProxy.getComparator(dbiFlagSet)) - .withDbiFlags(dbiFlagSet) - .open(); - - // Use LMDB comparator for start/stop keys - dbLmdbComparator = env.buildDbi() - .withDbName(DB_2) - .withDefaultComparator() - .withDbiFlags(dbiFlagSet) - .open(); - // Use a java comparator for start/stop keys and as a callback comparaotr - dbCallbackComparator = env.buildDbi() - .withDbName(DB_3) - .withCallbackComparator(bufferProxy.getComparator(dbiFlagSet)) - .withDbiFlags(dbiFlagSet) - .open(); - - populateList(); - - populateDatabase(dbJavaComparator); - populateDatabase(dbLmdbComparator); - populateDatabase(dbCallbackComparator); - - dbs.add(dbJavaComparator); - dbs.add(dbLmdbComparator); - dbs.add(dbCallbackComparator); - } - - private void populateList() { + private void populateTestDataList() { list = new LinkedList<>(); list.addAll(asList(2, 3, 4, 5, 6, 7, 8, 9)); } @@ -226,40 +285,37 @@ public void greaterThanTest() { @Test(expected = IllegalStateException.class) public void iterableOnlyReturnedOnce() { - for (final Dbi db : dbs) { - try (Txn txn = env.txnRead(); - CursorIterable c = db.iterate(txn)) { - c.iterator(); // ok - c.iterator(); // fails - } + final Dbi db = getDb(); + try (Txn txn = env.txnRead(); + CursorIterable c = db.iterate(txn)) { + c.iterator(); // ok + c.iterator(); // fails } } @Test public void iterate() { - for (final Dbi db : dbs) { - populateList(); - try (Txn txn = env.txnRead(); - CursorIterable c = db.iterate(txn)) { + populateTestDataList(); + final Dbi db = getDb(); + try (Txn txn = env.txnRead(); + CursorIterable c = db.iterate(txn)) { - int cnt = 0; - for (final KeyVal kv : c) { - assertThat(getNativeInt(kv.key()), is(list.pollFirst())); - assertThat(kv.val().getInt(), is(list.pollFirst())); - } + int cnt = 0; + for (final KeyVal kv : c) { + assertThat(getNativeInt(kv.key()), is(list.pollFirst())); + assertThat(kv.val().getInt(), is(list.pollFirst())); } } } @Test(expected = IllegalStateException.class) public void iteratorOnlyReturnedOnce() { - for (final Dbi db : dbs) { + final Dbi db = getDb(); try (Txn txn = env.txnRead(); - CursorIterable c = db.iterate(txn)) { + CursorIterable c = db.iterate(txn)) { c.iterator(); // ok c.iterator(); // fails } - } } @Test @@ -276,19 +332,18 @@ public void lessThanTest() { @Test(expected = NoSuchElementException.class) public void nextThrowsNoSuchElementExceptionIfNoMoreElements() { - for (final Dbi db : dbs) { - populateList(); - try (Txn txn = env.txnRead(); - CursorIterable c = db.iterate(txn)) { - final Iterator> i = c.iterator(); - while (i.hasNext()) { - final KeyVal kv = i.next(); - assertThat(TestUtils.getNativeInt(kv.key()), is(list.pollFirst())); - assertThat(kv.val().getInt(), is(list.pollFirst())); - } - assertThat(i.hasNext(), is(false)); - i.next(); + populateTestDataList(); + final Dbi db = getDb(); + try (Txn txn = env.txnRead(); + CursorIterable c = db.iterate(txn)) { + final Iterator> i = c.iterator(); + while (i.hasNext()) { + final KeyVal kv = i.next(); + assertThat(TestUtils.getNativeInt(kv.key()), is(list.pollFirst())); + assertThat(kv.val().getInt(), is(list.pollFirst())); } + assertThat(i.hasNext(), is(false)); + i.next(); } } @@ -341,29 +396,28 @@ public void openTest() { @Test public void removeOddElements() { - for (final Dbi db : dbs) { - verify(db, all(), 2, 4, 6, 8); - int idx = -1; - try (Txn txn = env.txnWrite()) { - try (CursorIterable ci = db.iterate(txn)) { - final Iterator> c = ci.iterator(); - while (c.hasNext()) { - c.next(); - idx++; - if (idx % 2 == 0) { - c.remove(); - } + final Dbi db = getDb(); + verify(db, all(), 2, 4, 6, 8); + int idx = -1; + try (Txn txn = env.txnWrite()) { + try (CursorIterable ci = db.iterate(txn)) { + final Iterator> c = ci.iterator(); + while (c.hasNext()) { + c.next(); + idx++; + if (idx % 2 == 0) { + c.remove(); } } - txn.commit(); } - verify(db, all(), 4, 8); + txn.commit(); } + verify(db, all(), 4, 8); } @Test(expected = Env.AlreadyClosedException.class) public void nextWithClosedEnvTest() { - for (final Dbi db : dbs) { + final Dbi db = getDb(); try (Txn txn = env.txnRead()) { try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { final Iterator> c = ci.iterator(); @@ -372,12 +426,11 @@ public void nextWithClosedEnvTest() { c.next(); } } - } } @Test(expected = Env.AlreadyClosedException.class) public void removeWithClosedEnvTest() { - for (final Dbi db : dbs) { + final Dbi db = getDb(); try (Txn txn = env.txnWrite()) { try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { final Iterator> c = ci.iterator(); @@ -389,12 +442,11 @@ public void removeWithClosedEnvTest() { c.remove(); } } - } } @Test(expected = Env.AlreadyClosedException.class) public void hasNextWithClosedEnvTest() { - for (final Dbi db : dbs) { + final Dbi db = getDb(); try (Txn txn = env.txnRead()) { try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { final Iterator> c = ci.iterator(); @@ -403,21 +455,20 @@ public void hasNextWithClosedEnvTest() { c.hasNext(); } } - } } @Test(expected = Env.AlreadyClosedException.class) public void forEachRemainingWithClosedEnvTest() { - for (final Dbi db : dbs) { + final Dbi db = getDb(); try (Txn txn = env.txnRead()) { try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { final Iterator> c = ci.iterator(); env.close(); - c.forEachRemaining(keyVal -> {}); + c.forEachRemaining(keyVal -> { + }); } } - } } // @Test @@ -469,9 +520,8 @@ public void forEachRemainingWithClosedEnvTest() { private void verify(final KeyRange range, final int... expected) { // Verify using all comparator types - for (final Dbi db : dbs) { - verify(range, db, expected); - } + final Dbi db = getDb(); + verify(range, db, expected); } private void verify( @@ -485,7 +535,7 @@ private void verify( final List results = new ArrayList<>(); try (Txn txn = env.txnRead(); - CursorIterable c = dbi.iterate(txn, range)) { + CursorIterable c = dbi.iterate(txn, range)) { for (final KeyVal kv : c) { final int key = kv.key().order(ByteOrder.nativeOrder()).getInt(); final int val = kv.val().getInt(); @@ -499,4 +549,29 @@ private void verify( assertThat(results.get(idx), is(expected[idx])); } } + + private Dbi getDb() { + final Dbi dbi = dbiFactory.factory.apply(env); + populateDatabase(dbi); + return dbi; + } + + + // -------------------------------------------------------------------------------- + + + private static class DbiFactory { + private final String name; + private final Function, Dbi> factory; + + private DbiFactory(String name, Function, Dbi> factory) { + this.name = name; + this.factory = factory; + } + + @Override + public String toString() { + return name; + } + } } diff --git a/src/test/java/org/lmdbjava/CursorIterableTest.java b/src/test/java/org/lmdbjava/CursorIterableTest.java index bf2eb9eb..7bcbd851 100644 --- a/src/test/java/org/lmdbjava/CursorIterableTest.java +++ b/src/test/java/org/lmdbjava/CursorIterableTest.java @@ -77,8 +77,8 @@ @RunWith(Parameterized.class) public final class CursorIterableTest { - private static final DbiFlagSet dbiFlagSet = MDB_CREATE; - private static final BufferProxy bufferProxy = ByteBufferProxy.PROXY_OPTIMAL; + private static final DbiFlagSet DBI_FLAGS = MDB_CREATE; + private static final BufferProxy BUFFER_PROXY = ByteBufferProxy.PROXY_OPTIMAL; @Rule public final TemporaryFolder tmp = new TemporaryFolder(); @@ -87,48 +87,35 @@ public final class CursorIterableTest { private Deque list; /** Injected by {@link #data()} with appropriate runner. */ + @SuppressWarnings("ClassEscapesDefinedScope") @Parameterized.Parameter public DbiFactory dbiFactory; - @Before - public void before() throws IOException { - final File path = tmp.newFile(); - final BufferProxy bufferProxy = ByteBufferProxy.PROXY_OPTIMAL; - env = - create(bufferProxy) - .setMapSize(KIBIBYTES.toBytes(256)) - .setMaxReaders(1) - .setMaxDbs(3) - .open(path, POSIX_MODE, MDB_NOSUBDIR); - - populateTestDataList(); - } - @Parameterized.Parameters(name = "{index}: dbi: {0}") public static Object[] data() { final DbiFactory defaultComparator = new DbiFactory("defaultComparator", env -> env.buildDbi() .withDbName(DB_1) .withDefaultComparator() - .withDbiFlags(dbiFlagSet) + .withDbiFlags(DBI_FLAGS) .open()); final DbiFactory nativeComparator = new DbiFactory("nativeComparator", env -> env.buildDbi() .withDbName(DB_2) .withNativeComparator() - .withDbiFlags(dbiFlagSet) + .withDbiFlags(DBI_FLAGS) .open()); final DbiFactory callbackComparator = new DbiFactory("callbackComparator", env -> env.buildDbi() .withDbName(DB_3) - .withCallbackComparator(bufferProxy.getComparator(dbiFlagSet)) - .withDbiFlags(dbiFlagSet) + .withCallbackComparator(BUFFER_PROXY.getComparator(DBI_FLAGS)) + .withDbiFlags(DBI_FLAGS) .open()); final DbiFactory iteratorComparator = new DbiFactory("iteratorComparator", env -> env.buildDbi() .withDbName(DB_4) - .withIteratorComparator(bufferProxy.getComparator(dbiFlagSet)) - .withDbiFlags(dbiFlagSet) + .withIteratorComparator(BUFFER_PROXY.getComparator(DBI_FLAGS)) + .withDbiFlags(DBI_FLAGS) .open()); return new Object[] { defaultComparator, @@ -137,6 +124,20 @@ public static Object[] data() { iteratorComparator}; } + @Before + public void before() throws IOException { + final File path = tmp.newFile(); + final BufferProxy bufferProxy = ByteBufferProxy.PROXY_OPTIMAL; + env = + create(bufferProxy) + .setMapSize(KIBIBYTES.toBytes(256)) + .setMaxReaders(1) + .setMaxDbs(3) + .open(path, POSIX_MODE, MDB_NOSUBDIR); + + populateTestDataList(); + } + @After public void after() { env.close(); diff --git a/src/test/java/org/lmdbjava/DbiBuilderTest.java b/src/test/java/org/lmdbjava/DbiBuilderTest.java index da9341c6..74fbd8f5 100644 --- a/src/test/java/org/lmdbjava/DbiBuilderTest.java +++ b/src/test/java/org/lmdbjava/DbiBuilderTest.java @@ -45,7 +45,6 @@ public void after() { @Before public void before() throws IOException { - System.out.println("before"); final File path = tmp.newFile(); env = create() .setMapSize(MEBIBYTES.toBytes(64)) From 58dcc6e8bb1b393188ef3c1d86015ae15b50e1fb Mon Sep 17 00:00:00 2001 From: at055612 <22818309+at055612@users.noreply.github.com> Date: Wed, 29 Oct 2025 16:09:53 +0000 Subject: [PATCH 17/21] Tidy code --- .../java/org/lmdbjava/ByteArrayProxy.java | 6 +- .../java/org/lmdbjava/ByteBufferProxy.java | 2 +- .../CursorIterableIntegerDupTest.java | 563 ++++++++++++++++++ .../CursorIterableIntegerKeyTest.java | 89 --- 4 files changed, 566 insertions(+), 94 deletions(-) create mode 100644 src/test/java/org/lmdbjava/CursorIterableIntegerDupTest.java diff --git a/src/main/java/org/lmdbjava/ByteArrayProxy.java b/src/main/java/org/lmdbjava/ByteArrayProxy.java index d7c23919..82b7721c 100644 --- a/src/main/java/org/lmdbjava/ByteArrayProxy.java +++ b/src/main/java/org/lmdbjava/ByteArrayProxy.java @@ -36,8 +36,6 @@ public final class ByteArrayProxy extends BufferProxy { private static final MemoryManager MEM_MGR = RUNTIME.getMemoryManager(); - private static final Comparator unsignedComparator = ByteArrayProxy::compareArrays; - private ByteArrayProxy() {} /** @@ -47,7 +45,7 @@ private ByteArrayProxy() {} * @param o2 right operand (required) * @return as specified by {@link Comparable} interface */ - public static int compareArrays(final byte[] o1, final byte[] o2) { + public static int compareLexicographically(final byte[] o1, final byte[] o2) { requireNonNull(o1); requireNonNull(o2); if (o1 == o2) { @@ -84,7 +82,7 @@ protected byte[] getBytes(final byte[] buffer) { @Override public Comparator getComparator(final DbiFlagSet dbiFlagSet) { - return unsignedComparator; + return ByteArrayProxy::compareLexicographically; } @Override diff --git a/src/main/java/org/lmdbjava/ByteBufferProxy.java b/src/main/java/org/lmdbjava/ByteBufferProxy.java index ca4deba3..89931587 100644 --- a/src/main/java/org/lmdbjava/ByteBufferProxy.java +++ b/src/main/java/org/lmdbjava/ByteBufferProxy.java @@ -154,7 +154,7 @@ public static int compareLexicographically(final ByteBuffer o1, final ByteBuffer * Buffer comparator specifically for 4/8 byte keys that are unsigned ints/longs, * i.e. when using MDB_INTEGER_KEY/MDB_INTEGERDUP. Compares the buffers numerically. *

- * Both buffer must have 4 or 8 bytes remaining + * Both buffers must have 4 or 8 bytes remaining *

* * @param o1 left operand (required) diff --git a/src/test/java/org/lmdbjava/CursorIterableIntegerDupTest.java b/src/test/java/org/lmdbjava/CursorIterableIntegerDupTest.java new file mode 100644 index 00000000..1acf4328 --- /dev/null +++ b/src/test/java/org/lmdbjava/CursorIterableIntegerDupTest.java @@ -0,0 +1,563 @@ +/* + * Copyright © 2016-2025 The LmdbJava Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lmdbjava; + +import static com.jakewharton.byteunits.BinaryByteUnit.KIBIBYTES; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasSize; +import static org.lmdbjava.DbiFlags.MDB_CREATE; +import static org.lmdbjava.DbiFlags.MDB_DUPSORT; +import static org.lmdbjava.DbiFlags.MDB_INTEGERDUP; +import static org.lmdbjava.Env.create; +import static org.lmdbjava.EnvFlags.MDB_NOSUBDIR; +import static org.lmdbjava.KeyRange.all; +import static org.lmdbjava.KeyRange.allBackward; +import static org.lmdbjava.KeyRange.atLeast; +import static org.lmdbjava.KeyRange.atLeastBackward; +import static org.lmdbjava.KeyRange.atMost; +import static org.lmdbjava.KeyRange.atMostBackward; +import static org.lmdbjava.KeyRange.closed; +import static org.lmdbjava.KeyRange.closedBackward; +import static org.lmdbjava.KeyRange.closedOpen; +import static org.lmdbjava.KeyRange.closedOpenBackward; +import static org.lmdbjava.KeyRange.greaterThan; +import static org.lmdbjava.KeyRange.greaterThanBackward; +import static org.lmdbjava.KeyRange.lessThan; +import static org.lmdbjava.KeyRange.lessThanBackward; +import static org.lmdbjava.KeyRange.open; +import static org.lmdbjava.KeyRange.openBackward; +import static org.lmdbjava.KeyRange.openClosed; +import static org.lmdbjava.KeyRange.openClosedBackward; +import static org.lmdbjava.TestUtils.DB_1; +import static org.lmdbjava.TestUtils.DB_2; +import static org.lmdbjava.TestUtils.DB_3; +import static org.lmdbjava.TestUtils.DB_4; +import static org.lmdbjava.TestUtils.POSIX_MODE; +import static org.lmdbjava.TestUtils.bb; +import static org.lmdbjava.TestUtils.bbNative; +import static org.lmdbjava.TestUtils.getNativeInt; + +import com.google.common.primitives.UnsignedBytes; +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.Deque; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.hamcrest.CoreMatchers; +import org.hamcrest.Matchers; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.lmdbjava.CursorIterable.KeyVal; + +/** + * Test {@link CursorIterable} using {@link DbiFlags#MDB_INTEGERKEY} to ensure that + * comparators work with native order integer keys. + */ +@Ignore // Waiting for the merge of stroomdev66's cursor tests +@RunWith(Parameterized.class) +public final class CursorIterableIntegerDupTest { + + private static final DbiFlagSet DBI_FLAGS = DbiFlagSet.of(MDB_CREATE, MDB_INTEGERDUP, MDB_DUPSORT); + private static final BufferProxy BUFFER_PROXY = ByteBufferProxy.PROXY_OPTIMAL; + private static final List> INPUT_DATA; + + static { + // 2 => 21 + // 2 => 22 + // 3 => 31 + // ... + // 9 => 92 + INPUT_DATA = new ArrayList<>(); + for (int i = 2; i <= 9; i++) { + final int val1 = (i * 10) + 1; + final int val2 = (i * 10) + 2; + INPUT_DATA.add(new AbstractMap.SimpleEntry<>(i, val1)); + INPUT_DATA.add(new AbstractMap.SimpleEntry<>(i, val2)); + } + } + + @Rule + public final TemporaryFolder tmp = new TemporaryFolder(); + + private Env env; + private Deque> expectedEntriesDeque; + + /** + * Injected by {@link #data()} with appropriate runner. + */ + @SuppressWarnings("ClassEscapesDefinedScope") + @Parameterized.Parameter + public DbiFactory dbiFactory; + + @Parameterized.Parameters(name = "{index}: dbi: {0}") + public static Object[] data() { + final DbiFactory defaultComparator = new DbiFactory("defaultComparator", env -> + env.buildDbi() + .withDbName(DB_1) + .withDefaultComparator() + .withDbiFlags(DBI_FLAGS) + .open()); + final DbiFactory nativeComparator = new DbiFactory("nativeComparator", env -> + env.buildDbi() + .withDbName(DB_2) + .withNativeComparator() + .withDbiFlags(DBI_FLAGS) + .open()); + final DbiFactory callbackComparator = new DbiFactory("callbackComparator", env -> + env.buildDbi() + .withDbName(DB_3) + .withCallbackComparator(BUFFER_PROXY.getComparator(DBI_FLAGS)) + .withDbiFlags(DBI_FLAGS) + .open()); + final DbiFactory iteratorComparator = new DbiFactory("iteratorComparator", env -> + env.buildDbi() + .withDbName(DB_4) + .withIteratorComparator(BUFFER_PROXY.getComparator(DBI_FLAGS)) + .withDbiFlags(DBI_FLAGS) + .open()); + return new Object[]{ + defaultComparator, + nativeComparator, + callbackComparator, + iteratorComparator}; + } + + @Before + public void before() throws IOException { + final File path = tmp.newFile(); + final BufferProxy bufferProxy = ByteBufferProxy.PROXY_OPTIMAL; + env = + create(bufferProxy) + .setMapSize(KIBIBYTES.toBytes(256)) + .setMaxReaders(1) + .setMaxDbs(3) + .open(path, POSIX_MODE, MDB_NOSUBDIR); + + populateExpectedEntriesDeque(); + } + + @After + public void after() { + env.close(); + } + + @Test + public void allBackwardTest() { + verify(allBackward(), 8, 6, 4, 2); + } + + @Test + public void allTest() { + verify(all(), 2, 4, 6, 8); + } + + @Test + public void atLeastBackwardTest() { + verify(atLeastBackward(bbNative(5)), 4, 2); + verify(atLeastBackward(bbNative(6)), 6, 4, 2); + verify(atLeastBackward(bbNative(9)), 8, 6, 4, 2); + } + + @Test + public void atLeastTest() { + verify(atLeast(bbNative(5)), 6, 8); + verify(atLeast(bbNative(6)), 6, 8); + } + + @Test + public void atMostBackwardTest() { + verify(atMostBackward(bbNative(5)), 8, 6); + verify(atMostBackward(bbNative(6)), 8, 6); + } + + @Test + public void atMostTest() { + verify(atMost(bbNative(5)), 2, 4); + verify(atMost(bbNative(6)), 2, 4, 6); + } + + private void populateExpectedEntriesDeque() { + expectedEntriesDeque = new LinkedList<>(); + expectedEntriesDeque.addAll(INPUT_DATA); + } + + private void populateDatabase(final Dbi dbi) { + try (Txn txn = env.txnWrite()) { + final Cursor c = dbi.openCursor(txn); + for (Map.Entry entry : INPUT_DATA) { + c.put(bbNative(entry.getKey()), bb(entry.getValue())); + } + txn.commit(); + } + + try (Txn txn = env.txnRead(); + CursorIterable c = dbi.iterate(txn)) { + + for (final KeyVal kv : c) { + System.out.print(getNativeInt(kv.key()) + " => " + kv.val().getInt()); + System.out.print(", "); + } + System.out.println(); + } + } + + private int[] rangeInc(final int fromInc, final int toInc) { + int idx = 0; + if (fromInc <= toInc) { + // Forwards + final int[] arr = new int[toInc - fromInc + 1]; + for (int i = fromInc; i <= toInc; i++) { + arr[idx++] = i; + } + return arr; + } else { + // Backwards + final int[] arr = new int[fromInc - toInc + 1]; + for (int i = fromInc; i >= toInc; i--) { + arr[idx++] = i; + } + return arr; + } + } + + @Test + public void closedBackwardTest() { + verify(closedBackward(bbNative(7), bbNative(3)), rangeInc(7, 3)); + verify(closedBackward(bbNative(6), bbNative(2)), rangeInc(6, 2)); + verify(closedBackward(bbNative(9), bbNative(3)), rangeInc(9, 3)); + } + + @Test + public void closedOpenBackwardTest() { + verify(closedOpenBackward(bbNative(8), bbNative(3)), rangeInc(8, 4)); + verify(closedOpenBackward(bbNative(7), bbNative(2)), rangeInc(7, 3)); + verify(closedOpenBackward(bbNative(9), bbNative(3)), rangeInc(9, 4)); + } + + @Test + public void closedOpenTest() { + verify(closedOpen(bbNative(3), bbNative(8)), rangeInc(3, 7)); + verify(closedOpen(bbNative(2), bbNative(6)), rangeInc(2, 5)); + } + + @Test + public void closedTest() { + verify(closed(bbNative(3), bbNative(7)), rangeInc(3, 7)); + verify(closed(bbNative(2), bbNative(6)), rangeInc(2, 6)); + verify(closed(bbNative(1), bbNative(7)), rangeInc(2, 7)); + } + + @Test + public void greaterThanBackwardTest() { + verify(greaterThanBackward(bbNative(6)), rangeInc(5, 2)); + verify(greaterThanBackward(bbNative(7)), rangeInc(6, 2)); + verify(greaterThanBackward(bbNative(9)), rangeInc(8, 2)); + } + + @Test + public void greaterThanTest() { + verify(greaterThan(bbNative(4)), rangeInc(5, 9)); + verify(greaterThan(bbNative(3)), rangeInc(4, 9)); + } + + @Test(expected = IllegalStateException.class) + public void iterableOnlyReturnedOnce() { + final Dbi db = getDb(); + try (Txn txn = env.txnRead(); + CursorIterable c = db.iterate(txn)) { + c.iterator(); // ok + c.iterator(); // fails + } + } + + @Test + public void iterate() { + populateExpectedEntriesDeque(); + final Dbi db = getDb(); + try (Txn txn = env.txnRead(); + CursorIterable c = db.iterate(txn)) { + + for (final KeyVal kv : c) { + final Map.Entry entry = expectedEntriesDeque.pollFirst(); +// System.out.println(entry.getKey() + " => " + entry.getValue()); + assertThat(getNativeInt(kv.key()), is(entry.getKey())); + assertThat(kv.val().getInt(), is(entry.getValue())); + } + } + } + + @Test(expected = IllegalStateException.class) + public void iteratorOnlyReturnedOnce() { + final Dbi db = getDb(); + try (Txn txn = env.txnRead(); + CursorIterable c = db.iterate(txn)) { + c.iterator(); // ok + c.iterator(); // fails + } + } + + @Test + public void lessThanBackwardTest() { + verify(lessThanBackward(bbNative(5)), 8, 6); + verify(lessThanBackward(bbNative(2)), 8, 6, 4); + } + + @Test + public void lessThanTest() { + verify(lessThan(bbNative(5)), 2, 4); + verify(lessThan(bbNative(8)), 2, 4, 6); + } + + @Test(expected = NoSuchElementException.class) + public void nextThrowsNoSuchElementExceptionIfNoMoreElements() { + populateExpectedEntriesDeque(); + final Dbi db = getDb(); + try (Txn txn = env.txnRead(); + CursorIterable c = db.iterate(txn)) { + final Iterator> i = c.iterator(); + while (i.hasNext()) { + final KeyVal kv = i.next(); + assertThat(getNativeInt(kv.key()), is(expectedEntriesDeque.pollFirst())); + assertThat(kv.val().getInt(), is(expectedEntriesDeque.pollFirst())); + } + assertThat(i.hasNext(), is(false)); + i.next(); + } + } + + @Test + public void openBackwardTest() { + verify(openBackward(bbNative(7), bbNative(2)), 6, 4); + verify(openBackward(bbNative(8), bbNative(1)), 6, 4, 2); + verify(openBackward(bbNative(9), bbNative(4)), 8, 6); + } + + @Test + public void openClosedBackwardTest() { + verify(openClosedBackward(bbNative(7), bbNative(2)), 6, 4, 2); + verify(openClosedBackward(bbNative(8), bbNative(4)), 6, 4); + verify(openClosedBackward(bbNative(9), bbNative(4)), 8, 6, 4); + } + + @Test + public void openClosedBackwardTestWithGuava() { + final Comparator guava = UnsignedBytes.lexicographicalComparator(); + final Comparator comparator = + (bb1, bb2) -> { + final byte[] array1 = new byte[bb1.remaining()]; + final byte[] array2 = new byte[bb2.remaining()]; + bb1.mark(); + bb2.mark(); + bb1.get(array1); + bb2.get(array2); + bb1.reset(); + bb2.reset(); + return guava.compare(array1, array2); + }; + final Dbi guavaDbi = env.openDbi(DB_1, comparator, MDB_CREATE); + populateDatabase(guavaDbi); + verify(openClosedBackward(bbNative(7), bbNative(2)), guavaDbi, 6, 4, 2); + verify(openClosedBackward(bbNative(8), bbNative(4)), guavaDbi, 6, 4); + } + + @Test + public void openClosedTest() { + verify(openClosed(bbNative(3), bbNative(8)), 4, 6, 8); + verify(openClosed(bbNative(2), bbNative(6)), 4, 6); + } + + @Test + public void openTest() { + verify(open(bbNative(3), bbNative(7)), 4, 6); + verify(open(bbNative(2), bbNative(8)), 4, 6); + } + + @Test + public void removeOddElements() { + final Dbi db = getDb(); + verify(db, all(), 2, 4, 6, 8); + int idx = -1; + try (Txn txn = env.txnWrite()) { + try (CursorIterable ci = db.iterate(txn)) { + final Iterator> c = ci.iterator(); + while (c.hasNext()) { + c.next(); + idx++; + if (idx % 2 == 0) { + c.remove(); + } + } + } + txn.commit(); + } + verify(db, all(), 4, 8); + } + + @Test(expected = Env.AlreadyClosedException.class) + public void nextWithClosedEnvTest() { + final Dbi db = getDb(); + try (Txn txn = env.txnRead()) { + try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { + final Iterator> c = ci.iterator(); + + env.close(); + c.next(); + } + } + } + + @Test(expected = Env.AlreadyClosedException.class) + public void removeWithClosedEnvTest() { + final Dbi db = getDb(); + try (Txn txn = env.txnWrite()) { + try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { + final Iterator> c = ci.iterator(); + + final KeyVal keyVal = c.next(); + assertThat(keyVal, Matchers.notNullValue()); + + env.close(); + c.remove(); + } + } + } + + @Test(expected = Env.AlreadyClosedException.class) + public void hasNextWithClosedEnvTest() { + final Dbi db = getDb(); + try (Txn txn = env.txnRead()) { + try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { + final Iterator> c = ci.iterator(); + + env.close(); + c.hasNext(); + } + } + } + + @Test(expected = Env.AlreadyClosedException.class) + public void forEachRemainingWithClosedEnvTest() { + final Dbi db = getDb(); + try (Txn txn = env.txnRead()) { + try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { + final Iterator> c = ci.iterator(); + + env.close(); + c.forEachRemaining(keyVal -> { + }); + } + } + } + + private void verify(final KeyRange range, final int... expectedKeys) { + // Verify using all comparator types + final Dbi db = getDb(); + verify(range, db, expectedKeys); + } + + private void verify(final Dbi dbi, + final KeyRange range, + final int... expectedKeys) { + verify(range, dbi, expectedKeys); + } + + private void verify(final KeyRange range, + final Dbi dbi, + final int... expectedKeys) { + final boolean isForward = range.getType().isDirectionForward(); + + final List expectedValues = Arrays.stream(expectedKeys) + .boxed() + .flatMap(key -> { + final int base = key * 10; + return isForward + ? Stream.of(base + 1, base + 2) + : Stream.of(base + 2, base + 1); + }) + .collect(Collectors.toList()); + + final List results = new ArrayList<>(); + System.out.println(rangeToString(range) + ", expected: " + expectedValues); + + try (Txn txn = env.txnRead(); + CursorIterable c = dbi.iterate(txn, range)) { + for (final KeyVal kv : c) { + final int key = getNativeInt(kv.key()); + final int val = kv.val().getInt(); + System.out.println(key + " => " + val); + results.add(val); + assertThat(val, CoreMatchers.anyOf( + CoreMatchers.is((key * 10) + 1), + CoreMatchers.is((key * 10) + 2))); + } + } + + assertThat(results, hasSize(expectedValues.size())); + for (int idx = 0; idx < results.size(); idx++) { + assertThat(results.get(idx), is(expectedValues.get(idx))); + } + } + + private String rangeToString(final KeyRange range) { + final ByteBuffer start = range.getStart(); + final ByteBuffer stop = range.getStop(); + return range.getType() + " start: " + (start != null ? getNativeInt(start) : "") + + " stop: " + (stop != null ? getNativeInt(stop) : ""); + } + + private Dbi getDb() { + final Dbi dbi = dbiFactory.factory.apply(env); + populateDatabase(dbi); + return dbi; + } + + + // -------------------------------------------------------------------------------- + + + private static class DbiFactory { + private final String name; + private final Function, Dbi> factory; + + private DbiFactory(String name, Function, Dbi> factory) { + this.name = name; + this.factory = factory; + } + + @Override + public String toString() { + return name; + } + } +} diff --git a/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java b/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java index 85c0a567..bac95e13 100644 --- a/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java +++ b/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java @@ -143,47 +143,6 @@ public void before() throws IOException { .open(path, POSIX_MODE, MDB_NOSUBDIR); populateTestDataList(); -// final File path = tmp.newFile(); -// final BufferProxy bufferProxy = ByteBufferProxy.PROXY_OPTIMAL; -// env = -// create(bufferProxy) -// .setMapSize(KIBIBYTES.toBytes(256)) -// .setMaxReaders(1) -// .setMaxDbs(3) -// .open(path, POSIX_MODE, MDB_NOSUBDIR); -// -// // Use a java comparator for start/stop keys only -// DbiFlagSet dbiFlagSet = DbiFlagSet.of(MDB_CREATE, MDB_INTEGERKEY); -// -// dbJavaComparator = env.buildDbi() -// .withDbName(DB_1) -// .withIteratorComparator(bufferProxy.getComparator(dbiFlagSet)) -// .withDbiFlags(dbiFlagSet) -// .open(); -// -// // Use LMDB comparator for start/stop keys -// dbLmdbComparator = env.buildDbi() -// .withDbName(DB_2) -// .withDefaultComparator() -// .withDbiFlags(dbiFlagSet) -// .open(); -// -// // Use a java comparator for start/stop keys and as a callback comparaotr -// dbCallbackComparator = env.buildDbi() -// .withDbName(DB_3) -// .withCallbackComparator(bufferProxy.getComparator(dbiFlagSet)) -// .withDbiFlags(dbiFlagSet) -// .open(); -// -// populateTestDataList(); -// -// populateDatabase(dbJavaComparator); -// populateDatabase(dbLmdbComparator); -// populateDatabase(dbCallbackComparator); -// -// dbs.add(dbJavaComparator); -// dbs.add(dbLmdbComparator); -// dbs.add(dbCallbackComparator); } @After @@ -226,7 +185,6 @@ public void atMostTest() { verify(atMost(bbNative(6)), 2, 4, 6); } - private void populateTestDataList() { list = new LinkedList<>(); list.addAll(asList(2, 3, 4, 5, 6, 7, 8, 9)); @@ -471,53 +429,6 @@ public void forEachRemainingWithClosedEnvTest() { } } -// @Test -// public void testSignedVsUnsigned() { -// final ByteBuffer val1 = bbNative(1); -// final ByteBuffer val2 = bbNative(2); -// final ByteBuffer val110 = bbNative(110); -// final ByteBuffer val111 = bbNative(111); -// final ByteBuffer val150 = bbNative(150); -// -// final BufferProxy bufferProxy = ByteBufferProxy.PROXY_OPTIMAL; -// final Comparator unsignedComparator = bufferProxy.getUnsignedComparator(); -// final Comparator signedComparator = bufferProxy.getSignedComparator(); -// -// // Compare the same -// assertThat( -// unsignedComparator.compare(val1, val2), Matchers.is(signedComparator.compare(val1, val2))); -// -// // Compare differently -// assertThat( -// unsignedComparator.compare(val110, val150), -// Matchers.not(signedComparator.compare(val110, val150))); -// -// // Compare differently -// assertThat( -// unsignedComparator.compare(val111, val150), -// Matchers.not(signedComparator.compare(val111, val150))); -// -// // This will fail if the db is using a signed comparator for the start/stop keys -// for (final Dbi db : dbs) { -// db.put(val110, val110); -// db.put(val150, val150); -// -// final ByteBuffer startKeyBuf = val111; -// KeyRange keyRange = KeyRange.atLeastBackward(startKeyBuf); -// -// try (Txn txn = env.txnRead(); -// CursorIterable c = db.iterate(txn, keyRange)) { -// for (final KeyVal kv : c) { -// final int key = getNativeInt(kv.key()); -// final int val = kv.val().getInt(); -// // System.out.println("key: " + key + " val: " + val); -// assertThat(key, is(110)); -// break; -// } -// } -// } -// } - private void verify(final KeyRange range, final int... expected) { // Verify using all comparator types final Dbi db = getDb(); From 26665ba0cfaafaa082d3e22feeb4080cb0bc7849 Mon Sep 17 00:00:00 2001 From: at055612 <22818309+at055612@users.noreply.github.com> Date: Wed, 29 Oct 2025 17:45:21 +0000 Subject: [PATCH 18/21] Fix byte order issues with compareAsIntegerKeys --- src/main/java/org/lmdbjava/BufferProxy.java | 4 - .../java/org/lmdbjava/ByteBufferProxy.java | 32 ++++--- src/main/java/org/lmdbjava/Dbi.java | 30 ++++-- src/main/java/org/lmdbjava/DbiFlagSet.java | 6 ++ src/main/java/org/lmdbjava/DbiFlags.java | 2 +- .../java/org/lmdbjava/DirectBufferProxy.java | 2 +- .../org/lmdbjava/ByteBufferProxyTest.java | 39 +++++--- .../CursorIterableIntegerKeyTest.java | 96 +++++++++++++++++++ src/test/java/org/lmdbjava/TestUtils.java | 33 ++++++- 9 files changed, 204 insertions(+), 40 deletions(-) diff --git a/src/main/java/org/lmdbjava/BufferProxy.java b/src/main/java/org/lmdbjava/BufferProxy.java index af0c7f06..f857ade7 100644 --- a/src/main/java/org/lmdbjava/BufferProxy.java +++ b/src/main/java/org/lmdbjava/BufferProxy.java @@ -40,10 +40,6 @@ public abstract class BufferProxy { /** Offset from a pointer of the MDB_val.mv_size field. */ protected static final int STRUCT_FIELD_OFFSET_SIZE = 0; - /** The set of {@link DbiFlags} that indicate unsigned integer keys are being used. */ - protected static final DbiFlagSet INTEGER_KEY_FLAGS = DbiFlagSet.of( - DbiFlags.MDB_INTEGERKEY, - DbiFlags.MDB_INTEGERDUP); /** Explicitly-defined default constructor to avoid warnings. */ protected BufferProxy() {} diff --git a/src/main/java/org/lmdbjava/ByteBufferProxy.java b/src/main/java/org/lmdbjava/ByteBufferProxy.java index 89931587..5d5aa2ca 100644 --- a/src/main/java/org/lmdbjava/ByteBufferProxy.java +++ b/src/main/java/org/lmdbjava/ByteBufferProxy.java @@ -27,6 +27,7 @@ import java.lang.reflect.Field; import java.nio.Buffer; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.ArrayDeque; import java.util.Comparator; import jnr.ffi.Pointer; @@ -153,9 +154,6 @@ public static int compareLexicographically(final ByteBuffer o1, final ByteBuffer /** * Buffer comparator specifically for 4/8 byte keys that are unsigned ints/longs, * i.e. when using MDB_INTEGER_KEY/MDB_INTEGERDUP. Compares the buffers numerically. - *

- * Both buffers must have 4 or 8 bytes remaining - *

* * @param o1 left operand (required) * @param o2 right operand (required) @@ -164,26 +162,34 @@ public static int compareLexicographically(final ByteBuffer o1, final ByteBuffer public static int compareAsIntegerKeys(final ByteBuffer o1, final ByteBuffer o2) { requireNonNull(o1); requireNonNull(o2); - // Both buffers should be same len + // Both buffers should be same lenght according to LMDB API. final int len1 = o1.limit(); final int len2 = o2.limit(); if (len1 != len2) { throw new RuntimeException("Length mismatch, len1: " + len1 + ", len2: " + len2 + ". Lengths must be identical and either 4 or 8 bytes."); } - final boolean reverse1 = o1.order() == LITTLE_ENDIAN; - final boolean reverse2 = o2.order() == LITTLE_ENDIAN; + // Keys for MDB_INTEGER_KEY are written in native order so ensure we read them in that order + o1.order(ByteOrder.nativeOrder()); + o2.order(ByteOrder.nativeOrder()); + // TODO it might be worth the DbiBuilder having a method to capture fixedKeyLength() or -1 + // for variable length keys. This can be passed to getComparator(..) so it can return a + // comparator that doesn't need to test the length every time. There may be other benefits + // to the Dbi knowing the key length if it is fixed. if (len1 == 8) { - final long lw = reverse1 ? Long.reverseBytes(o1.getLong(0)) : o1.getLong(0); - final long rw = reverse2 ? Long.reverseBytes(o2.getLong(0)) : o2.getLong(0); + final long lw = o1.getLong(0); + final long rw = o2.getLong(0); return Long.compareUnsigned(lw, rw); } else if (len1 == 4) { - final int lw = reverse1 ? Integer.reverseBytes(o1.getInt(0)) : o1.getInt(0); - final int rw = reverse2 ? Integer.reverseBytes(o2.getInt(0)) : o2.getInt(0); + final int lw = o1.getInt(0); + final int rw = o2.getInt(0); return Integer.compareUnsigned(lw, rw); } else { - throw new RuntimeException("Unexpected length1: " + len1 - + ". Lengths must be identical and either 4 or 8 bytes."); + // size_t and int are likely to be 8bytes and 4bytes respectively on 64bit. + // If 32bit then would be 4/2 respectively. + // Short.compareUnsigned is not available in Java8. + // For now just fall back to our standard comparator + return compareLexicographically(o1, o2); } } @@ -222,7 +228,7 @@ protected final ByteBuffer allocate() { @Override public Comparator getComparator(final DbiFlagSet dbiFlagSet) { - if (dbiFlagSet.areAnySet(INTEGER_KEY_FLAGS)) { + if (dbiFlagSet.areAnySet(DbiFlagSet.INTEGER_KEY_FLAGS)) { return AbstractByteBufferProxy::compareAsIntegerKeys; } else { return AbstractByteBufferProxy::compareLexicographically; diff --git a/src/main/java/org/lmdbjava/Dbi.java b/src/main/java/org/lmdbjava/Dbi.java index bcb1ccfc..5e8fa2f2 100644 --- a/src/main/java/org/lmdbjava/Dbi.java +++ b/src/main/java/org/lmdbjava/Dbi.java @@ -82,15 +82,27 @@ public final class Dbi { if (nativeCb) { requireNonNull(comparator, "comparator cannot be null if nativeCb is set"); // LMDB will call back to this comparator for insertion/iteration order - this.callbackComparator = - (keyA, keyB) -> { - final T compKeyA = proxy.out(proxy.allocate(), keyA); - final T compKeyB = proxy.out(proxy.allocate(), keyB); - final int result = this.comparator.compare(compKeyA, compKeyB); - proxy.deallocate(compKeyA); - proxy.deallocate(compKeyB); - return result; - }; +// if (dbiFlagSet.areAnySet(DbiFlagSet.INTEGER_KEY_FLAGS)) { +// this.callbackComparator = +// (keyA, keyB) -> { +// final T compKeyA = proxy.out(proxy.allocate(), keyA); +// final T compKeyB = proxy.out(proxy.allocate(), keyB); +// final int result = this.comparator.compare(compKeyA, compKeyB); +// proxy.deallocate(compKeyA); +// proxy.deallocate(compKeyB); +// return result; +// }; +// } else { + this.callbackComparator = + (keyA, keyB) -> { + final T compKeyA = proxy.out(proxy.allocate(), keyA); + final T compKeyB = proxy.out(proxy.allocate(), keyB); + final int result = this.comparator.compare(compKeyA, compKeyB); + proxy.deallocate(compKeyA); + proxy.deallocate(compKeyB); + return result; + }; +// } LIB.mdb_set_compare(txn.pointer(), ptr, callbackComparator); } else { callbackComparator = null; diff --git a/src/main/java/org/lmdbjava/DbiFlagSet.java b/src/main/java/org/lmdbjava/DbiFlagSet.java index 5edf10c7..5a0bc83e 100644 --- a/src/main/java/org/lmdbjava/DbiFlagSet.java +++ b/src/main/java/org/lmdbjava/DbiFlagSet.java @@ -21,8 +21,14 @@ public interface DbiFlagSet extends FlagSet { + /** An immutable empty {@link DbiFlagSet}. */ DbiFlagSet EMPTY = DbiFlagSetImpl.EMPTY; + /** The set of {@link DbiFlags} that indicate unsigned integer keys are being used. */ + DbiFlagSet INTEGER_KEY_FLAGS = DbiFlagSet.of( + DbiFlags.MDB_INTEGERKEY, + DbiFlags.MDB_INTEGERDUP); + static DbiFlagSet empty() { return DbiFlagSetImpl.EMPTY; } diff --git a/src/main/java/org/lmdbjava/DbiFlags.java b/src/main/java/org/lmdbjava/DbiFlags.java index 10952da9..7c4b6794 100644 --- a/src/main/java/org/lmdbjava/DbiFlags.java +++ b/src/main/java/org/lmdbjava/DbiFlags.java @@ -41,7 +41,7 @@ public enum DbiFlags implements MaskedFlag, DbiFlagSet { MDB_DUPSORT(0x04), /** * Numeric keys in native byte order: either unsigned int or size_t. - * The keys must all be of the same size. + * The keys must all be of the same size. *

* This is an optimisation that is available when your keys are 4 or 8 byte unsigned numeric values. * There are performance benefits for both ordered and un-ordered puts as compared to not using diff --git a/src/main/java/org/lmdbjava/DirectBufferProxy.java b/src/main/java/org/lmdbjava/DirectBufferProxy.java index 9c90d98b..180eee0a 100644 --- a/src/main/java/org/lmdbjava/DirectBufferProxy.java +++ b/src/main/java/org/lmdbjava/DirectBufferProxy.java @@ -138,7 +138,7 @@ protected DirectBuffer allocate() { @Override public Comparator getComparator(final DbiFlagSet dbiFlagSet) { - if (dbiFlagSet.areAnySet(INTEGER_KEY_FLAGS)) { + if (dbiFlagSet.areAnySet(DbiFlagSet.INTEGER_KEY_FLAGS)) { return DirectBufferProxy::compareAsIntegerKeys; } else { return DirectBufferProxy::compareLexicographically; diff --git a/src/test/java/org/lmdbjava/ByteBufferProxyTest.java b/src/test/java/org/lmdbjava/ByteBufferProxyTest.java index 1372b74a..c7d8333f 100644 --- a/src/test/java/org/lmdbjava/ByteBufferProxyTest.java +++ b/src/test/java/org/lmdbjava/ByteBufferProxyTest.java @@ -40,6 +40,7 @@ import java.io.IOException; import java.lang.reflect.Field; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.time.Duration; import java.time.Instant; import java.util.Comparator; @@ -158,8 +159,10 @@ public void comparatorPerformance() { int x = 0; for (int rounds = 0; rounds < 100; rounds++) { for (int i = 1; i < values.length; i++) { - buffer1.putLong(0, values[i - 1]); - buffer2.putLong(0, values[i]); + buffer1.order(ByteOrder.nativeOrder()) + .putLong(0, values[i - 1]); + buffer2.order(ByteOrder.nativeOrder()) + .putLong(0, values[i]); final int result = ByteBufferProxy.AbstractByteBufferProxy.compareAsIntegerKeys(buffer1, buffer2); x += result; } @@ -170,8 +173,10 @@ public void comparatorPerformance() { x = 0; for (int rounds = 0; rounds < 100; rounds++) { for (int i = 1; i < values.length; i++) { - buffer1.putLong(0, values[i - 1]); - buffer2.putLong(0, values[i]); + buffer1.order(BIG_ENDIAN) + .putLong(0, values[i - 1]); + buffer2.order(BIG_ENDIAN) + .putLong(0, values[i]); final int result = ByteBufferProxy.AbstractByteBufferProxy.compareLexicographically(buffer1, buffer2); x += result; } @@ -182,10 +187,14 @@ public void comparatorPerformance() { @Test public void verifyComparators() { final Random random = new Random(203948); - final ByteBuffer buffer1 = ByteBuffer.allocateDirect(Long.BYTES); - final ByteBuffer buffer2 = ByteBuffer.allocateDirect(Long.BYTES); - buffer1.limit(Long.BYTES); - buffer2.limit(Long.BYTES); + final ByteBuffer buffer1native = ByteBuffer.allocateDirect(Long.BYTES).order(ByteOrder.nativeOrder()); + final ByteBuffer buffer2native = ByteBuffer.allocateDirect(Long.BYTES).order(ByteOrder.nativeOrder()); + final ByteBuffer buffer1be = ByteBuffer.allocateDirect(Long.BYTES).order(BIG_ENDIAN); + final ByteBuffer buffer2be = ByteBuffer.allocateDirect(Long.BYTES).order(BIG_ENDIAN); + buffer1native.limit(Long.BYTES); + buffer2native.limit(Long.BYTES); + buffer1be.limit(Long.BYTES); + buffer2be.limit(Long.BYTES); final long[] values = random.longs(10_000_000).toArray(); final LinkedHashMap> comparators = new LinkedHashMap<>(); @@ -198,13 +207,21 @@ public void verifyComparators() { for (int i = 1; i < values.length; i++) { final long val1 = values[i - 1]; final long val2 = values[i]; - buffer1.putLong(0, val1); - buffer2.putLong(0, val2); + buffer1native.putLong(0, val1); + buffer2native.putLong(0, val2); + buffer1be.putLong(0, val1); + buffer2be.putLong(0, val2); uniqueResults.clear(); // Make sure all comparators give the same result for the same inputs comparators.forEach((name, comparator) -> { - final int result = comparator.compare(buffer1, buffer2); + final int result; + // IntegerKey comparator expects keys to have been written in native order so need different buffers. + if (name.equals("compareAsIntegerKeys")) { + result = comparator.compare(buffer1native, buffer2native); + } else { + result = comparator.compare(buffer1be, buffer2be); + } results.put(name, result); uniqueResults.add(result); }); diff --git a/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java b/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java index bac95e13..be91ea4c 100644 --- a/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java +++ b/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java @@ -51,20 +51,24 @@ import static org.lmdbjava.TestUtils.bb; import static org.lmdbjava.TestUtils.bbNative; import static org.lmdbjava.TestUtils.getNativeInt; +import static org.lmdbjava.TestUtils.getString; import com.google.common.primitives.UnsignedBytes; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.util.AbstractMap; import java.util.ArrayList; import java.util.Comparator; import java.util.Deque; import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.NoSuchElementException; import java.util.function.Function; +import java.util.stream.Collectors; import org.hamcrest.Matchers; import org.junit.After; import org.junit.Before; @@ -150,6 +154,98 @@ public void after() { env.close(); } + @Test + public void testNumericOrderLong() { + final Dbi dbi = dbiFactory.factory.apply(env); + + try (Txn txn = env.txnWrite()) { + final Cursor c = dbi.openCursor(txn); + long i = 1; + while (true) { +// System.out.println("putting " + i); + c.put(bbNative(i), bb(i + "-long")); + final long i2 = i * 10; + if (i2 < i) { + // Overflowed + break; + } + i = i2; + } + txn.commit(); + } + + final List> entries = new ArrayList<>(); + try (Txn txn = env.txnRead()) { + try (CursorIterable iterable = dbi.iterate(txn)) { + for (KeyVal keyVal : iterable) { + final String val = getString(keyVal.val()); + final long key = TestUtils.getNativeLong(keyVal.key()); + entries.add(new AbstractMap.SimpleEntry<>(key, val)); +// System.out.println(val); + } + } + } + + final List dbKeys = entries.stream() + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + final List dbKeysSorted = entries.stream() + .map(Map.Entry::getKey) + .sorted() + .collect(Collectors.toList()); + for (int i = 0; i < dbKeys.size(); i++) { + final long dbKey1 = dbKeys.get(i); + final long dbKey2 = dbKeysSorted.get(i); + assertThat(dbKey1, is(dbKey2)); + } + } + + @Test + public void testNumericOrderInt() { + final Dbi dbi = dbiFactory.factory.apply(env); + + try (Txn txn = env.txnWrite()) { + final Cursor c = dbi.openCursor(txn); + int i = 1; + while (true) { +// System.out.println("putting " + i); + c.put(bbNative(i), bb(i + "-int")); + final int i2 = i * 10; + if (i2 < i) { + // Overflowed + break; + } + i = i2; + } + txn.commit(); + } + + final List> entries = new ArrayList<>(); + try (Txn txn = env.txnRead()) { + try (CursorIterable iterable = dbi.iterate(txn)) { + for (KeyVal keyVal : iterable) { + final String val = getString(keyVal.val()); + final int key = TestUtils.getNativeInt(keyVal.key()); + entries.add(new AbstractMap.SimpleEntry<>(key, val)); +// System.out.println(val); + } + } + } + + final List dbKeys = entries.stream() + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + final List dbKeysSorted = entries.stream() + .map(Map.Entry::getKey) + .sorted() + .collect(Collectors.toList()); + for (int i = 0; i < dbKeys.size(); i++) { + final long dbKey1 = dbKeys.get(i); + final long dbKey2 = dbKeysSorted.get(i); + assertThat(dbKey1, is(dbKey2)); + } + } + @Test public void allBackwardTest() { verify(allBackward(), 8, 6, 4, 2); diff --git a/src/test/java/org/lmdbjava/TestUtils.java b/src/test/java/org/lmdbjava/TestUtils.java index c26c1c52..511619fe 100644 --- a/src/test/java/org/lmdbjava/TestUtils.java +++ b/src/test/java/org/lmdbjava/TestUtils.java @@ -24,6 +24,7 @@ import java.lang.reflect.InvocationTargetException; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; import org.agrona.MutableDirectBuffer; import org.agrona.concurrent.UnsafeBuffer; @@ -55,13 +56,29 @@ static ByteBuffer bb(final int value) { return bb; } + static ByteBuffer bb(final String value) { + final ByteBuffer bb = allocateDirect(100); + if (value != null) { + bb.put(value.getBytes(StandardCharsets.UTF_8)); + bb.flip(); + } + return bb; + } + static ByteBuffer bbNative(final int value) { - final ByteBuffer bb = allocateDirect(Long.BYTES) + final ByteBuffer bb = allocateDirect(Integer.BYTES) .order(ByteOrder.nativeOrder()); bb.putInt(value).flip(); return bb; } + static ByteBuffer bbNative(final long value) { + final ByteBuffer bb = allocateDirect(Long.BYTES) + .order(ByteOrder.nativeOrder()); + bb.putLong(value).flip(); + return bb; + } + static int getNativeInt(final ByteBuffer bb) { final int val = bb.order(ByteOrder.nativeOrder()) .getInt(); @@ -69,6 +86,20 @@ static int getNativeInt(final ByteBuffer bb) { return val; } + static long getNativeLong(final ByteBuffer bb) { + final long val = bb.order(ByteOrder.nativeOrder()) + .getLong(); + bb.rewind(); + return val; + } + + static String getString(final ByteBuffer bb) { + final String str = StandardCharsets.UTF_8.decode(bb) + .toString(); + bb.rewind(); + return str; + } + static void invokePrivateConstructor(final Class clazz) { try { final Constructor c = clazz.getDeclaredConstructor(); From c4278017c23b10fe2e052e27ecf4fe6702cad5f8 Mon Sep 17 00:00:00 2001 From: at055612 <22818309+at055612@users.noreply.github.com> Date: Tue, 4 Nov 2025 15:01:16 +0000 Subject: [PATCH 19/21] Add/refactor tests --- .../java/org/lmdbjava/AbstractFlagSet.java | 13 + .../java/org/lmdbjava/ByteBufferProxy.java | 4 +- src/main/java/org/lmdbjava/Cursor.java | 2 +- src/main/java/org/lmdbjava/DbiBuilder.java | 21 +- .../org/lmdbjava/ByteBufferProxyTest.java | 17 +- .../CursorIterableIntegerDupTest.java | 39 ++- .../CursorIterableIntegerKeyTest.java | 154 ++++++++---- .../java/org/lmdbjava/CursorIterableTest.java | 16 +- .../java/org/lmdbjava/DbiBuilderTest.java | 6 +- src/test/java/org/lmdbjava/DbiTest.java | 34 +-- .../java/org/lmdbjava/PutFlagSetTest.java | 225 ++++++++++-------- src/test/java/org/lmdbjava/TestUtils.java | 8 + 12 files changed, 351 insertions(+), 188 deletions(-) diff --git a/src/main/java/org/lmdbjava/AbstractFlagSet.java b/src/main/java/org/lmdbjava/AbstractFlagSet.java index 25aa328b..6e9729fb 100644 --- a/src/main/java/org/lmdbjava/AbstractFlagSet.java +++ b/src/main/java/org/lmdbjava/AbstractFlagSet.java @@ -325,6 +325,19 @@ public Builder setFlag(final E flag) { return this; } + /** + * Sets multiple flag in the builder. + * + * @param flags The flags to set in the builder. + * @return this builder instance. + */ + public Builder setFlags(final Collection flags) { + if (flags != null) { + enumSet.addAll(flags); + } + return this; + } + /** * Clears any flags already set in this {@link Builder} * diff --git a/src/main/java/org/lmdbjava/ByteBufferProxy.java b/src/main/java/org/lmdbjava/ByteBufferProxy.java index 5d5aa2ca..27ae375e 100644 --- a/src/main/java/org/lmdbjava/ByteBufferProxy.java +++ b/src/main/java/org/lmdbjava/ByteBufferProxy.java @@ -162,7 +162,9 @@ public static int compareLexicographically(final ByteBuffer o1, final ByteBuffer public static int compareAsIntegerKeys(final ByteBuffer o1, final ByteBuffer o2) { requireNonNull(o1); requireNonNull(o2); - // Both buffers should be same lenght according to LMDB API. + // Both buffers should be same length according to LMDB API. + // From the LMDB docs for MDB_INTEGER_KEY + // numeric keys in native byte order: either unsigned int or size_t. The keys must all be of the same size. final int len1 = o1.limit(); final int len2 = o2.limit(); if (len1 != len2) { diff --git a/src/main/java/org/lmdbjava/Cursor.java b/src/main/java/org/lmdbjava/Cursor.java index c1ac7374..127fffc0 100644 --- a/src/main/java/org/lmdbjava/Cursor.java +++ b/src/main/java/org/lmdbjava/Cursor.java @@ -472,7 +472,7 @@ public T reserve(final T key, final int size) { * Reserve space for data of the given size, but don't copy the given val. Instead, return a * pointer to the reserved space, which the caller can fill in later - before the next update * operation or the transaction ends. This saves an extra memcpy if the data is being generated - * later. LMDB does nothing else with this memory, the caller is expected to modify all of the + * later. LMDB does nothing else with this memory, the caller is expected to modify all the * space requested. * *

This flag must not be specified if the database was opened with MDB_DUPSORT diff --git a/src/main/java/org/lmdbjava/DbiBuilder.java b/src/main/java/org/lmdbjava/DbiBuilder.java index 2b4e6ad8..1bf2489d 100644 --- a/src/main/java/org/lmdbjava/DbiBuilder.java +++ b/src/main/java/org/lmdbjava/DbiBuilder.java @@ -80,6 +80,9 @@ public DbiBuilderStage2 withDbName(final byte[] name) { * Equivalent to passing null to * {@link DbiBuilder#withDbName(String)} or {@link DbiBuilder#withDbName(byte[])}. *

+ *

Note: The 'unnamed database' is used by LMDB to store the names of named databases, with + * the database name being the key. Use of the unnamed database is intended for simple applications + * with only one database.

* @return The next builder stage. */ public DbiBuilderStage2 withoutDbName() { @@ -323,7 +326,7 @@ public DbiBuilderStage3 withDbiFlags(final DbiFlagSet dbiFlagSet) { * {@link DbiBuilderStage3#withDbiFlags(Collection)} * or {@link DbiBuilderStage3#addDbiFlag(DbiFlags)}. * - * @param dbiFlag to open the database with. A null value is a no-op. + * @param dbiFlag to add to any existing flags. A null value is a no-op. * @return this builder instance. */ public DbiBuilderStage3 addDbiFlag(final DbiFlags dbiFlag) { @@ -331,6 +334,22 @@ public DbiBuilderStage3 addDbiFlag(final DbiFlags dbiFlag) { return this; } + /** + * Adds a dbiFlag to those flags already added to this builder by + * {@link DbiBuilderStage3#withDbiFlags(DbiFlags...)}, + * {@link DbiBuilderStage3#withDbiFlags(Collection)} + * or {@link DbiBuilderStage3#addDbiFlag(DbiFlags)}. + * + * @param dbiFlagSet to add to any existing flags. A null value is a no-op. + * @return this builder instance. + */ + public DbiBuilderStage3 addDbiFlags(final DbiFlagSet dbiFlagSet) { + if (dbiFlagSet != null) { + flagSetBuilder.setFlags(dbiFlagSet.getFlags()); + } + return this; + } + /** * Use the supplied transaction to open the {@link Dbi}. *

diff --git a/src/test/java/org/lmdbjava/ByteBufferProxyTest.java b/src/test/java/org/lmdbjava/ByteBufferProxyTest.java index c7d8333f..fb736f41 100644 --- a/src/test/java/org/lmdbjava/ByteBufferProxyTest.java +++ b/src/test/java/org/lmdbjava/ByteBufferProxyTest.java @@ -43,6 +43,7 @@ import java.nio.ByteOrder; import java.time.Duration; import java.time.Instant; +import java.util.Arrays; import java.util.Comparator; import java.util.HashSet; import java.util.LinkedHashMap; @@ -142,9 +143,9 @@ public void unsafeIsDefault() { } /** - * For 100 rounds of 1,000,000 comparisons - * compareAsIntegerKeys: PT0.267813487S - * compareLexicographically: PT0.644165235S + * For 100 rounds of 5,000,000 comparisons + * compareAsIntegerKeys: PT1.600525631S + * compareLexicographically: PT3.381935001S */ @Test public void comparatorPerformance() { @@ -153,7 +154,7 @@ public void comparatorPerformance() { final ByteBuffer buffer2 = ByteBuffer.allocateDirect(Long.BYTES); buffer1.limit(Long.BYTES); buffer2.limit(Long.BYTES); - final long[] values = random.longs(1_000_000).toArray(); + final long[] values = random.longs(5_000_000).toArray(); Instant time = Instant.now(); int x = 0; @@ -195,7 +196,13 @@ public void verifyComparators() { buffer2native.limit(Long.BYTES); buffer1be.limit(Long.BYTES); buffer2be.limit(Long.BYTES); - final long[] values = random.longs(10_000_000).toArray(); + final long[] values = random.longs() + .filter(i -> i >= 0) + .limit(5_000_000) + .toArray(); + System.out.println("stats: " + Arrays.stream(values) + .summaryStatistics() + .toString()); final LinkedHashMap> comparators = new LinkedHashMap<>(); comparators.put("compareAsIntegerKeys", ByteBufferProxy.AbstractByteBufferProxy::compareAsIntegerKeys); diff --git a/src/test/java/org/lmdbjava/CursorIterableIntegerDupTest.java b/src/test/java/org/lmdbjava/CursorIterableIntegerDupTest.java index 1acf4328..189aad24 100644 --- a/src/test/java/org/lmdbjava/CursorIterableIntegerDupTest.java +++ b/src/test/java/org/lmdbjava/CursorIterableIntegerDupTest.java @@ -50,6 +50,7 @@ import static org.lmdbjava.TestUtils.bb; import static org.lmdbjava.TestUtils.bbNative; import static org.lmdbjava.TestUtils.getNativeInt; +import static org.lmdbjava.TestUtils.getNativeIntOrLong; import com.google.common.primitives.UnsignedBytes; import java.io.File; @@ -71,6 +72,7 @@ import org.hamcrest.CoreMatchers; import org.hamcrest.Matchers; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Ignore; import org.junit.Rule; @@ -88,7 +90,10 @@ @RunWith(Parameterized.class) public final class CursorIterableIntegerDupTest { - private static final DbiFlagSet DBI_FLAGS = DbiFlagSet.of(MDB_CREATE, MDB_INTEGERDUP, MDB_DUPSORT); + private static final DbiFlagSet DBI_FLAGS = DbiFlagSet.of( + MDB_CREATE, + MDB_INTEGERDUP, + MDB_DUPSORT); private static final BufferProxy BUFFER_PROXY = ByteBufferProxy.PROXY_OPTIMAL; private static final List> INPUT_DATA; @@ -122,35 +127,47 @@ public final class CursorIterableIntegerDupTest { @Parameterized.Parameters(name = "{index}: dbi: {0}") public static Object[] data() { - final DbiFactory defaultComparator = new DbiFactory("defaultComparator", env -> + final DbiFactory defaultComparatorDb = new DbiFactory("defaultComparator", env -> env.buildDbi() .withDbName(DB_1) .withDefaultComparator() .withDbiFlags(DBI_FLAGS) .open()); - final DbiFactory nativeComparator = new DbiFactory("nativeComparator", env -> + final DbiFactory nativeComparatorDb = new DbiFactory("nativeComparator", env -> env.buildDbi() .withDbName(DB_2) .withNativeComparator() .withDbiFlags(DBI_FLAGS) .open()); - final DbiFactory callbackComparator = new DbiFactory("callbackComparator", env -> + final DbiFactory callbackComparatorDb = new DbiFactory("callbackComparator", env -> env.buildDbi() .withDbName(DB_3) - .withCallbackComparator(BUFFER_PROXY.getComparator(DBI_FLAGS)) + .withCallbackComparator(buildComparator()) .withDbiFlags(DBI_FLAGS) .open()); - final DbiFactory iteratorComparator = new DbiFactory("iteratorComparator", env -> + final DbiFactory iteratorComparatorDb = new DbiFactory("iteratorComparator", env -> env.buildDbi() .withDbName(DB_4) - .withIteratorComparator(BUFFER_PROXY.getComparator(DBI_FLAGS)) + .withIteratorComparator(buildComparator()) .withDbiFlags(DBI_FLAGS) .open()); return new Object[]{ - defaultComparator, - nativeComparator, - callbackComparator, - iteratorComparator}; + defaultComparatorDb, + nativeComparatorDb, + callbackComparatorDb, + iteratorComparatorDb}; + } + + private static Comparator buildComparator() { + final Comparator baseComparator = BUFFER_PROXY.getComparator(DBI_FLAGS); + return (o1, o2) -> { + if (o1.remaining() != o2.remaining()) { + // Make sure LMDB is always giving us consistent key lengths. + Assert.fail("o1: " + o1 + " " + getNativeIntOrLong(o1) + + ", o2: " + o2 + " " + getNativeIntOrLong(o2)); + } + return baseComparator.compare(o1, o2); + }; } @Before diff --git a/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java b/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java index be91ea4c..1e664fb1 100644 --- a/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java +++ b/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java @@ -51,6 +51,8 @@ import static org.lmdbjava.TestUtils.bb; import static org.lmdbjava.TestUtils.bbNative; import static org.lmdbjava.TestUtils.getNativeInt; +import static org.lmdbjava.TestUtils.getNativeIntOrLong; +import static org.lmdbjava.TestUtils.getNativeLong; import static org.lmdbjava.TestUtils.getString; import com.google.common.primitives.UnsignedBytes; @@ -71,6 +73,7 @@ import java.util.stream.Collectors; import org.hamcrest.Matchers; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -104,35 +107,49 @@ public final class CursorIterableIntegerKeyTest { @Parameterized.Parameters(name = "{index}: dbi: {0}") public static Object[] data() { - final DbiFactory defaultComparator = new DbiFactory("defaultComparator", env -> + final DbiFactory defaultComparatorDb = new DbiFactory("defaultComparator", env -> env.buildDbi() .withDbName(DB_1) .withDefaultComparator() .withDbiFlags(DBI_FLAGS) .open()); - final DbiFactory nativeComparator = new DbiFactory("nativeComparator", env -> + final DbiFactory nativeComparatorDb = new DbiFactory("nativeComparator", env -> env.buildDbi() .withDbName(DB_2) .withNativeComparator() .withDbiFlags(DBI_FLAGS) .open()); - final DbiFactory callbackComparator = new DbiFactory("callbackComparator", env -> + final Comparator comparator = buildComparator(); + + final DbiFactory callbackComparatorDb = new DbiFactory("callbackComparator", env -> env.buildDbi() .withDbName(DB_3) - .withCallbackComparator(BUFFER_PROXY.getComparator(DBI_FLAGS)) + .withCallbackComparator(comparator) .withDbiFlags(DBI_FLAGS) .open()); - final DbiFactory iteratorComparator = new DbiFactory("iteratorComparator", env -> + final DbiFactory iteratorComparatorDb = new DbiFactory("iteratorComparator", env -> env.buildDbi() .withDbName(DB_4) - .withIteratorComparator(BUFFER_PROXY.getComparator(DBI_FLAGS)) + .withIteratorComparator(comparator) .withDbiFlags(DBI_FLAGS) .open()); return new Object[]{ - defaultComparator, - nativeComparator, - callbackComparator, - iteratorComparator}; + defaultComparatorDb, + nativeComparatorDb, + callbackComparatorDb, + iteratorComparatorDb}; + } + + private static Comparator buildComparator() { + final Comparator baseComparator = BUFFER_PROXY.getComparator(DBI_FLAGS); + return (o1, o2) -> { + if (o1.remaining() != o2.remaining()) { + // Make sure LMDB is always giving us consistent key lengths. + Assert.fail("o1: " + o1 + " " + getNativeIntOrLong(o1) + + ", o2: " + o2 + " " + getNativeIntOrLong(o2)); + } + return baseComparator.compare(o1, o2); + }; } @Before @@ -162,13 +179,13 @@ public void testNumericOrderLong() { final Cursor c = dbi.openCursor(txn); long i = 1; while (true) { -// System.out.println("putting " + i); + System.out.println("putting " + i); c.put(bbNative(i), bb(i + "-long")); - final long i2 = i * 10; - if (i2 < i) { - // Overflowed - break; - } + final long i2 = i * 10; + if (i2 < i) { + // Overflowed + break; + } i = i2; } txn.commit(); @@ -178,8 +195,9 @@ public void testNumericOrderLong() { try (Txn txn = env.txnRead()) { try (CursorIterable iterable = dbi.iterate(txn)) { for (KeyVal keyVal : iterable) { + assertThat(keyVal.key().remaining(), is(Long.BYTES)); final String val = getString(keyVal.val()); - final long key = TestUtils.getNativeLong(keyVal.key()); + final long key = getNativeLong(keyVal.key()); entries.add(new AbstractMap.SimpleEntry<>(key, val)); // System.out.println(val); } @@ -208,7 +226,7 @@ public void testNumericOrderInt() { final Cursor c = dbi.openCursor(txn); int i = 1; while (true) { -// System.out.println("putting " + i); + System.out.println("putting " + i); c.put(bbNative(i), bb(i + "-int")); final int i2 = i * 10; if (i2 < i) { @@ -224,6 +242,7 @@ public void testNumericOrderInt() { try (Txn txn = env.txnRead()) { try (CursorIterable iterable = dbi.iterate(txn)) { for (KeyVal keyVal : iterable) { + assertThat(keyVal.key().remaining(), is(Integer.BYTES)); final String val = getString(keyVal.val()); final int key = TestUtils.getNativeInt(keyVal.key()); entries.add(new AbstractMap.SimpleEntry<>(key, val)); @@ -246,6 +265,41 @@ public void testNumericOrderInt() { } } + @Test + public void testIntegerKeyKeySize() { + final Dbi db = dbiFactory.factory.apply(env); + long maxIntAsLong = Integer.MAX_VALUE; + + try (Txn txn = env.txnWrite()) { + System.out.println("Flags: " + db.listFlags(txn)); + int val = 0; + db.put(txn, bbNative(0L), bb("val_" + ++val)); + db.put(txn, bbNative(10L), bb("val_" + ++val)); + db.put(txn, bbNative(maxIntAsLong - 1_111_111_111), bb("val_" + ++val)); + db.put(txn, bbNative(maxIntAsLong - 111_111_111), bb("val_" + ++val)); + db.put(txn, bbNative(maxIntAsLong - 111_111), bb("val_" + ++val)); + db.put(txn, bbNative(maxIntAsLong - 111), bb("val_" + ++val)); + db.put(txn, bbNative(maxIntAsLong + 111), bb("val_" + ++val)); + db.put(txn, bbNative(maxIntAsLong + 111_111), bb("val_" + ++val)); + db.put(txn, bbNative(maxIntAsLong + 111_111_111), bb("val_" + ++val)); + db.put(txn, bbNative(maxIntAsLong + 1_111_111_111), bb("val_" + ++val)); + db.put(txn, bbNative(Long.MAX_VALUE), bb("val_" + ++val)); + txn.commit(); + } + + try (Txn txn = env.txnRead()) { + try (CursorIterable iterable = db.iterate(txn)) { + for (KeyVal keyVal : iterable) { + final String val = getString(keyVal.val()); + final long key = getNativeLong(keyVal.key()); + final int remaining = keyVal.key().remaining(); + System.out.println("key: " + key + ", val: " + val + ", remaining: " + remaining); + } + } + } + + } + @Test public void allBackwardTest() { verify(allBackward(), 8, 6, 4, 2); @@ -365,11 +419,11 @@ public void iterate() { @Test(expected = IllegalStateException.class) public void iteratorOnlyReturnedOnce() { final Dbi db = getDb(); - try (Txn txn = env.txnRead(); - CursorIterable c = db.iterate(txn)) { - c.iterator(); // ok - c.iterator(); // fails - } + try (Txn txn = env.txnRead(); + CursorIterable c = db.iterate(txn)) { + c.iterator(); // ok + c.iterator(); // fails + } } @Test @@ -472,57 +526,57 @@ public void removeOddElements() { @Test(expected = Env.AlreadyClosedException.class) public void nextWithClosedEnvTest() { final Dbi db = getDb(); - try (Txn txn = env.txnRead()) { - try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { - final Iterator> c = ci.iterator(); + try (Txn txn = env.txnRead()) { + try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { + final Iterator> c = ci.iterator(); - env.close(); - c.next(); - } + env.close(); + c.next(); } + } } @Test(expected = Env.AlreadyClosedException.class) public void removeWithClosedEnvTest() { final Dbi db = getDb(); - try (Txn txn = env.txnWrite()) { - try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { - final Iterator> c = ci.iterator(); + try (Txn txn = env.txnWrite()) { + try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { + final Iterator> c = ci.iterator(); - final KeyVal keyVal = c.next(); - assertThat(keyVal, Matchers.notNullValue()); + final KeyVal keyVal = c.next(); + assertThat(keyVal, Matchers.notNullValue()); - env.close(); - c.remove(); - } + env.close(); + c.remove(); } + } } @Test(expected = Env.AlreadyClosedException.class) public void hasNextWithClosedEnvTest() { final Dbi db = getDb(); - try (Txn txn = env.txnRead()) { - try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { - final Iterator> c = ci.iterator(); + try (Txn txn = env.txnRead()) { + try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { + final Iterator> c = ci.iterator(); - env.close(); - c.hasNext(); - } + env.close(); + c.hasNext(); } + } } @Test(expected = Env.AlreadyClosedException.class) public void forEachRemainingWithClosedEnvTest() { final Dbi db = getDb(); - try (Txn txn = env.txnRead()) { - try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { - final Iterator> c = ci.iterator(); + try (Txn txn = env.txnRead()) { + try (CursorIterable ci = db.iterate(txn, KeyRange.all())) { + final Iterator> c = ci.iterator(); - env.close(); - c.forEachRemaining(keyVal -> { - }); - } + env.close(); + c.forEachRemaining(keyVal -> { + }); } + } } private void verify(final KeyRange range, final int... expected) { diff --git a/src/test/java/org/lmdbjava/CursorIterableTest.java b/src/test/java/org/lmdbjava/CursorIterableTest.java index 7bcbd851..a6b5d7d1 100644 --- a/src/test/java/org/lmdbjava/CursorIterableTest.java +++ b/src/test/java/org/lmdbjava/CursorIterableTest.java @@ -93,35 +93,35 @@ public final class CursorIterableTest { @Parameterized.Parameters(name = "{index}: dbi: {0}") public static Object[] data() { - final DbiFactory defaultComparator = new DbiFactory("defaultComparator", env -> + final DbiFactory defaultComparatorDb = new DbiFactory("defaultComparator", env -> env.buildDbi() .withDbName(DB_1) .withDefaultComparator() .withDbiFlags(DBI_FLAGS) .open()); - final DbiFactory nativeComparator = new DbiFactory("nativeComparator", env -> + final DbiFactory nativeComparatorDb = new DbiFactory("nativeComparator", env -> env.buildDbi() .withDbName(DB_2) .withNativeComparator() .withDbiFlags(DBI_FLAGS) .open()); - final DbiFactory callbackComparator = new DbiFactory("callbackComparator", env -> + final DbiFactory callbackComparatorDb = new DbiFactory("callbackComparator", env -> env.buildDbi() .withDbName(DB_3) .withCallbackComparator(BUFFER_PROXY.getComparator(DBI_FLAGS)) .withDbiFlags(DBI_FLAGS) .open()); - final DbiFactory iteratorComparator = new DbiFactory("iteratorComparator", env -> + final DbiFactory iteratorComparatorDb = new DbiFactory("iteratorComparator", env -> env.buildDbi() .withDbName(DB_4) .withIteratorComparator(BUFFER_PROXY.getComparator(DBI_FLAGS)) .withDbiFlags(DBI_FLAGS) .open()); return new Object[] { - defaultComparator, - nativeComparator, - callbackComparator, - iteratorComparator}; + defaultComparatorDb, + nativeComparatorDb, + callbackComparatorDb, + iteratorComparatorDb}; } @Before diff --git a/src/test/java/org/lmdbjava/DbiBuilderTest.java b/src/test/java/org/lmdbjava/DbiBuilderTest.java index 74fbd8f5..6f6a4f39 100644 --- a/src/test/java/org/lmdbjava/DbiBuilderTest.java +++ b/src/test/java/org/lmdbjava/DbiBuilderTest.java @@ -60,13 +60,12 @@ public void unnamed() { .withDefaultComparator() .withDbiFlags(DbiFlags.MDB_CREATE) .open(); - + assertThat(dbi.getName(), Matchers.nullValue()); + assertThat(dbi.getNameAsString(), Matchers.emptyString()); assertThat(env.getDbiNames().size(), Matchers.is(0)); - assertPutAndGet(dbi); } - @Test public void named() { final Dbi dbi = env.buildDbi() @@ -89,6 +88,7 @@ private void assertPutAndGet(Dbi dbi) { try (Txn readTxn = env.txnRead()) { final ByteBuffer byteBuffer = dbi.get(readTxn, bb(123)); + assertThat(byteBuffer, Matchers.notNullValue()); final int val = byteBuffer.getInt(); assertThat(val, Matchers.is(123_000)); } diff --git a/src/test/java/org/lmdbjava/DbiTest.java b/src/test/java/org/lmdbjava/DbiTest.java index 2209e614..962aacab 100644 --- a/src/test/java/org/lmdbjava/DbiTest.java +++ b/src/test/java/org/lmdbjava/DbiTest.java @@ -74,10 +74,13 @@ import org.lmdbjava.Env.MapFullException; import org.lmdbjava.LmdbNativeException.ConstantDerivedException; -/** Test {@link Dbi}. */ +/** + * Test {@link Dbi}. + */ public final class DbiTest { - @Rule public final TemporaryFolder tmp = new TemporaryFolder(); + @Rule + public final TemporaryFolder tmp = new TemporaryFolder(); private Env env; private Env envBa; @@ -105,10 +108,13 @@ public void before() throws IOException { } - @Test(expected = ConstantDerivedException.class) public void close() { - final Dbi db = env.openDbi(DB_1, MDB_CREATE); + final Dbi db = env.buildDbi() + .withDbName(DB_1) + .withDefaultComparator() + .addDbiFlag(MDB_CREATE) + .open(); db.put(bb(1), bb(42)); db.close(); db.put(bb(2), bb(42)); // error @@ -154,7 +160,7 @@ private void doCustomComparator( txn.commit(); } try (Txn txn = env.txnRead(); - CursorIterable ci = db.iterate(txn, atMost(serializer.apply(4)))) { + CursorIterable ci = db.iterate(txn, atMost(serializer.apply(4)))) { final Iterator> iter = ci.iterator(); assertThat(deserializer.applyAsInt(iter.next().key()), is(8)); assertThat(deserializer.applyAsInt(iter.next().key()), is(6)); @@ -186,7 +192,7 @@ public void doDbiWithComparatorThreadSafety( Supplier> comparatorSupplier, IntFunction serializer, ToIntFunction deserializer) { - final DbiFlags[] flags = new DbiFlags[] {MDB_CREATE, MDB_INTEGERKEY}; + final DbiFlags[] flags = new DbiFlags[]{MDB_CREATE, MDB_INTEGERKEY}; final Comparator c = comparatorSupplier.get(); final Dbi db = env.openDbi(DB_1, c, true, flags); @@ -212,7 +218,7 @@ public void doDbiWithComparatorThreadSafety( } try (Txn txn = env.txnRead(); - CursorIterable ci = db.iterate(txn)) { + CursorIterable ci = db.iterate(txn)) { final Iterator> iter = ci.iterator(); final List result = new ArrayList<>(); while (iter.hasNext()) { @@ -292,8 +298,8 @@ public void getName() { @Test public void getNamesWhenDbisPresent() { - final byte[] dbHello = new byte[] {'h', 'e', 'l', 'l', 'o'}; - final byte[] dbWorld = new byte[] {'w', 'o', 'r', 'l', 'd'}; + final byte[] dbHello = new byte[]{'h', 'e', 'l', 'l', 'o'}; + final byte[] dbWorld = new byte[]{'w', 'o', 'r', 'l', 'd'}; env.openDbi(dbHello, MDB_CREATE); env.openDbi(dbWorld, MDB_CREATE); final List dbiNames = env.getDbiNames(); @@ -369,11 +375,11 @@ public void putCommitGet() { public void putCommitGetByteArray() throws IOException { final File path = tmp.newFile(); try (Env envBa = - create(PROXY_BA) - .setMapSize(MEBIBYTES.toBytes(64)) - .setMaxReaders(1) - .setMaxDbs(2) - .open(path, MDB_NOSUBDIR)) { + create(PROXY_BA) + .setMapSize(MEBIBYTES.toBytes(64)) + .setMaxReaders(1) + .setMaxDbs(2) + .open(path, MDB_NOSUBDIR)) { final Dbi db = envBa.openDbi(DB_1, MDB_CREATE); try (Txn txn = envBa.txnWrite()) { db.put(txn, ba(5), ba(5)); diff --git a/src/test/java/org/lmdbjava/PutFlagSetTest.java b/src/test/java/org/lmdbjava/PutFlagSetTest.java index 3e402732..b8b44254 100644 --- a/src/test/java/org/lmdbjava/PutFlagSetTest.java +++ b/src/test/java/org/lmdbjava/PutFlagSetTest.java @@ -19,113 +19,150 @@ import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.MatcherAssert.assertThat; +import java.time.Duration; +import java.time.Instant; import java.util.Arrays; import java.util.HashSet; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import org.junit.Test; public class PutFlagSetTest { - @Test - public void testEmpty() { - final PutFlagSet putFlagSet = PutFlagSet.empty(); - assertThat( - putFlagSet.getMask(), - is(0)); - assertThat( - putFlagSet.size(), - is(0)); - assertThat( - putFlagSet.isEmpty(), - is(true)); - assertThat( - putFlagSet.isSet(PutFlags.MDB_MULTIPLE), - is(false)); - final PutFlagSet putFlagSet2 = PutFlagSet.builder() - .build(); - assertThat(putFlagSet, is(putFlagSet2)); - assertThat(putFlagSet, not(PutFlagSet.of(PutFlags.MDB_APPEND))); - assertThat(putFlagSet, not(PutFlagSet.of(PutFlags.MDB_APPEND, PutFlags.MDB_RESERVE))); - assertThat(putFlagSet, not(PutFlagSet.builder() - .setFlag(PutFlags.MDB_CURRENT) - .setFlag(PutFlags.MDB_MULTIPLE) - .build())); + @Test + public void testEmpty() { + final PutFlagSet putFlagSet = PutFlagSet.empty(); + assertThat( + putFlagSet.getMask(), + is(0)); + assertThat( + putFlagSet.size(), + is(0)); + assertThat( + putFlagSet.isEmpty(), + is(true)); + assertThat( + putFlagSet.isSet(PutFlags.MDB_MULTIPLE), + is(false)); + final PutFlagSet putFlagSet2 = PutFlagSet.builder() + .build(); + assertThat(putFlagSet, is(putFlagSet2)); + assertThat(putFlagSet, not(PutFlagSet.of(PutFlags.MDB_APPEND))); + assertThat(putFlagSet, not(PutFlagSet.of(PutFlags.MDB_APPEND, PutFlags.MDB_RESERVE))); + assertThat(putFlagSet, not(PutFlagSet.builder() + .setFlag(PutFlags.MDB_CURRENT) + .setFlag(PutFlags.MDB_MULTIPLE) + .build())); + } + + @Test + public void testOf() { + final PutFlags putFlag = PutFlags.MDB_APPEND; + final PutFlagSet putFlagSet = PutFlagSet.of(putFlag); + assertThat( + putFlagSet.getMask(), + is(MaskedFlag.mask(putFlag))); + assertThat( + putFlagSet.size(), + is(1)); + assertThat( + putFlagSet.isSet(PutFlags.MDB_MULTIPLE), + is(false)); + for (PutFlags flag : putFlagSet) { + assertThat( + putFlagSet.isSet(flag), + is(true)); } - @Test - public void testOf() { - final PutFlags putFlag = PutFlags.MDB_APPEND; - final PutFlagSet putFlagSet = PutFlagSet.of(putFlag); - assertThat( - putFlagSet.getMask(), - is(MaskedFlag.mask(putFlag))); - assertThat( - putFlagSet.size(), - is(1)); - assertThat( - putFlagSet.isSet(PutFlags.MDB_MULTIPLE), - is(false)); - for (PutFlags flag : putFlagSet) { - assertThat( - putFlagSet.isSet(flag), - is(true)); - } + final PutFlagSet putFlagSet2 = PutFlagSet.builder() + .setFlag(putFlag) + .build(); + assertThat(putFlagSet, is(putFlagSet2)); + } - final PutFlagSet putFlagSet2 = PutFlagSet.builder() - .setFlag(putFlag) - .build(); - assertThat(putFlagSet, is(putFlagSet2)); + @Test + public void testOf2() { + final PutFlags putFlag1 = PutFlags.MDB_APPEND; + final PutFlags putFlag2 = PutFlags.MDB_NOOVERWRITE; + final PutFlagSet putFlagSet = PutFlagSet.of(putFlag1, putFlag2); + assertThat( + putFlagSet.getMask(), + is(MaskedFlag.mask(putFlag1, putFlag2))); + assertThat( + putFlagSet.size(), + is(2)); + assertThat( + putFlagSet.isSet(PutFlags.MDB_MULTIPLE), + is(false)); + for (PutFlags flag : putFlagSet) { + assertThat( + putFlagSet.isSet(flag), + is(true)); } + } - @Test - public void testOf2() { - final PutFlags putFlag1 = PutFlags.MDB_APPEND; - final PutFlags putFlag2 = PutFlags.MDB_NOOVERWRITE; - final PutFlagSet putFlagSet = PutFlagSet.of(putFlag1, putFlag2); - assertThat( - putFlagSet.getMask(), - is(MaskedFlag.mask(putFlag1, putFlag2))); - assertThat( - putFlagSet.size(), - is(2)); - assertThat( - putFlagSet.isSet(PutFlags.MDB_MULTIPLE), - is(false)); - for (PutFlags flag : putFlagSet) { - assertThat( - putFlagSet.isSet(flag), - is(true)); - } + @Test + public void testBuilder() { + final PutFlags putFlag1 = PutFlags.MDB_APPEND; + final PutFlags putFlag2 = PutFlags.MDB_NOOVERWRITE; + final PutFlagSet putFlagSet = PutFlagSet.builder() + .setFlag(putFlag1) + .setFlag(putFlag2) + .build(); + assertThat( + putFlagSet.getMask(), + is(MaskedFlag.mask(putFlag1, putFlag2))); + assertThat( + putFlagSet.size(), + is(2)); + assertThat( + putFlagSet.isSet(PutFlags.MDB_MULTIPLE), + is(false)); + for (PutFlags flag : putFlagSet) { + assertThat( + putFlagSet.isSet(flag), + is(true)); } + final PutFlagSet putFlagSet2 = PutFlagSet.builder() + .withFlags(putFlag1, putFlag2) + .build(); + final PutFlagSet putFlagSet3 = PutFlagSet.builder() + .withFlags(new HashSet<>(Arrays.asList(putFlag1, putFlag2))) + .build(); + assertThat(putFlagSet, is(putFlagSet2)); + assertThat(putFlagSet, is(putFlagSet3)); + } - @Test - public void testBuilder() { - final PutFlags putFlag1 = PutFlags.MDB_APPEND; - final PutFlags putFlag2 = PutFlags.MDB_NOOVERWRITE; - final PutFlagSet putFlagSet = PutFlagSet.builder() - .setFlag(putFlag1) - .setFlag(putFlag2) - .build(); - assertThat( - putFlagSet.getMask(), - is(MaskedFlag.mask(putFlag1, putFlag2))); - assertThat( - putFlagSet.size(), - is(2)); - assertThat( - putFlagSet.isSet(PutFlags.MDB_MULTIPLE), - is(false)); - for (PutFlags flag : putFlagSet) { - assertThat( - putFlagSet.isSet(flag), - is(true)); + @Test + public void testAddFlagVsCheckPresence() { + + final int cnt = 10_000_000; + final int[] arr = new int[cnt]; + final List flagSets = IntStream.range(0, cnt) + .boxed() + .map(i -> PutFlagSet.of(PutFlags.MDB_APPEND, PutFlags.MDB_NOOVERWRITE, PutFlags.MDB_RESERVE)) + .collect(Collectors.toList()); + + Instant time; + for (int i = 0; i < 5; i++) { + time = Instant.now(); + for (int j = 0; j < flagSets.size(); j++) { + PutFlagSet flagSet = flagSets.get(j); + if (!flagSet.isSet(PutFlags.MDB_RESERVE)) { + throw new RuntimeException("Not set"); } - final PutFlagSet putFlagSet2 = PutFlagSet.builder() - .withFlags(putFlag1, putFlag2) - .build(); - final PutFlagSet putFlagSet3 = PutFlagSet.builder() - .withFlags(new HashSet<>(Arrays.asList(putFlag1, putFlag2))) - .build(); - assertThat(putFlagSet, is(putFlagSet2)); - assertThat(putFlagSet, is(putFlagSet3)); + arr[j] = flagSet.getMask(); + } + System.out.println("Check: " + Duration.between(time, Instant.now())); + + time = Instant.now(); + for (int j = 0; j < flagSets.size(); j++) { + PutFlagSet flagSet = flagSets.get(j); + final int mask = flagSet.getMaskWith(PutFlags.MDB_RESERVE); + arr[j] = mask; + } + System.out.println("Append:" + Duration.between(time, Instant.now())); } + } } diff --git a/src/test/java/org/lmdbjava/TestUtils.java b/src/test/java/org/lmdbjava/TestUtils.java index 511619fe..94ceb3a7 100644 --- a/src/test/java/org/lmdbjava/TestUtils.java +++ b/src/test/java/org/lmdbjava/TestUtils.java @@ -93,6 +93,14 @@ static long getNativeLong(final ByteBuffer bb) { return val; } + static long getNativeIntOrLong(final ByteBuffer bb) { + if (bb.remaining() == BYTES) { + return getNativeInt(bb); + } else { + return getNativeLong(bb); + } + } + static String getString(final ByteBuffer bb) { final String str = StandardCharsets.UTF_8.decode(bb) .toString(); From e2be6bf145a100fb2b8d40446dd3b70a54f51e2d Mon Sep 17 00:00:00 2001 From: at055612 <22818309+at055612@users.noreply.github.com> Date: Wed, 5 Nov 2025 13:06:57 +0000 Subject: [PATCH 20/21] Add integer key comparator tests --- .../java/org/lmdbjava/AbstractFlagSet.java | 2 - src/main/java/org/lmdbjava/ByteBufProxy.java | 74 +++- .../java/org/lmdbjava/ByteBufferProxy.java | 5 +- src/main/java/org/lmdbjava/DbiBuilder.java | 38 +- .../java/org/lmdbjava/DirectBufferProxy.java | 18 +- src/main/java/org/lmdbjava/Env.java | 185 +++++++-- .../org/lmdbjava/ByteBufferProxyTest.java | 17 +- .../lmdbjava/ComparatorIntegerKeyTest.java | 369 ++++++++++++++++++ .../java/org/lmdbjava/ComparatorTest.java | 10 + .../CursorIterableIntegerDupTest.java | 30 +- .../CursorIterableIntegerKeyTest.java | 32 +- .../org/lmdbjava/CursorIterablePerfTest.java | 12 +- .../java/org/lmdbjava/CursorIterableTest.java | 56 +-- .../java/org/lmdbjava/DbiBuilderTest.java | 125 +++++- src/test/java/org/lmdbjava/DbiTest.java | 2 +- src/test/java/org/lmdbjava/TestUtils.java | 35 ++ .../java/org/lmdbjava/TxnFlagSetTest.java | 111 +++--- 17 files changed, 906 insertions(+), 215 deletions(-) create mode 100644 src/test/java/org/lmdbjava/ComparatorIntegerKeyTest.java diff --git a/src/main/java/org/lmdbjava/AbstractFlagSet.java b/src/main/java/org/lmdbjava/AbstractFlagSet.java index 6e9729fb..058969de 100644 --- a/src/main/java/org/lmdbjava/AbstractFlagSet.java +++ b/src/main/java/org/lmdbjava/AbstractFlagSet.java @@ -26,8 +26,6 @@ /** * Encapsulates an immutable set of flags and the associated bit mask for the flags in the set. - * - * @param */ public abstract class AbstractFlagSet & MaskedFlag> implements FlagSet { diff --git a/src/main/java/org/lmdbjava/ByteBufProxy.java b/src/main/java/org/lmdbjava/ByteBufProxy.java index 319256fb..19d94392 100644 --- a/src/main/java/org/lmdbjava/ByteBufProxy.java +++ b/src/main/java/org/lmdbjava/ByteBufProxy.java @@ -23,6 +23,7 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.PooledByteBufAllocator; import java.lang.reflect.Field; +import java.nio.ByteOrder; import java.util.Comparator; import jnr.ffi.Pointer; @@ -44,13 +45,6 @@ public final class ByteBufProxy extends BufferProxy { private static final String FIELD_NAME_ADDRESS = "memoryAddress"; private static final String FIELD_NAME_LENGTH = "length"; private static final String NAME = "io.netty.buffer.PooledUnsafeDirectByteBuf"; - private static final Comparator comparator = - (o1, o2) -> { - requireNonNull(o1); - requireNonNull(o2); - - return o1.compareTo(o2); - }; private final long lengthOffset; private final long addressOffset; @@ -81,6 +75,66 @@ public ByteBufProxy(final PooledByteBufAllocator allocator) { } } + /** + * Lexicographically compare two buffers. + * + * @param o1 left operand (required) + * @param o2 right operand (required) + * @return as specified by {@link Comparable} interface + */ + public static int compareLexicographically(final ByteBuf o1, final ByteBuf o2) { + requireNonNull(o1); + requireNonNull(o2); + return o1.compareTo(o2); + } + + /** + * Buffer comparator specifically for 4/8 byte keys that are unsigned ints/longs, + * i.e. when using MDB_INTEGER_KEY/MDB_INTEGERDUP. Compares the buffers numerically. + * + * @param o1 left operand (required) + * @param o2 right operand (required) + * @return as specified by {@link Comparable} interface + */ + public static int compareAsIntegerKeys(final ByteBuf o1, final ByteBuf o2) { + requireNonNull(o1); + requireNonNull(o2); + // Both buffers should be same length according to LMDB API. + // From the LMDB docs for MDB_INTEGER_KEY + // numeric keys in native byte order: either unsigned int or size_t. The keys must all be of the same size. + final int len1 = o1.readableBytes(); + final int len2 = o2.readableBytes(); + if (len1 != len2) { + throw new RuntimeException("Length mismatch, len1: " + len1 + ", len2: " + len2 + + ". Lengths must be identical and either 4 or 8 bytes."); + } + if (len1 == 8) { + final long lw; + final long rw; + if (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN) { + lw = o1.readLongLE(); + rw = o2.readLongLE(); + } else { + lw = o1.readLong(); + rw = o2.readLong(); + } + return Long.compareUnsigned(lw, rw); + } else if (len1 == 4) { + final int lw; + final int rw; + if (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN) { + lw = o1.readIntLE(); + rw = o2.readIntLE(); + } else { + lw = o1.readInt(); + rw = o2.readInt(); + } + return Integer.compareUnsigned(lw, rw); + } else { + return compareLexicographically(o1, o2); + } + } + static Field findField(final String c, final String name) { Class clazz; try { @@ -115,7 +169,11 @@ protected ByteBuf allocate() { @Override public Comparator getComparator(final DbiFlagSet dbiFlagSet) { - return comparator; + if (dbiFlagSet.areAnySet(DbiFlagSet.INTEGER_KEY_FLAGS)) { + return ByteBufProxy::compareAsIntegerKeys; + } else { + return ByteBufProxy::compareLexicographically; + } } @Override diff --git a/src/main/java/org/lmdbjava/ByteBufferProxy.java b/src/main/java/org/lmdbjava/ByteBufferProxy.java index 27ae375e..c682bec8 100644 --- a/src/main/java/org/lmdbjava/ByteBufferProxy.java +++ b/src/main/java/org/lmdbjava/ByteBufferProxy.java @@ -60,6 +60,7 @@ public final class ByteBufferProxy { */ public static final BufferProxy PROXY_SAFE; + private static final ByteOrder NATIVE_ORDER = ByteOrder.nativeOrder(); static { PROXY_SAFE = new ReflectiveProxy(); @@ -172,8 +173,8 @@ public static int compareAsIntegerKeys(final ByteBuffer o1, final ByteBuffer o2) + ". Lengths must be identical and either 4 or 8 bytes."); } // Keys for MDB_INTEGER_KEY are written in native order so ensure we read them in that order - o1.order(ByteOrder.nativeOrder()); - o2.order(ByteOrder.nativeOrder()); + o1.order(NATIVE_ORDER); + o2.order(NATIVE_ORDER); // TODO it might be worth the DbiBuilder having a method to capture fixedKeyLength() or -1 // for variable length keys. This can be passed to getComparator(..) so it can return a // comparator that doesn't need to test the length every time. There may be other benefits diff --git a/src/main/java/org/lmdbjava/DbiBuilder.java b/src/main/java/org/lmdbjava/DbiBuilder.java index 1bf2489d..f94118a7 100644 --- a/src/main/java/org/lmdbjava/DbiBuilder.java +++ b/src/main/java/org/lmdbjava/DbiBuilder.java @@ -53,12 +53,12 @@ public class DbiBuilder { * (see also {@link DbiBuilder#withoutDbName()}) * @return The next builder stage. */ - public DbiBuilderStage2 withDbName(final String name) { + public DbiBuilderStage2 setDbName(final String name) { // Null name is allowed so no null check final byte[] nameBytes = name == null ? null : name.getBytes(Env.DEFAULT_NAME_CHARSET); - return withDbName(nameBytes); + return setDbName(nameBytes); } /** @@ -66,7 +66,7 @@ public DbiBuilderStage2 withDbName(final String name) { * @param name The name of the database in byte form. * @return The next builder stage. */ - public DbiBuilderStage2 withDbName(final byte[] name) { + public DbiBuilderStage2 setDbName(final byte[] name) { // Null name is allowed so no null check this.name = name; return new DbiBuilderStage2<>(this); @@ -78,7 +78,7 @@ public DbiBuilderStage2 withDbName(final byte[] name) { *

*

* Equivalent to passing null to - * {@link DbiBuilder#withDbName(String)} or {@link DbiBuilder#withDbName(byte[])}. + * {@link DbiBuilder#setDbName(String)} or {@link DbiBuilder#setDbName(byte[])}. *

*

Note: The 'unnamed database' is used by LMDB to store the names of named databases, with * the database name being the key. Use of the unnamed database is intended for simple applications @@ -86,7 +86,7 @@ public DbiBuilderStage2 withDbName(final byte[] name) { * @return The next builder stage. */ public DbiBuilderStage2 withoutDbName() { - return withDbName((byte[]) null); + return setDbName((byte[]) null); } @@ -254,8 +254,8 @@ private DbiBuilderStage3(DbiBuilderStage2 dbiBuilderStage2) { *

*

* Clears all flags currently set by previous calls to - * {@link DbiBuilderStage3#withDbiFlags(Collection)}, - * {@link DbiBuilderStage3#withDbiFlags(DbiFlags...)} + * {@link DbiBuilderStage3#setDbiFlags(Collection)}, + * {@link DbiBuilderStage3#setDbiFlags(DbiFlags...)} * or {@link DbiBuilderStage3#addDbiFlag(DbiFlags)}. *

* @@ -263,7 +263,7 @@ private DbiBuilderStage3(DbiBuilderStage2 dbiBuilderStage2) { * A null {@link Collection} will just clear all set flags. * Null items are ignored. */ - public DbiBuilderStage3 withDbiFlags(final Collection dbiFlags) { + public DbiBuilderStage3 setDbiFlags(final Collection dbiFlags) { flagSetBuilder.clear(); if (dbiFlags != null) { dbiFlags.stream() @@ -279,8 +279,8 @@ public DbiBuilderStage3 withDbiFlags(final Collection dbiFlags) { *

*

* Clears all flags currently set by previous calls to - * {@link DbiBuilderStage3#withDbiFlags(Collection)}, - * {@link DbiBuilderStage3#withDbiFlags(DbiFlags...)} + * {@link DbiBuilderStage3#setDbiFlags(Collection)}, + * {@link DbiBuilderStage3#setDbiFlags(DbiFlags...)} * or {@link DbiBuilderStage3#addDbiFlag(DbiFlags)}. *

* @@ -288,7 +288,7 @@ public DbiBuilderStage3 withDbiFlags(final Collection dbiFlags) { * A null array will just clear all set flags. * Null items are ignored. */ - public DbiBuilderStage3 withDbiFlags(final DbiFlags... dbiFlags) { + public DbiBuilderStage3 setDbiFlags(final DbiFlags... dbiFlags) { flagSetBuilder.clear(); if (dbiFlags != null) { Arrays.stream(dbiFlags) @@ -304,15 +304,15 @@ public DbiBuilderStage3 withDbiFlags(final DbiFlags... dbiFlags) { *

*

* Clears all flags currently set by previous calls to - * {@link DbiBuilderStage3#withDbiFlags(Collection)}, - * {@link DbiBuilderStage3#withDbiFlags(DbiFlags...)} + * {@link DbiBuilderStage3#setDbiFlags(Collection)}, + * {@link DbiBuilderStage3#setDbiFlags(DbiFlags...)} * or {@link DbiBuilderStage3#addDbiFlag(DbiFlags)}. *

* * @param dbiFlagSet to open the database with. * A null value will just clear all set flags. */ - public DbiBuilderStage3 withDbiFlags(final DbiFlagSet dbiFlagSet) { + public DbiBuilderStage3 setDbiFlags(final DbiFlagSet dbiFlagSet) { flagSetBuilder.clear(); if (dbiFlagSet != null) { this.flagSetBuilder.withFlags(dbiFlagSet.getFlags()); @@ -322,8 +322,8 @@ public DbiBuilderStage3 withDbiFlags(final DbiFlagSet dbiFlagSet) { /** * Adds a dbiFlag to those flags already added to this builder by - * {@link DbiBuilderStage3#withDbiFlags(DbiFlags...)}, - * {@link DbiBuilderStage3#withDbiFlags(Collection)} + * {@link DbiBuilderStage3#setDbiFlags(DbiFlags...)}, + * {@link DbiBuilderStage3#setDbiFlags(Collection)} * or {@link DbiBuilderStage3#addDbiFlag(DbiFlags)}. * * @param dbiFlag to add to any existing flags. A null value is a no-op. @@ -336,8 +336,8 @@ public DbiBuilderStage3 addDbiFlag(final DbiFlags dbiFlag) { /** * Adds a dbiFlag to those flags already added to this builder by - * {@link DbiBuilderStage3#withDbiFlags(DbiFlags...)}, - * {@link DbiBuilderStage3#withDbiFlags(Collection)} + * {@link DbiBuilderStage3#setDbiFlags(DbiFlags...)}, + * {@link DbiBuilderStage3#setDbiFlags(Collection)} * or {@link DbiBuilderStage3#addDbiFlag(DbiFlags)}. * * @param dbiFlagSet to add to any existing flags. A null value is a no-op. @@ -367,7 +367,7 @@ public DbiBuilderStage3 addDbiFlags(final DbiFlagSet dbiFlagSet) { * else it needs to be a read/write {@link Txn}. * @return this builder instance. */ - public DbiBuilderStage3 withTxn(final Txn txn) { + public DbiBuilderStage3 setTxn(final Txn txn) { this.txn = Objects.requireNonNull(txn); return this; } diff --git a/src/main/java/org/lmdbjava/DirectBufferProxy.java b/src/main/java/org/lmdbjava/DirectBufferProxy.java index 180eee0a..46bdfd22 100644 --- a/src/main/java/org/lmdbjava/DirectBufferProxy.java +++ b/src/main/java/org/lmdbjava/DirectBufferProxy.java @@ -22,6 +22,7 @@ import static org.lmdbjava.UnsafeAccess.UNSAFE; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.ArrayDeque; import java.util.Comparator; import jnr.ffi.Pointer; @@ -50,6 +51,8 @@ public final class DirectBufferProxy extends BufferProxy { private static final ThreadLocal> BUFFERS = withInitial(() -> new ArrayDeque<>(16)); + private static final ByteOrder NATIVE_ORDER = ByteOrder.nativeOrder(); + private DirectBufferProxy() {} /** @@ -110,16 +113,19 @@ public static int compareAsIntegerKeys(final DirectBuffer o1, final DirectBuffer + ". Lengths must be identical and either 4 or 8 bytes."); } if (len1 == 8) { - final long lw = o1.getLong(0, BIG_ENDIAN); - final long rw = o2.getLong(0, BIG_ENDIAN); + final long lw = o1.getLong(0, NATIVE_ORDER); + final long rw = o2.getLong(0, NATIVE_ORDER); return Long.compareUnsigned(lw, rw); } else if (len1 == 4) { - final int lw = o1.getInt(0, BIG_ENDIAN); - final int rw = o2.getInt(0, BIG_ENDIAN); + final int lw = o1.getInt(0, NATIVE_ORDER); + final int rw = o2.getInt(0, NATIVE_ORDER); return Integer.compareUnsigned(lw, rw); } else { - throw new RuntimeException("Unexpected length len1: " + len1 + ", len2: " + len2 - + ". Lengths must be identical and either 4 or 8 bytes."); + // size_t and int are likely to be 8bytes and 4bytes respectively on 64bit. + // If 32bit then would be 4/2 respectively. + // Short.compareUnsigned is not available in Java8. + // For now just fall back to our standard comparator + return compareLexicographically(o1, o2); } } diff --git a/src/main/java/org/lmdbjava/Env.java b/src/main/java/org/lmdbjava/Env.java index 8ea08ccb..b8003cbb 100644 --- a/src/main/java/org/lmdbjava/Env.java +++ b/src/main/java/org/lmdbjava/Env.java @@ -23,18 +23,20 @@ import static org.lmdbjava.EnvFlags.MDB_RDONLY_ENV; import static org.lmdbjava.Library.LIB; import static org.lmdbjava.Library.RUNTIME; -import static org.lmdbjava.MaskedFlag.isSet; -import static org.lmdbjava.MaskedFlag.mask; import static org.lmdbjava.ResultCodeMapper.checkRc; import java.io.File; import java.nio.ByteBuffer; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.Objects; import jnr.ffi.Pointer; import jnr.ffi.byref.IntByReference; import jnr.ffi.byref.PointerByReference; @@ -102,6 +104,8 @@ public static Builder create(final BufferProxy proxy) { } /** + * @deprecated Instead use {@link Env#create()} or {@link Env#create(BufferProxy)} + *

* Opens an environment with a single default database in 0664 mode using the {@link * ByteBufferProxy#PROXY_OPTIMAL}. * @@ -110,6 +114,7 @@ public static Builder create(final BufferProxy proxy) { * @param flags the flags for this new environment * @return env the environment (never null) */ + @Deprecated public static Env open(final File path, final int size, final EnvFlags... flags) { return new Builder<>(PROXY_OPTIMAL) .setMapSize(size * 1_024L * 1_024L) @@ -496,18 +501,6 @@ public Txn txn(final Txn parent) { return new Txn<>(this, parent, proxy, TxnFlagSet.EMPTY); } - /** - * Obtain a transaction with the requested parent and flags. - * - * @param parent parent transaction (may be null if no parent) - * @param flag applicable flag (eg for a reusable, read-only transaction) - * @return a transaction (never null) - */ - public Txn txn(final Txn parent, final TxnFlags flag) { - checkNotClosed(); - return new Txn<>(this, parent, proxy, flag); - } - /** * Obtain a transaction with the requested parent and flags. * @@ -616,6 +609,10 @@ public AlreadyOpenException() { } } + + // -------------------------------------------------------------------------------- + + /** * Builder for configuring and opening Env. * @@ -629,6 +626,8 @@ public static final class Builder { private int maxReaders = MAX_READERS_DEFAULT; private boolean opened; private final BufferProxy proxy; + private int mode = 0664; + private AbstractFlagSet.Builder flagSetBuilder = EnvFlagSet.builder(); Builder(final BufferProxy proxy) { requireNonNull(proxy); @@ -642,10 +641,49 @@ public static final class Builder { * @param mode Unix permissions to set on created files and semaphores * @param flags the flags for this new environment * @return an environment ready for use + * @deprecated Instead use {@link Builder#open(Path)}, {@link Builder#setFilePermissions(int)} + * and {@link Builder#setEnvFlags(EnvFlags...)}. */ + @Deprecated public Env open(final File path, final int mode, final EnvFlags... flags) { - // TODO Use EnvFlagSet and deprecate - // TODO Make setUnixPermissions(int) method on builder. + setFilePermissions(mode); + setEnvFlags(flags); + return open(requireNonNull(path).toPath()); + } + + /** + * Opens the environment. + * + * @param path file system destination + * @return an environment ready for use + * @deprecated Instead use {@link Builder#open(Path)} + */ + @Deprecated + public Env open(final File path) { + return open(requireNonNull(path).toPath()); + } + + /** + * Opens the environment with 0664 mode. + * + * @param path file system destination + * @param flags the flags for this new environment + * @return an environment ready for use + * @deprecated Instead use {@link Builder#open(Path)} and {@link Builder#setEnvFlags(EnvFlags...)}. + */ + @Deprecated + public Env open(final File path, final EnvFlags... flags) { + setEnvFlags(flags); + return open(requireNonNull(path).toPath()); + } + + /** + * Opens the environment. + * + * @param path file system destination + * @return an environment ready for use + */ + public Env open(final Path path) { requireNonNull(path); if (opened) { throw new AlreadyOpenException(); @@ -658,10 +696,10 @@ public Env open(final File path, final int mode, final EnvFlags... flags) { checkRc(LIB.mdb_env_set_mapsize(ptr, mapSize)); checkRc(LIB.mdb_env_set_maxdbs(ptr, maxDbs)); checkRc(LIB.mdb_env_set_maxreaders(ptr, maxReaders)); - final int flagsMask = mask(flags); - final boolean readOnly = isSet(flagsMask, MDB_RDONLY_ENV); - final boolean noSubDir = isSet(flagsMask, MDB_NOSUBDIR); - checkRc(LIB.mdb_env_open(ptr, path.getAbsolutePath(), flagsMask, mode)); + final EnvFlagSet flags = flagSetBuilder.build(); + final boolean readOnly = flags.isSet(MDB_RDONLY_ENV); + final boolean noSubDir = flags.isSet(MDB_NOSUBDIR); + checkRc(LIB.mdb_env_open(ptr, path.toAbsolutePath().toString(), flags.getMask(), mode)); return new Env<>(proxy, ptr, readOnly, noSubDir); } catch (final LmdbNativeException e) { LIB.mdb_env_close(ptr); @@ -670,19 +708,7 @@ public Env open(final File path, final int mode, final EnvFlags... flags) { } /** - * Opens the environment with 0664 mode. - * - * @param path file system destination - * @param flags the flags for this new environment - * @return an environment ready for use - */ - public Env open(final File path, final EnvFlags... flags) { - // TODO make constant - return open(path, 0664, flags); - } - - /** - * Sets the map size. + * Sets the map size in bytes. * * @param mapSize new limit in bytes * @return the builder @@ -725,6 +751,99 @@ public Builder setMaxReaders(final int readers) { this.maxReaders = readers; return this; } + + /** + * Sets the Unix file permissions to use on created files and semaphores, e.g. {@code 0664}. + * If this method is not called, the default of {@code 0664} will be used. + * + * @param mode Unix permissions to set on created files and semaphores + * @return the builder + */ + public Builder setFilePermissions(final int mode) { + if (opened) { + throw new AlreadyOpenException(); + } + this.mode = mode; + return this; + } + + /** + * Sets all the flags used to open this {@link Env}. + * + * @param envFlags The flags to use. + * Clears any existing flags. + * A null value results in no flags being set. + * @return this builder instance. + */ + public Builder setEnvFlags(final Collection envFlags) { + flagSetBuilder.clear(); + if (envFlags != null) { + envFlags.stream() + .filter(Objects::nonNull) + .forEach(envFlags::add); + } + return this; + } + + /** + * Sets all the flags used to open this {@link Env}. + * + * @param envFlags The flags to use. + * Clears any existing flags. + * A null value results in no flags being set. + * @return this builder instance. + */ + public Builder setEnvFlags(final EnvFlags... envFlags) { + flagSetBuilder.clear(); + if (envFlags != null) { + Arrays.stream(envFlags) + .filter(Objects::nonNull) + .forEach(this.flagSetBuilder::setFlag); + } + return this; + } + + /** + * Sets all the flags used to open this {@link Env}. + * + * @param envFlagSet The flags to use. + * Clears any existing flags. + * A null value results in no flags being set. + * @return this builder instance. + */ + public Builder setEnvFlags(final EnvFlagSet envFlagSet) { + flagSetBuilder.clear(); + if (envFlagSet != null) { + this.flagSetBuilder.withFlags(envFlagSet.getFlags()); + } + return this; + } + + /** + * Adds a single {@link EnvFlags} to any existing flags. + * + * @param dbiFlag The flag to add to any existing flags. + * A null value is a no-op. + * @return this builder instance. + */ + public Builder addEnvFlag(final EnvFlags dbiFlag) { + this.flagSetBuilder.setFlag(dbiFlag); + return this; + } + + /** + * Adds a set of {@link EnvFlags} to any existing flags. + * + * @param dbiFlagSet The set of flags to add to any existing flags. + * A null value is a no-op. + * @return this builder instance. + */ + public Builder addEnvFlags(final EnvFlagSet dbiFlagSet) { + if (dbiFlagSet != null) { + flagSetBuilder.setFlags(dbiFlagSet.getFlags()); + } + return this; + } } /** diff --git a/src/test/java/org/lmdbjava/ByteBufferProxyTest.java b/src/test/java/org/lmdbjava/ByteBufferProxyTest.java index 0c83f31e..dc034f7f 100644 --- a/src/test/java/org/lmdbjava/ByteBufferProxyTest.java +++ b/src/test/java/org/lmdbjava/ByteBufferProxyTest.java @@ -39,7 +39,6 @@ import java.nio.ByteOrder; import java.time.Duration; import java.time.Instant; -import java.util.Arrays; import java.util.Comparator; import java.util.HashSet; import java.util.LinkedHashMap; @@ -65,8 +64,14 @@ void buffersMustBeDirect() { () -> { FileUtil.useTempDir( dir -> { - try (Env env = create().setMaxReaders(1).open(dir.toFile())) { - final Dbi db = env.openDbi(DB_1, MDB_CREATE); + try (Env env = create() + .setMaxReaders(1) + .open(dir)) { + final Dbi db = env.buildDbi() + .setDbName(DB_1) + .withDefaultComparator() + .setDbiFlags(MDB_CREATE) + .open(); final ByteBuffer key = allocate(100); key.putInt(1).flip(); final ByteBuffer val = allocate(100); @@ -201,9 +206,9 @@ public void verifyComparators() { .filter(i -> i >= 0) .limit(5_000_000) .toArray(); - System.out.println("stats: " + Arrays.stream(values) - .summaryStatistics() - .toString()); +// System.out.println("stats: " + Arrays.stream(values) +// .summaryStatistics() +// .toString()); final LinkedHashMap> comparators = new LinkedHashMap<>(); comparators.put("compareAsIntegerKeys", ByteBufferProxy.AbstractByteBufferProxy::compareAsIntegerKeys); diff --git a/src/test/java/org/lmdbjava/ComparatorIntegerKeyTest.java b/src/test/java/org/lmdbjava/ComparatorIntegerKeyTest.java new file mode 100644 index 00000000..5b9e0761 --- /dev/null +++ b/src/test/java/org/lmdbjava/ComparatorIntegerKeyTest.java @@ -0,0 +1,369 @@ +/* + * Copyright © 2016-2025 The LmdbJava Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.lmdbjava; + +import static io.netty.buffer.PooledByteBufAllocator.DEFAULT; +import static org.assertj.core.api.Assertions.assertThat; +import static org.lmdbjava.ByteBufProxy.PROXY_NETTY; +import static org.lmdbjava.ByteBufferProxy.PROXY_OPTIMAL; +import static org.lmdbjava.ComparatorTest.ComparatorResult.EQUAL_TO; +import static org.lmdbjava.ComparatorTest.ComparatorResult.GREATER_THAN; +import static org.lmdbjava.ComparatorTest.ComparatorResult.LESS_THAN; +import static org.lmdbjava.ComparatorTest.ComparatorResult.get; +import static org.lmdbjava.DirectBufferProxy.PROXY_DB; + +import io.netty.buffer.ByteBuf; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Random; +import java.util.stream.Stream; +import org.agrona.DirectBuffer; +import org.agrona.concurrent.UnsafeBuffer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Tests comparator functions are consistent across buffers. + */ +public final class ComparatorIntegerKeyTest { + + static Stream comparatorProvider() { + return Stream.of( + Arguments.argumentSet("LongRunner", new DirectBufferRunner()), + Arguments.argumentSet("DirectBufferRunner", new DirectBufferRunner()), + Arguments.argumentSet("ByteBufferRunner", new ByteBufferRunner()), + Arguments.argumentSet("NettyRunner", new NettyRunner())); + } + + private static byte[] buffer(final int... bytes) { + final byte[] array = new byte[bytes.length]; + for (int i = 0; i < bytes.length; i++) { + array[i] = (byte) bytes[i]; + } + return array; + } + + @ParameterizedTest + @MethodSource("comparatorProvider") + void testLong(final ComparatorRunner comparator) { + + assertThat(get(comparator.compare(0L, 0L))).isEqualTo(EQUAL_TO); + assertThat(get(comparator.compare(Long.MAX_VALUE, Long.MAX_VALUE))).isEqualTo(EQUAL_TO); + + assertThat(get(comparator.compare(0L, 1L))).isEqualTo(LESS_THAN); + assertThat(get(comparator.compare(0L, Long.MAX_VALUE))).isEqualTo(LESS_THAN); + assertThat(get(comparator.compare(0L, 10L))).isEqualTo(LESS_THAN); + assertThat(get(comparator.compare(10L, 100L))).isEqualTo(LESS_THAN); + assertThat(get(comparator.compare(10L, 100L))).isEqualTo(LESS_THAN); + assertThat(get(comparator.compare(10L, 1000L))).isEqualTo(LESS_THAN); + + assertThat(get(comparator.compare(1L, 0L))).isEqualTo(GREATER_THAN); + assertThat(get(comparator.compare(Long.MAX_VALUE, 0L))).isEqualTo(GREATER_THAN); + } + + @ParameterizedTest + @MethodSource("comparatorProvider") + void testInt(final ComparatorRunner comparator) { + + assertThat(get(comparator.compare(0, 0))).isEqualTo(EQUAL_TO); + assertThat(get(comparator.compare(Integer.MAX_VALUE, Integer.MAX_VALUE))).isEqualTo(EQUAL_TO); + + assertThat(get(comparator.compare(0, 1))).isEqualTo(LESS_THAN); + assertThat(get(comparator.compare(0, Integer.MAX_VALUE))).isEqualTo(LESS_THAN); + assertThat(get(comparator.compare(0, 10))).isEqualTo(LESS_THAN); + assertThat(get(comparator.compare(10, 100))).isEqualTo(LESS_THAN); + assertThat(get(comparator.compare(10, 100))).isEqualTo(LESS_THAN); + assertThat(get(comparator.compare(10, 1000))).isEqualTo(LESS_THAN); + + assertThat(get(comparator.compare(1, 0))).isEqualTo(GREATER_THAN); + assertThat(get(comparator.compare(Integer.MAX_VALUE, 0))).isEqualTo(GREATER_THAN); + } + + @Test + void testRandomLong() { + final Random random = new Random(3239480); + final Map nameToRunnerMap = new LinkedHashMap<>(); + nameToRunnerMap.put("DirectBufferRunner", new DirectBufferRunner()); + nameToRunnerMap.put("ByteBufferRunner", new ByteBufferRunner()); + nameToRunnerMap.put("NettyRunner", new NettyRunner()); + + // 5mil random longs to compare + final long[] values = random.longs() + .filter(i -> i >= 0) + .limit(5_000_000) + .toArray(); + + for (int i = 1; i < values.length; i++) { + final long long1 = values[i - 1]; + final long long2 = values[i]; + for (Map.Entry entry : nameToRunnerMap.entrySet()) { + final String name = entry.getKey(); + final ComparatorRunner runner = entry.getValue(); + // Make sure the comparator under test gives the same outcome as just comparing two longs + final ComparatorTest.ComparatorResult result = get(runner.compare(long1, long2)); + final ComparatorTest.ComparatorResult expectedResult = get(Long.compare(long1, long2)); + + assertThat(result) + .withFailMessage(() -> "Compare mismatch for " + name + " - long1: " + long1 + + ", long2: " + long2 + + ", expected: " + expectedResult + + ", actual: " + result) + .isEqualTo(expectedResult); + + final ComparatorTest.ComparatorResult result2 = get(runner.compare(long2, long1)); + final ComparatorTest.ComparatorResult expectedResult2 = expectedResult.opposite(); + + assertThat(result) + .withFailMessage(() -> "Compare mismatch for " + name + " - long2: " + long2 + + ", long1: " + long1 + + ", expected2: " + expectedResult2 + + ", actual2: " + result2) + .isEqualTo(expectedResult); + } + } + } + + @Test + void testRandomInt() { + final Random random = new Random(3239480); + final Map nameToRunnerMap = new LinkedHashMap<>(); + nameToRunnerMap.put("DirectBufferRunner", new DirectBufferRunner()); + nameToRunnerMap.put("ByteBufferRunner", new ByteBufferRunner()); + nameToRunnerMap.put("NettyRunner", new NettyRunner()); + + // 5mil random ints to compare + final int[] values = random.ints() + .filter(i -> i >= 0) + .limit(5_000_000) + .toArray(); + + for (int i = 1; i < values.length; i++) { + final int int1 = values[i - 1]; + final int int2 = values[i]; + for (Map.Entry entry : nameToRunnerMap.entrySet()) { + final String name = entry.getKey(); + final ComparatorRunner runner = entry.getValue(); + // Make sure the comparator under test gives the same outcome as just comparing two ints + final ComparatorTest.ComparatorResult result = get(runner.compare(int1, int2)); + final ComparatorTest.ComparatorResult expectedResult = get(Integer.compare(int1, int2)); + + assertThat(result) + .withFailMessage(() -> "Compare mismatch for " + name + " - int1: " + int1 + + ", int2: " + int2 + + ", expected: " + expectedResult + + ", actual: " + result) + .isEqualTo(expectedResult); + + final ComparatorTest.ComparatorResult result2 = get(runner.compare(int2, int1)); + final ComparatorTest.ComparatorResult expectedResult2 = expectedResult.opposite(); + + assertThat(result) + .withFailMessage(() -> "Compare mismatch for " + name + " - int2: " + int2 + + ", int1: " + int1 + + ", expected2: " + expectedResult2 + + ", actual2: " + result2) + .isEqualTo(expectedResult); + } + } + } + + + // -------------------------------------------------------------------------------- + + + /** + * Tests {@link ByteBufferProxy}. + */ + private static final class ByteBufferRunner implements ComparatorRunner { + + private static final Comparator COMPARATOR = PROXY_OPTIMAL.getComparator(DbiFlags.MDB_INTEGERKEY); + + @Override + public int compare(long long1, long long2) { + // Convert arrays to buffers that are larger than the array, with + // limit set at the array length. One buffer bigger than the other. + ByteBuffer o1b = longToBuffer(long1, Long.BYTES * 3); + ByteBuffer o2b = longToBuffer(long2, Long.BYTES * 2); + final int result = COMPARATOR.compare(o1b, o2b); + + // Now swap which buffer is bigger + o1b = longToBuffer(long1, Long.BYTES * 2); + o2b = longToBuffer(long2, Long.BYTES * 3); + final int result2 = COMPARATOR.compare(o1b, o2b); + + assertThat(result2).isEqualTo(result); + + // Now try with buffers sized to the array. + o1b = longToBuffer(long1, Long.BYTES); + o2b = longToBuffer(long2, Long.BYTES); + final int result3 = COMPARATOR.compare(o1b, o2b); + + assertThat(result3).isEqualTo(result); + return result; + } + + @Override + public int compare(int int1, int int2) { + // Convert arrays to buffers that are larger than the array, with + // limit set at the array length. One buffer bigger than the other. + ByteBuffer o1b = intToBuffer(int1, Integer.BYTES * 3); + ByteBuffer o2b = intToBuffer(int2, Integer.BYTES * 2); + final int result = COMPARATOR.compare(o1b, o2b); + + // Now swap which buffer is bigger + o1b = intToBuffer(int1, Integer.BYTES * 2); + o2b = intToBuffer(int2, Integer.BYTES * 3); + final int result2 = COMPARATOR.compare(o1b, o2b); + + assertThat(result2).isEqualTo(result); + + // Now try with buffers sized to the array. + o1b = intToBuffer(int1, Integer.BYTES); + o2b = intToBuffer(int2, Integer.BYTES); + final int result3 = COMPARATOR.compare(o1b, o2b); + + assertThat(result3).isEqualTo(result); + return result; + } + + private ByteBuffer longToBuffer(final long val, final int bufferCapacity) { + final ByteBuffer byteBuffer = ByteBuffer.allocate(bufferCapacity); + byteBuffer.order(ByteOrder.nativeOrder()); + byteBuffer.putLong(0, val); + byteBuffer.limit(Long.BYTES); + byteBuffer.position(0); + return byteBuffer; + } + + private ByteBuffer intToBuffer(final int val, final int bufferCapacity) { + final ByteBuffer byteBuffer = ByteBuffer.allocate(bufferCapacity); + byteBuffer.order(ByteOrder.nativeOrder()); + byteBuffer.putInt(0, val); + byteBuffer.limit(Integer.BYTES); + byteBuffer.position(0); + return byteBuffer; + } + } + + + // -------------------------------------------------------------------------------- + + + /** + * Tests {@link DirectBufferProxy}. + */ + private static final class DirectBufferRunner implements ComparatorRunner { + private static final Comparator COMPARATOR = PROXY_DB.getComparator(DbiFlags.MDB_INTEGERKEY); + + @Override + public int compare(long long1, long long2) { + final UnsafeBuffer o1b = new UnsafeBuffer(new byte[Long.BYTES]); + final UnsafeBuffer o2b = new UnsafeBuffer(new byte[Long.BYTES]); + o1b.putLong(0, long1, ByteOrder.nativeOrder()); + o2b.putLong(0, long2, ByteOrder.nativeOrder()); + return COMPARATOR.compare(o1b, o2b); + } + + @Override + public int compare(int int1, int int2) { + final UnsafeBuffer o1b = new UnsafeBuffer(new byte[Integer.BYTES]); + final UnsafeBuffer o2b = new UnsafeBuffer(new byte[Integer.BYTES]); + o1b.putInt(0, int1, ByteOrder.nativeOrder()); + o2b.putInt(0, int2, ByteOrder.nativeOrder()); + return COMPARATOR.compare(o1b, o2b); + } + } + + /** + * Tests {@link ByteBufProxy}. + */ + private static final class NettyRunner implements ComparatorRunner { + + private static final Comparator COMPARATOR = PROXY_NETTY.getComparator(DbiFlags.MDB_INTEGERKEY); + + @Override + public int compare(long long1, long long2) { + final ByteBuf o1b = DEFAULT.directBuffer(Long.BYTES); + final ByteBuf o2b = DEFAULT.directBuffer(Long.BYTES); + if (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN) { + o1b.writeLongLE(long1); + o2b.writeLongLE(long2); + } else { + o1b.writeLong(long1); + o2b.writeLong(long2); + } + o1b.resetReaderIndex(); + o2b.resetReaderIndex(); + final int res = COMPARATOR.compare(o1b, o2b); + o1b.release(); + o2b.release(); + return res; + } + + @Override + public int compare(int int1, int int2) { + final ByteBuf o1b = DEFAULT.directBuffer(Integer.BYTES); + final ByteBuf o2b = DEFAULT.directBuffer(Integer.BYTES); + if (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN) { + o1b.writeIntLE(int1); + o2b.writeIntLE(int2); + } else { + o1b.writeInt(int1); + o2b.writeInt(int2); + } + o1b.resetReaderIndex(); + o2b.resetReaderIndex(); + final int res = COMPARATOR.compare(o1b, o2b); + o1b.release(); + o2b.release(); + return res; + } + } + + + // -------------------------------------------------------------------------------- + + + /** + * Interface that can test a {@link BufferProxy} compare method. + */ + private interface ComparatorRunner { + + /** + * Write the two longs to a buffer using native order and compare the resulting buffers. + * + * @param long1 lhs value + * @param long2 rhs value + * @return as per {@link Comparable} + */ + int compare(final long long1, final long long2); + + /** + * Write the two int to a buffer using native order and compare the resulting buffers. + * + * @param int1 lhs value + * @param int2 rhs value + * @return as per {@link Comparable} + */ + int compare(final int int1, final int int2); + } +} diff --git a/src/test/java/org/lmdbjava/ComparatorTest.java b/src/test/java/org/lmdbjava/ComparatorTest.java index bc7ddefe..1d4ed4c8 100644 --- a/src/test/java/org/lmdbjava/ComparatorTest.java +++ b/src/test/java/org/lmdbjava/ComparatorTest.java @@ -259,6 +259,16 @@ static ComparatorResult get(final int comparatorResult) { } return comparatorResult < 0 ? LESS_THAN : GREATER_THAN; } + + ComparatorResult opposite() { + if (this == LESS_THAN) { + return GREATER_THAN; + } else if (this == GREATER_THAN) { + return LESS_THAN; + } else { + return EQUAL_TO; + } + } } /** Interface that can test a {@link BufferProxy} compare method. */ diff --git a/src/test/java/org/lmdbjava/CursorIterableIntegerDupTest.java b/src/test/java/org/lmdbjava/CursorIterableIntegerDupTest.java index c55687c4..393065a3 100644 --- a/src/test/java/org/lmdbjava/CursorIterableIntegerDupTest.java +++ b/src/test/java/org/lmdbjava/CursorIterableIntegerDupTest.java @@ -191,11 +191,11 @@ private void populateDatabase(final Dbi dbi) { try (Txn txn = env.txnRead(); CursorIterable c = dbi.iterate(txn)) { - for (final KeyVal kv : c) { - System.out.print(getNativeInt(kv.key()) + " => " + kv.val().getInt()); - System.out.print(", "); - } - System.out.println(); +// for (final KeyVal kv : c) { +// System.out.print(getNativeInt(kv.key()) + " => " + kv.val().getInt()); +// System.out.print(", "); +// } +// System.out.println(); } } @@ -480,14 +480,14 @@ private void verify(final KeyRange range, .collect(Collectors.toList()); final List results = new ArrayList<>(); - System.out.println(rangeToString(range) + ", expected: " + expectedValues); +// System.out.println(rangeToString(range) + ", expected: " + expectedValues); try (Txn txn = env.txnRead(); CursorIterable c = dbi.iterate(txn, range)) { for (final KeyVal kv : c) { final int key = getNativeInt(kv.key()); final int val = kv.val().getInt(); - System.out.println(key + " => " + val); +// System.out.println(key + " => " + val); results.add(val); assertThat(val).satisfiesAnyOf( v -> assertThat(v).isEqualTo((key * 10) + 1), @@ -544,27 +544,27 @@ public Stream provideArguments(ParameterDeclarations parame ExtensionContext context) throws Exception { final DbiFactory defaultComparatorDb = new DbiFactory("defaultComparator", env -> env.buildDbi() - .withDbName(DB_1) + .setDbName(DB_1) .withDefaultComparator() - .withDbiFlags(DBI_FLAGS) + .setDbiFlags(DBI_FLAGS) .open()); final DbiFactory nativeComparatorDb = new DbiFactory("nativeComparator", env -> env.buildDbi() - .withDbName(DB_2) + .setDbName(DB_2) .withNativeComparator() - .withDbiFlags(DBI_FLAGS) + .setDbiFlags(DBI_FLAGS) .open()); final DbiFactory callbackComparatorDb = new DbiFactory("callbackComparator", env -> env.buildDbi() - .withDbName(DB_3) + .setDbName(DB_3) .withCallbackComparator(buildComparator()) - .withDbiFlags(DBI_FLAGS) + .setDbiFlags(DBI_FLAGS) .open()); final DbiFactory iteratorComparatorDb = new DbiFactory("iteratorComparator", env -> env.buildDbi() - .withDbName(DB_4) + .setDbName(DB_4) .withIteratorComparator(buildComparator()) - .withDbiFlags(DBI_FLAGS) + .setDbiFlags(DBI_FLAGS) .open()); return Stream.of( defaultComparatorDb, diff --git a/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java b/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java index 21d34d30..aa23235a 100644 --- a/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java +++ b/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java @@ -20,7 +20,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.lmdbjava.DbiFlags.MDB_CREATE; import static org.lmdbjava.DbiFlags.MDB_INTEGERKEY; -import static org.lmdbjava.Env.create; import static org.lmdbjava.EnvFlags.MDB_NOSUBDIR; import static org.lmdbjava.KeyRange.all; import static org.lmdbjava.KeyRange.allBackward; @@ -45,7 +44,6 @@ import static org.lmdbjava.TestUtils.DB_2; import static org.lmdbjava.TestUtils.DB_3; import static org.lmdbjava.TestUtils.DB_4; -import static org.lmdbjava.TestUtils.POSIX_MODE; import static org.lmdbjava.TestUtils.bb; import static org.lmdbjava.TestUtils.bbNative; import static org.lmdbjava.TestUtils.getNativeInt; @@ -101,16 +99,16 @@ public final class CursorIterableIntegerKeyTest { @Parameter public DbiFactory dbiFactory; - @BeforeEach public void before() throws IOException { file = FileUtil.createTempFile(); final BufferProxy bufferProxy = ByteBufferProxy.PROXY_OPTIMAL; - env = create(bufferProxy) + env = Env.create(bufferProxy) .setMapSize(KIBIBYTES.toBytes(256)) .setMaxReaders(1) .setMaxDbs(3) - .open(file.toFile(), POSIX_MODE, MDB_NOSUBDIR); + .setEnvFlags(MDB_NOSUBDIR) + .open(file); populateTestDataList(); } @@ -129,7 +127,7 @@ public void testNumericOrderLong() { final Cursor c = dbi.openCursor(txn); long i = 1; while (true) { - System.out.println("putting " + i); +// System.out.println("putting " + i); c.put(bbNative(i), bb(i + "-long")); final long i2 = i * 10; if (i2 < i) { @@ -176,7 +174,7 @@ public void testNumericOrderInt() { final Cursor c = dbi.openCursor(txn); int i = 1; while (true) { - System.out.println("putting " + i); +// System.out.println("putting " + i); c.put(bbNative(i), bb(i + "-int")); final int i2 = i * 10; if (i2 < i) { @@ -221,7 +219,7 @@ public void testIntegerKeyKeySize() { long maxIntAsLong = Integer.MAX_VALUE; try (Txn txn = env.txnWrite()) { - System.out.println("Flags: " + db.listFlags(txn)); +// System.out.println("Flags: " + db.listFlags(txn)); int val = 0; db.put(txn, bbNative(0L), bb("val_" + ++val)); db.put(txn, bbNative(10L), bb("val_" + ++val)); @@ -243,7 +241,7 @@ public void testIntegerKeyKeySize() { final String val = getString(keyVal.val()); final long key = getNativeLong(keyVal.key()); final int remaining = keyVal.key().remaining(); - System.out.println("key: " + key + ", val: " + val + ", remaining: " + remaining); +// System.out.println("key: " + key + ", val: " + val + ", remaining: " + remaining); } } } @@ -603,29 +601,29 @@ public Stream provideArguments(ParameterDeclarations parame ExtensionContext context) throws Exception { final DbiFactory defaultComparatorDb = new DbiFactory("defaultComparator", env -> env.buildDbi() - .withDbName(DB_1) + .setDbName(DB_1) .withDefaultComparator() - .withDbiFlags(DBI_FLAGS) + .setDbiFlags(DBI_FLAGS) .open()); final DbiFactory nativeComparatorDb = new DbiFactory("nativeComparator", env -> env.buildDbi() - .withDbName(DB_2) + .setDbName(DB_2) .withNativeComparator() - .withDbiFlags(DBI_FLAGS) + .setDbiFlags(DBI_FLAGS) .open()); final Comparator comparator = buildComparator(); final DbiFactory callbackComparatorDb = new DbiFactory("callbackComparator", env -> env.buildDbi() - .withDbName(DB_3) + .setDbName(DB_3) .withCallbackComparator(comparator) - .withDbiFlags(DBI_FLAGS) + .setDbiFlags(DBI_FLAGS) .open()); final DbiFactory iteratorComparatorDb = new DbiFactory("iteratorComparator", env -> env.buildDbi() - .withDbName(DB_4) + .setDbName(DB_4) .withIteratorComparator(comparator) - .withDbiFlags(DBI_FLAGS) + .setDbiFlags(DBI_FLAGS) .open()); return Stream.of( defaultComparatorDb, diff --git a/src/test/java/org/lmdbjava/CursorIterablePerfTest.java b/src/test/java/org/lmdbjava/CursorIterablePerfTest.java index 6bf1ee71..008695fd 100644 --- a/src/test/java/org/lmdbjava/CursorIterablePerfTest.java +++ b/src/test/java/org/lmdbjava/CursorIterablePerfTest.java @@ -65,22 +65,22 @@ public void before() throws IOException { final DbiFlagSet dbiFlagSet = MDB_CREATE; // Use a java comparator for start/stop keys only dbJavaComparator = env.buildDbi() - .withDbName("JavaComparator") + .setDbName("JavaComparator") .withDefaultComparator() - .withDbiFlags(dbiFlagSet) + .setDbiFlags(dbiFlagSet) .open(); // Use LMDB comparator for start/stop keys dbLmdbComparator = env.buildDbi() - .withDbName("LmdbComparator") + .setDbName("LmdbComparator") .withNativeComparator() - .withDbiFlags(dbiFlagSet) + .setDbiFlags(dbiFlagSet) .open(); // Use a java comparator for start/stop keys and as a callback comparator dbCallbackComparator = env.buildDbi() - .withDbName("CallBackComparator") + .setDbName("CallBackComparator") .withCallbackComparator(bufferProxy.getComparator(dbiFlagSet)) - .withDbiFlags(dbiFlagSet) + .setDbiFlags(dbiFlagSet) .open(); dbs.add(dbJavaComparator); diff --git a/src/test/java/org/lmdbjava/CursorIterableTest.java b/src/test/java/org/lmdbjava/CursorIterableTest.java index 30894a9c..0286fa88 100644 --- a/src/test/java/org/lmdbjava/CursorIterableTest.java +++ b/src/test/java/org/lmdbjava/CursorIterableTest.java @@ -46,7 +46,6 @@ import static org.lmdbjava.TestUtils.DB_2; import static org.lmdbjava.TestUtils.DB_3; import static org.lmdbjava.TestUtils.DB_4; -import static org.lmdbjava.TestUtils.POSIX_MODE; import static org.lmdbjava.TestUtils.bb; import com.google.common.primitives.UnsignedBytes; @@ -95,42 +94,6 @@ public final class CursorIterableTest { @Parameter public DbiFactory dbiFactory; -// ArgumentsSource - -// @Parameterized.Parameters(name = "{index}: dbi: {0}") - -// public static Object[] data() { -// final DbiFactory defaultComparatorDb = new DbiFactory("defaultComparator", env -> -// env.buildDbi() -// .withDbName(DB_1) -// .withDefaultComparator() -// .withDbiFlags(DBI_FLAGS) -// .open()); -// final DbiFactory nativeComparatorDb = new DbiFactory("nativeComparator", env -> -// env.buildDbi() -// .withDbName(DB_2) -// .withNativeComparator() -// .withDbiFlags(DBI_FLAGS) -// .open()); -// final DbiFactory callbackComparatorDb = new DbiFactory("callbackComparator", env -> -// env.buildDbi() -// .withDbName(DB_3) -// .withCallbackComparator(BUFFER_PROXY.getComparator(DBI_FLAGS)) -// .withDbiFlags(DBI_FLAGS) -// .open()); -// final DbiFactory iteratorComparatorDb = new DbiFactory("iteratorComparator", env -> -// env.buildDbi() -// .withDbName(DB_4) -// .withIteratorComparator(BUFFER_PROXY.getComparator(DBI_FLAGS)) -// .withDbiFlags(DBI_FLAGS) -// .open()); -// return new Object[]{ -// defaultComparatorDb, -// nativeComparatorDb, -// callbackComparatorDb, -// iteratorComparatorDb}; -// } - @BeforeEach void beforeEach() { file = FileUtil.createTempFile(); @@ -139,7 +102,8 @@ void beforeEach() { .setMapSize(KIBIBYTES.toBytes(256)) .setMaxReaders(1) .setMaxDbs(3) - .open(file.toFile(), POSIX_MODE, MDB_NOSUBDIR); + .setEnvFlags(MDB_NOSUBDIR) + .open(file); populateTestDataList(); } @@ -571,27 +535,27 @@ public Stream provideArguments(ParameterDeclarations parame ExtensionContext context) throws Exception { final DbiFactory defaultComparatorDb = new DbiFactory("defaultComparator", env -> env.buildDbi() - .withDbName(DB_1) + .setDbName(DB_1) .withDefaultComparator() - .withDbiFlags(DBI_FLAGS) + .setDbiFlags(DBI_FLAGS) .open()); final DbiFactory nativeComparatorDb = new DbiFactory("nativeComparator", env -> env.buildDbi() - .withDbName(DB_2) + .setDbName(DB_2) .withNativeComparator() - .withDbiFlags(DBI_FLAGS) + .setDbiFlags(DBI_FLAGS) .open()); final DbiFactory callbackComparatorDb = new DbiFactory("callbackComparator", env -> env.buildDbi() - .withDbName(DB_3) + .setDbName(DB_3) .withCallbackComparator(BUFFER_PROXY.getComparator(DBI_FLAGS)) - .withDbiFlags(DBI_FLAGS) + .setDbiFlags(DBI_FLAGS) .open()); final DbiFactory iteratorComparatorDb = new DbiFactory("iteratorComparator", env -> env.buildDbi() - .withDbName(DB_4) + .setDbName(DB_4) .withIteratorComparator(BUFFER_PROXY.getComparator(DBI_FLAGS)) - .withDbiFlags(DBI_FLAGS) + .setDbiFlags(DBI_FLAGS) .open()); return Stream.of( defaultComparatorDb, diff --git a/src/test/java/org/lmdbjava/DbiBuilderTest.java b/src/test/java/org/lmdbjava/DbiBuilderTest.java index fa0a4792..25e622bf 100644 --- a/src/test/java/org/lmdbjava/DbiBuilderTest.java +++ b/src/test/java/org/lmdbjava/DbiBuilderTest.java @@ -20,11 +20,15 @@ import static org.lmdbjava.Env.create; import static org.lmdbjava.EnvFlags.MDB_NOSUBDIR; import static org.lmdbjava.TestUtils.bb; +import static org.lmdbjava.TestUtils.getString; -import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -35,18 +39,20 @@ public class DbiBuilderTest { private Env env; @BeforeEach - public void before() throws IOException { + public void before() { file = FileUtil.createTempFile(); env = create() .setMapSize(MEBIBYTES.toBytes(64)) .setMaxReaders(2) .setMaxDbs(2) - .open(file.toFile(), MDB_NOSUBDIR); + .setEnvFlags(MDB_NOSUBDIR) + .open(file); } @AfterEach public void after() { env.close(); + FileUtil.delete(file); } @Test @@ -54,7 +60,7 @@ public void unnamed() { final Dbi dbi = env.buildDbi() .withoutDbName() .withDefaultComparator() - .withDbiFlags(DbiFlags.MDB_CREATE) + .setDbiFlags(DbiFlags.MDB_CREATE) .open(); assertThat(dbi.getName()).isNull(); assertThat(dbi.getNameAsString()).isEmpty(); @@ -65,15 +71,120 @@ public void unnamed() { @Test public void named() { final Dbi dbi = env.buildDbi() - .withDbName("foo") + .setDbName("foo") .withDefaultComparator() - .withDbiFlags(DbiFlags.MDB_CREATE) + .setDbiFlags(DbiFlags.MDB_CREATE) .open(); assertPutAndGet(dbi); assertThat(env.getDbiNames()).hasSize(1); - assertThat(new String(env.getDbiNames().get(0), StandardCharsets.UTF_8)).isEqualTo("foo"); + assertThat(new String(env.getDbiNames().get(0), StandardCharsets.UTF_8)) + .isEqualTo("foo"); + assertThat(dbi.getNameAsString()) + .isEqualTo("foo"); + assertThat(dbi.getNameAsString(StandardCharsets.UTF_8)) + .isEqualTo("foo"); + } + + @Test + public void named2() { + final Dbi dbi = env.buildDbi() + .setDbName("foo".getBytes(StandardCharsets.US_ASCII)) + .withDefaultComparator() + .setDbiFlags(DbiFlags.MDB_CREATE) + .open(); + + assertPutAndGet(dbi); + + assertThat(env.getDbiNames()).hasSize(1); + assertThat(new String(env.getDbiNames().get(0), StandardCharsets.US_ASCII)) + .isEqualTo("foo"); + assertThat(dbi.getNameAsString()) + .isEqualTo("foo"); + assertThat(dbi.getNameAsString(StandardCharsets.US_ASCII)) + .isEqualTo("foo"); + } + + @Test + public void nativeComparator() { + final Dbi dbi = env.buildDbi() + .setDbName("foo") + .withNativeComparator() + .addDbiFlags(DbiFlags.MDB_CREATE) + .open(); + + assertPutAndGet(dbi); + assertThat(env.getDbiNames()).hasSize(1); + } + + @Test + public void callback() { + final Comparator proxyOptimal = ByteBufferProxy.PROXY_OPTIMAL.getComparator(); + // Compare on key length, falling back to default + final Comparator comparator = (o1, o2) -> { + final int res = Integer.compare(o1.remaining(), o2.remaining()); + if (res == 0) { + return proxyOptimal.compare(o1, o2); + } else { + return res; + } + }; + + final Dbi dbi = env.buildDbi() + .setDbName("foo") + .withCallbackComparator(comparator) + .addDbiFlags(DbiFlags.MDB_CREATE) + .open(); + + TestUtils.doWithWriteTxn(env, txn -> { + dbi.put(txn, bb("fox"), bb("val_1")); + dbi.put(txn, bb("rabbit"), bb("val_2")); + dbi.put(txn, bb("deer"), bb("val_3")); + dbi.put(txn, bb("badger"), bb("val_4")); + txn.commit(); + }); + + final List keys = new ArrayList<>(); + TestUtils.doWithReadTxn(env, txn -> { + try (CursorIterable cursorIterable = dbi.iterate(txn)) { + final Iterator> iterator = cursorIterable.iterator(); + iterator.forEachRemaining(keyVal -> { + keys.add(getString(keyVal.key())); + }); + } + }); + assertThat(keys).containsExactly( + "fox", + "deer", + "badger", + "rabbit"); + } + + @Test + public void flags() { + final Dbi dbi = env.buildDbi() + .setDbName("foo") + .withDefaultComparator() + .setDbiFlags(DbiFlags.MDB_DUPSORT, DbiFlags.MDB_DUPFIXED) // Will get overwritten + .setDbiFlags() // clear them + .addDbiFlags(DbiFlags.MDB_CREATE) // Not a dbi flag as far as lmdb is concerned. + .addDbiFlags(DbiFlags.MDB_INTEGERKEY) + .addDbiFlags(DbiFlags.MDB_REVERSEKEY) + .open(); + + assertPutAndGet(dbi); + + assertThat(env.getDbiNames()).hasSize(1); + assertThat(new String(env.getDbiNames().get(0), StandardCharsets.UTF_8)) + .isEqualTo("foo"); + + TestUtils.doWithReadTxn(env, readTxn -> { + assertThat(dbi.listFlags(readTxn)) + .containsExactlyInAnyOrder( + DbiFlags.MDB_INTEGERKEY, + DbiFlags.MDB_REVERSEKEY); + }); } private void assertPutAndGet(Dbi dbi) { diff --git a/src/test/java/org/lmdbjava/DbiTest.java b/src/test/java/org/lmdbjava/DbiTest.java index c369c193..7302018c 100644 --- a/src/test/java/org/lmdbjava/DbiTest.java +++ b/src/test/java/org/lmdbjava/DbiTest.java @@ -110,7 +110,7 @@ void close() { assertThatThrownBy( () -> { final Dbi db = env.buildDbi() - .withDbName(DB_1) + .setDbName(DB_1) .withDefaultComparator() .addDbiFlag(MDB_CREATE) .open(); diff --git a/src/test/java/org/lmdbjava/TestUtils.java b/src/test/java/org/lmdbjava/TestUtils.java index 94ceb3a7..68d988d1 100644 --- a/src/test/java/org/lmdbjava/TestUtils.java +++ b/src/test/java/org/lmdbjava/TestUtils.java @@ -25,6 +25,9 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; +import java.util.Objects; +import java.util.function.Consumer; +import java.util.function.Function; import org.agrona.MutableDirectBuffer; import org.agrona.concurrent.UnsafeBuffer; @@ -133,4 +136,36 @@ static ByteBuf nb(final int value) { b.writeInt(value); return b; } + + static void doWithReadTxn(final Env env, final Consumer> work) { + Objects.requireNonNull(env); + Objects.requireNonNull(work); + try (Txn readTxn = env.txnRead()) { + work.accept(readTxn); + } + } + + static R getWithReadTxn(final Env env, final Function, R> work) { + Objects.requireNonNull(env); + Objects.requireNonNull(work); + try (Txn readTxn = env.txnRead()) { + return work.apply(readTxn); + } + } + + static void doWithWriteTxn(final Env env, final Consumer> work) { + Objects.requireNonNull(env); + Objects.requireNonNull(work); + try (Txn readTxn = env.txnWrite()) { + work.accept(readTxn); + } + } + + static R getWithWriteTxn(final Env env, final Function, R> work) { + Objects.requireNonNull(env); + Objects.requireNonNull(work); + try (Txn readTxn = env.txnWrite()) { + return work.apply(readTxn); + } + } } diff --git a/src/test/java/org/lmdbjava/TxnFlagSetTest.java b/src/test/java/org/lmdbjava/TxnFlagSetTest.java index bdaf9320..58f75aa6 100644 --- a/src/test/java/org/lmdbjava/TxnFlagSetTest.java +++ b/src/test/java/org/lmdbjava/TxnFlagSetTest.java @@ -23,57 +23,74 @@ public class TxnFlagSetTest { - @Test - public void testEmpty() { - final TxnFlagSet txnFlagSet = TxnFlagSet.empty(); - assertThat(txnFlagSet.getMask()).isEqualTo(0); - assertThat(txnFlagSet.size()).isEqualTo(0); - assertThat(txnFlagSet.isEmpty()).isEqualTo(true); - assertThat(txnFlagSet.isSet(TxnFlags.MDB_RDONLY_TXN)).isEqualTo(false); - final TxnFlagSet txnFlagSet2 = TxnFlagSet.builder() - .build(); - assertThat(txnFlagSet).isEqualTo(txnFlagSet2); - assertThat(txnFlagSet).isNotEqualTo(TxnFlagSet.of(TxnFlags.MDB_RDONLY_TXN)); - assertThat(txnFlagSet).isNotEqualTo(TxnFlagSet.builder() - .setFlag(TxnFlags.MDB_RDONLY_TXN) - .build()); + @Test + void testSingleEnum() { + final TxnFlagSet txnFlagSet = TxnFlags.MDB_RDONLY_TXN; + assertThat(txnFlagSet.getMask()).isEqualTo(MaskedFlag.mask(TxnFlags.MDB_RDONLY_TXN)); + assertThat(txnFlagSet.size()).isEqualTo(1); + for (TxnFlags flag : txnFlagSet) { + assertThat(txnFlagSet.isSet(flag)).isEqualTo(true); } - @Test - public void testOf() { - final TxnFlags txnFlag = TxnFlags.MDB_RDONLY_TXN; - final TxnFlagSet txnFlagSet = TxnFlagSet.of(txnFlag); - assertThat(txnFlagSet.getMask()).isEqualTo(MaskedFlag.mask(txnFlag)); - assertThat(txnFlagSet.size()).isEqualTo(1); - for (TxnFlags flag : txnFlagSet) { - assertThat(txnFlagSet.isSet(flag)).isEqualTo(true); - } + final TxnFlagSet txnFlagSet2 = TxnFlagSet.builder() + .setFlag(TxnFlags.MDB_RDONLY_TXN) + .build(); + assertThat(txnFlagSet2.getFlags()).containsExactlyElementsOf(txnFlagSet.getFlags()); + assertThat(txnFlagSet.areAnySet(TxnFlags.MDB_RDONLY_TXN)).isTrue(); + assertThat(txnFlagSet.areAnySet(TxnFlagSet.empty())).isFalse(); + } - final TxnFlagSet txnFlagSet2 = TxnFlagSet.builder() - .setFlag(txnFlag) - .build(); - assertThat(txnFlagSet).isEqualTo(txnFlagSet2); + @Test + public void testEmpty() { + final TxnFlagSet txnFlagSet = TxnFlagSet.empty(); + assertThat(txnFlagSet.getMask()).isEqualTo(0); + assertThat(txnFlagSet.size()).isEqualTo(0); + assertThat(txnFlagSet.isEmpty()).isEqualTo(true); + assertThat(txnFlagSet.isSet(TxnFlags.MDB_RDONLY_TXN)).isEqualTo(false); + final TxnFlagSet txnFlagSet2 = TxnFlagSet.builder() + .build(); + assertThat(txnFlagSet).isEqualTo(txnFlagSet2); + assertThat(txnFlagSet).isNotEqualTo(TxnFlagSet.of(TxnFlags.MDB_RDONLY_TXN)); + assertThat(txnFlagSet).isNotEqualTo(TxnFlagSet.builder() + .setFlag(TxnFlags.MDB_RDONLY_TXN) + .build()); + } + + @Test + public void testOf() { + final TxnFlags txnFlag = TxnFlags.MDB_RDONLY_TXN; + final TxnFlagSet txnFlagSet = TxnFlagSet.of(txnFlag); + assertThat(txnFlagSet.getMask()).isEqualTo(MaskedFlag.mask(txnFlag)); + assertThat(txnFlagSet.size()).isEqualTo(1); + for (TxnFlags flag : txnFlagSet) { + assertThat(txnFlagSet.isSet(flag)).isEqualTo(true); } - @Test - public void testBuilder() { - final TxnFlags txnFlag1 = TxnFlags.MDB_RDONLY_TXN; - final TxnFlagSet txnFlagSet = TxnFlagSet.builder() - .setFlag(txnFlag1) - .build(); - assertThat(txnFlagSet.getMask()).isEqualTo(MaskedFlag.mask(txnFlag1)); - assertThat(txnFlagSet.size()).isEqualTo(1); - assertThat(txnFlagSet.isSet(TxnFlags.MDB_RDONLY_TXN)).isEqualTo(true); - for (TxnFlags flag : txnFlagSet) { - assertThat(txnFlagSet.isSet(flag)).isEqualTo(true); - } - final TxnFlagSet txnFlagSet2 = TxnFlagSet.builder() - .withFlags(txnFlag1) - .build(); - final TxnFlagSet txnFlagSet3 = TxnFlagSet.builder() - .withFlags(new HashSet<>(Collections.singletonList(txnFlag1))) - .build(); - assertThat(txnFlagSet).isEqualTo(txnFlagSet2); - assertThat(txnFlagSet).isEqualTo(txnFlagSet3); + final TxnFlagSet txnFlagSet2 = TxnFlagSet.builder() + .setFlag(txnFlag) + .build(); + assertThat(txnFlagSet).isEqualTo(txnFlagSet2); + } + + @Test + public void testBuilder() { + final TxnFlags txnFlag1 = TxnFlags.MDB_RDONLY_TXN; + final TxnFlagSet txnFlagSet = TxnFlagSet.builder() + .setFlag(txnFlag1) + .build(); + assertThat(txnFlagSet.getMask()).isEqualTo(MaskedFlag.mask(txnFlag1)); + assertThat(txnFlagSet.size()).isEqualTo(1); + assertThat(txnFlagSet.isSet(TxnFlags.MDB_RDONLY_TXN)).isEqualTo(true); + for (TxnFlags flag : txnFlagSet) { + assertThat(txnFlagSet.isSet(flag)).isEqualTo(true); } + final TxnFlagSet txnFlagSet2 = TxnFlagSet.builder() + .withFlags(txnFlag1) + .build(); + final TxnFlagSet txnFlagSet3 = TxnFlagSet.builder() + .withFlags(new HashSet<>(Collections.singletonList(txnFlag1))) + .build(); + assertThat(txnFlagSet).isEqualTo(txnFlagSet2); + assertThat(txnFlagSet).isEqualTo(txnFlagSet3); + } } From 1acd9711efb752013b6588e5e3d106474afa5822 Mon Sep 17 00:00:00 2001 From: at055612 <22818309+at055612@users.noreply.github.com> Date: Wed, 5 Nov 2025 16:57:43 +0000 Subject: [PATCH 21/21] Add ComparatorFactory --- src/main/java/org/lmdbjava/DbiBuilder.java | 50 +++++--- .../lmdbjava/ComparatorIntegerKeyTest.java | 109 ++++++++---------- .../CursorIterableIntegerDupTest.java | 6 +- .../CursorIterableIntegerKeyTest.java | 8 +- .../org/lmdbjava/CursorIterablePerfTest.java | 2 +- .../java/org/lmdbjava/CursorIterableTest.java | 11 +- .../java/org/lmdbjava/DbiBuilderTest.java | 2 +- 7 files changed, 96 insertions(+), 92 deletions(-) diff --git a/src/main/java/org/lmdbjava/DbiBuilder.java b/src/main/java/org/lmdbjava/DbiBuilder.java index f94118a7..9cb85616 100644 --- a/src/main/java/org/lmdbjava/DbiBuilder.java +++ b/src/main/java/org/lmdbjava/DbiBuilder.java @@ -49,8 +49,9 @@ public class DbiBuilder { *

* The name will be converted into bytes using {@link StandardCharsets#UTF_8}. *

+ * * @param name The name of the database or null for the unnamed database - * (see also {@link DbiBuilder#withoutDbName()}) + * (see also {@link DbiBuilder#withoutDbName()}) * @return The next builder stage. */ public DbiBuilderStage2 setDbName(final String name) { @@ -63,6 +64,7 @@ public DbiBuilderStage2 setDbName(final String name) { /** * Create the {@link Dbi} with the passed name in byte[] form. + * * @param name The name of the database in byte form. * @return The next builder stage. */ @@ -83,6 +85,7 @@ public DbiBuilderStage2 setDbName(final byte[] name) { *

Note: The 'unnamed database' is used by LMDB to store the names of named databases, with * the database name being the key. Use of the unnamed database is intended for simple applications * with only one database.

+ * * @return The next builder stage. */ public DbiBuilderStage2 withoutDbName() { @@ -102,7 +105,7 @@ public static class DbiBuilderStage2 { private final DbiBuilder dbiBuilder; - private java.util.Comparator customComparator; + private ComparatorFactory comparatorFactory; private ComparatorType comparatorType; private DbiBuilderStage2(final DbiBuilder dbiBuilder) { @@ -130,7 +133,7 @@ private DbiBuilderStage2(final DbiBuilder dbiBuilder) { * If you do not intend to use {@link CursorIterable} then it doesn't matter whether * you choose {@link DbiBuilderStage2#withNativeComparator()}, * {@link DbiBuilderStage2#withDefaultComparator()} or - * {@link DbiBuilderStage2#withIteratorComparator(Comparator)} as these comparators will + * {@link DbiBuilderStage2#withIteratorComparator(ComparatorFactory)} as these comparators will * never be used. *

* @@ -157,7 +160,7 @@ public DbiBuilderStage3 withDefaultComparator() { * If you do not intend to use {@link CursorIterable} then it doesn't matter whether * you choose {@link DbiBuilderStage2#withNativeComparator()}, * {@link DbiBuilderStage2#withDefaultComparator()} or - * {@link DbiBuilderStage2#withIteratorComparator(Comparator)} as these comparators will + * {@link DbiBuilderStage2#withIteratorComparator(ComparatorFactory)} as these comparators will * never be used. *

* @@ -186,11 +189,13 @@ public DbiBuilderStage3 withNativeComparator() { * are stored in the database. *

* - * @param comparator for all key comparison operations. + * @param comparatorFactory A factory to create a comparator. {@link ComparatorFactory#create(DbiFlagSet)} + * will be called once during the initialisation of the {@link Dbi}. It must + * not return null. * @return The next builder stage. */ - public DbiBuilderStage3 withCallbackComparator(final Comparator comparator) { - this.customComparator = Objects.requireNonNull(comparator); + public DbiBuilderStage3 withCallbackComparator(final ComparatorFactory comparatorFactory) { + this.comparatorFactory = Objects.requireNonNull(comparatorFactory); this.comparatorType = ComparatorType.CALLBACK; return new DbiBuilderStage3<>(this); } @@ -215,15 +220,17 @@ public DbiBuilderStage3 withCallbackComparator(final Comparator comparator * If you do not intend to use {@link CursorIterable} then it doesn't matter whether * you choose {@link DbiBuilderStage2#withNativeComparator()}, * {@link DbiBuilderStage2#withDefaultComparator()} or - * {@link DbiBuilderStage2#withIteratorComparator(Comparator)} as these comparators will + * {@link DbiBuilderStage2#withIteratorComparator(ComparatorFactory)} as these comparators will * never be used. *

* - * @param comparator The comparator to use with {@link CursorIterable}. + * @param comparatorFactory The comparator to use with {@link CursorIterable}. + * {@link ComparatorFactory#create(DbiFlagSet)} will be called once during the + * initialisation of the {@link Dbi}. It must not return null. * @return The next builder stage. */ - public DbiBuilderStage3 withIteratorComparator(final Comparator comparator) { - this.customComparator = Objects.requireNonNull(comparator); + public DbiBuilderStage3 withIteratorComparator(final ComparatorFactory comparatorFactory) { + this.comparatorFactory = Objects.requireNonNull(comparatorFactory); this.comparatorType = ComparatorType.ITERATOR; return new DbiBuilderStage3<>(this); } @@ -267,8 +274,8 @@ public DbiBuilderStage3 setDbiFlags(final Collection dbiFlags) { flagSetBuilder.clear(); if (dbiFlags != null) { dbiFlags.stream() - .filter(Objects::nonNull) - .forEach(dbiFlags::add); + .filter(Objects::nonNull) + .forEach(dbiFlags::add); } return this; } @@ -310,7 +317,7 @@ public DbiBuilderStage3 setDbiFlags(final DbiFlags... dbiFlags) { *

* * @param dbiFlagSet to open the database with. - * A null value will just clear all set flags. + * A null value will just clear all set flags. */ public DbiBuilderStage3 setDbiFlags(final DbiFlagSet dbiFlagSet) { flagSetBuilder.clear(); @@ -413,7 +420,9 @@ private Comparator getComparator(final DbiBuilder dbiBuilder, break; case CALLBACK: case ITERATOR: - comparator = dbiBuilderStage2.customComparator; + comparator = Objects.requireNonNull( + dbiBuilderStage2.comparatorFactory.create(dbiFlagSet), + () -> "comparatorFactory returned null"); break; case NATIVE: break; @@ -465,4 +474,15 @@ private enum ComparatorType { ITERATOR, ; } + + + // -------------------------------------------------------------------------------- + + + @FunctionalInterface + public interface ComparatorFactory { + + Comparator create(final DbiFlagSet dbiFlagSet); + + } } diff --git a/src/test/java/org/lmdbjava/ComparatorIntegerKeyTest.java b/src/test/java/org/lmdbjava/ComparatorIntegerKeyTest.java index 5b9e0761..32e0c8c5 100644 --- a/src/test/java/org/lmdbjava/ComparatorIntegerKeyTest.java +++ b/src/test/java/org/lmdbjava/ComparatorIntegerKeyTest.java @@ -30,13 +30,10 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Comparator; -import java.util.LinkedHashMap; -import java.util.Map; import java.util.Random; import java.util.stream.Stream; import org.agrona.DirectBuffer; import org.agrona.concurrent.UnsafeBuffer; -import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -98,13 +95,10 @@ void testInt(final ComparatorRunner comparator) { assertThat(get(comparator.compare(Integer.MAX_VALUE, 0))).isEqualTo(GREATER_THAN); } - @Test - void testRandomLong() { + @ParameterizedTest + @MethodSource("comparatorProvider") + void testRandomLong(final ComparatorRunner runner) { final Random random = new Random(3239480); - final Map nameToRunnerMap = new LinkedHashMap<>(); - nameToRunnerMap.put("DirectBufferRunner", new DirectBufferRunner()); - nameToRunnerMap.put("ByteBufferRunner", new ByteBufferRunner()); - nameToRunnerMap.put("NettyRunner", new NettyRunner()); // 5mil random longs to compare final long[] values = random.longs() @@ -115,40 +109,33 @@ void testRandomLong() { for (int i = 1; i < values.length; i++) { final long long1 = values[i - 1]; final long long2 = values[i]; - for (Map.Entry entry : nameToRunnerMap.entrySet()) { - final String name = entry.getKey(); - final ComparatorRunner runner = entry.getValue(); - // Make sure the comparator under test gives the same outcome as just comparing two longs - final ComparatorTest.ComparatorResult result = get(runner.compare(long1, long2)); - final ComparatorTest.ComparatorResult expectedResult = get(Long.compare(long1, long2)); - - assertThat(result) - .withFailMessage(() -> "Compare mismatch for " + name + " - long1: " + long1 - + ", long2: " + long2 - + ", expected: " + expectedResult - + ", actual: " + result) - .isEqualTo(expectedResult); - - final ComparatorTest.ComparatorResult result2 = get(runner.compare(long2, long1)); - final ComparatorTest.ComparatorResult expectedResult2 = expectedResult.opposite(); - - assertThat(result) - .withFailMessage(() -> "Compare mismatch for " + name + " - long2: " + long2 - + ", long1: " + long1 - + ", expected2: " + expectedResult2 - + ", actual2: " + result2) - .isEqualTo(expectedResult); - } + // Make sure the comparator under test gives the same outcome as just comparing two longs + final ComparatorTest.ComparatorResult result = get(runner.compare(long1, long2)); + final ComparatorTest.ComparatorResult expectedResult = get(Long.compare(long1, long2)); + + assertThat(result) + .withFailMessage(() -> "Compare mismatch - long1: " + long1 + + ", long2: " + long2 + + ", expected: " + expectedResult + + ", actual: " + result) + .isEqualTo(expectedResult); + + final ComparatorTest.ComparatorResult result2 = get(runner.compare(long2, long1)); + final ComparatorTest.ComparatorResult expectedResult2 = expectedResult.opposite(); + + assertThat(result) + .withFailMessage(() -> "Compare mismatch for - long2: " + long2 + + ", long1: " + long1 + + ", expected2: " + expectedResult2 + + ", actual2: " + result2) + .isEqualTo(expectedResult); } } - @Test - void testRandomInt() { + @ParameterizedTest + @MethodSource("comparatorProvider") + void testRandomInt(final ComparatorRunner runner) { final Random random = new Random(3239480); - final Map nameToRunnerMap = new LinkedHashMap<>(); - nameToRunnerMap.put("DirectBufferRunner", new DirectBufferRunner()); - nameToRunnerMap.put("ByteBufferRunner", new ByteBufferRunner()); - nameToRunnerMap.put("NettyRunner", new NettyRunner()); // 5mil random ints to compare final int[] values = random.ints() @@ -159,30 +146,26 @@ void testRandomInt() { for (int i = 1; i < values.length; i++) { final int int1 = values[i - 1]; final int int2 = values[i]; - for (Map.Entry entry : nameToRunnerMap.entrySet()) { - final String name = entry.getKey(); - final ComparatorRunner runner = entry.getValue(); - // Make sure the comparator under test gives the same outcome as just comparing two ints - final ComparatorTest.ComparatorResult result = get(runner.compare(int1, int2)); - final ComparatorTest.ComparatorResult expectedResult = get(Integer.compare(int1, int2)); - - assertThat(result) - .withFailMessage(() -> "Compare mismatch for " + name + " - int1: " + int1 - + ", int2: " + int2 - + ", expected: " + expectedResult - + ", actual: " + result) - .isEqualTo(expectedResult); - - final ComparatorTest.ComparatorResult result2 = get(runner.compare(int2, int1)); - final ComparatorTest.ComparatorResult expectedResult2 = expectedResult.opposite(); - - assertThat(result) - .withFailMessage(() -> "Compare mismatch for " + name + " - int2: " + int2 - + ", int1: " + int1 - + ", expected2: " + expectedResult2 - + ", actual2: " + result2) - .isEqualTo(expectedResult); - } + // Make sure the comparator under test gives the same outcome as just comparing two ints + final ComparatorTest.ComparatorResult result = get(runner.compare(int1, int2)); + final ComparatorTest.ComparatorResult expectedResult = get(Integer.compare(int1, int2)); + + assertThat(result) + .withFailMessage(() -> "Compare mismatch for - int1: " + int1 + + ", int2: " + int2 + + ", expected: " + expectedResult + + ", actual: " + result) + .isEqualTo(expectedResult); + + final ComparatorTest.ComparatorResult result2 = get(runner.compare(int2, int1)); + final ComparatorTest.ComparatorResult expectedResult2 = expectedResult.opposite(); + + assertThat(result) + .withFailMessage(() -> "Compare mismatch for - int2: " + int2 + + ", int1: " + int1 + + ", expected2: " + expectedResult2 + + ", actual2: " + result2) + .isEqualTo(expectedResult); } } diff --git a/src/test/java/org/lmdbjava/CursorIterableIntegerDupTest.java b/src/test/java/org/lmdbjava/CursorIterableIntegerDupTest.java index 393065a3..775ac4bc 100644 --- a/src/test/java/org/lmdbjava/CursorIterableIntegerDupTest.java +++ b/src/test/java/org/lmdbjava/CursorIterableIntegerDupTest.java @@ -557,13 +557,13 @@ public Stream provideArguments(ParameterDeclarations parame final DbiFactory callbackComparatorDb = new DbiFactory("callbackComparator", env -> env.buildDbi() .setDbName(DB_3) - .withCallbackComparator(buildComparator()) + .withCallbackComparator(MyArgumentProvider::buildComparator) .setDbiFlags(DBI_FLAGS) .open()); final DbiFactory iteratorComparatorDb = new DbiFactory("iteratorComparator", env -> env.buildDbi() .setDbName(DB_4) - .withIteratorComparator(buildComparator()) + .withIteratorComparator(MyArgumentProvider::buildComparator) .setDbiFlags(DBI_FLAGS) .open()); return Stream.of( @@ -574,7 +574,7 @@ public Stream provideArguments(ParameterDeclarations parame .map(Arguments::of); } - private static Comparator buildComparator() { + private static Comparator buildComparator(final DbiFlagSet dbiFlagSet) { final Comparator baseComparator = BUFFER_PROXY.getComparator(DBI_FLAGS); return (o1, o2) -> { if (o1.remaining() != o2.remaining()) { diff --git a/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java b/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java index aa23235a..a0d9ab0c 100644 --- a/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java +++ b/src/test/java/org/lmdbjava/CursorIterableIntegerKeyTest.java @@ -611,18 +611,16 @@ public Stream provideArguments(ParameterDeclarations parame .withNativeComparator() .setDbiFlags(DBI_FLAGS) .open()); - final Comparator comparator = buildComparator(); - final DbiFactory callbackComparatorDb = new DbiFactory("callbackComparator", env -> env.buildDbi() .setDbName(DB_3) - .withCallbackComparator(comparator) + .withCallbackComparator(MyArgumentProvider::buildComparator) .setDbiFlags(DBI_FLAGS) .open()); final DbiFactory iteratorComparatorDb = new DbiFactory("iteratorComparator", env -> env.buildDbi() .setDbName(DB_4) - .withIteratorComparator(comparator) + .withIteratorComparator(MyArgumentProvider::buildComparator) .setDbiFlags(DBI_FLAGS) .open()); return Stream.of( @@ -633,7 +631,7 @@ public Stream provideArguments(ParameterDeclarations parame .map(Arguments::of); } - private static Comparator buildComparator() { + private static Comparator buildComparator(final DbiFlagSet dbiFlagSet) { final Comparator baseComparator = BUFFER_PROXY.getComparator(DBI_FLAGS); return (o1, o2) -> { if (o1.remaining() != o2.remaining()) { diff --git a/src/test/java/org/lmdbjava/CursorIterablePerfTest.java b/src/test/java/org/lmdbjava/CursorIterablePerfTest.java index 008695fd..e0a40fd9 100644 --- a/src/test/java/org/lmdbjava/CursorIterablePerfTest.java +++ b/src/test/java/org/lmdbjava/CursorIterablePerfTest.java @@ -79,7 +79,7 @@ public void before() throws IOException { // Use a java comparator for start/stop keys and as a callback comparator dbCallbackComparator = env.buildDbi() .setDbName("CallBackComparator") - .withCallbackComparator(bufferProxy.getComparator(dbiFlagSet)) + .withCallbackComparator(bufferProxy::getComparator) .setDbiFlags(dbiFlagSet) .open(); diff --git a/src/test/java/org/lmdbjava/CursorIterableTest.java b/src/test/java/org/lmdbjava/CursorIterableTest.java index 0286fa88..e48f1e69 100644 --- a/src/test/java/org/lmdbjava/CursorIterableTest.java +++ b/src/test/java/org/lmdbjava/CursorIterableTest.java @@ -83,7 +83,6 @@ public final class CursorIterableTest { private static final BufferProxy BUFFER_PROXY = ByteBufferProxy.PROXY_OPTIMAL; private Path file; - private Dbi db; private Env env; private Deque list; @@ -309,7 +308,11 @@ void openClosedBackwardTestWithGuava() { bb2.reset(); return guava.compare(array1, array2); }; - final Dbi guavaDbi = env.openDbi(DB_1, comparator, MDB_CREATE); + final Dbi guavaDbi = env.buildDbi() + .setDbName(DB_1) + .withDefaultComparator() + .setDbiFlags(MDB_CREATE) + .open(); populateDatabase(guavaDbi); verify(openClosedBackward(bb(7), bb(2)), guavaDbi, 6, 4, 2); verify(openClosedBackward(bb(8), bb(4)), guavaDbi, 6, 4); @@ -548,13 +551,13 @@ public Stream provideArguments(ParameterDeclarations parame final DbiFactory callbackComparatorDb = new DbiFactory("callbackComparator", env -> env.buildDbi() .setDbName(DB_3) - .withCallbackComparator(BUFFER_PROXY.getComparator(DBI_FLAGS)) + .withCallbackComparator(BUFFER_PROXY::getComparator) .setDbiFlags(DBI_FLAGS) .open()); final DbiFactory iteratorComparatorDb = new DbiFactory("iteratorComparator", env -> env.buildDbi() .setDbName(DB_4) - .withIteratorComparator(BUFFER_PROXY.getComparator(DBI_FLAGS)) + .withIteratorComparator(BUFFER_PROXY::getComparator) .setDbiFlags(DBI_FLAGS) .open()); return Stream.of( diff --git a/src/test/java/org/lmdbjava/DbiBuilderTest.java b/src/test/java/org/lmdbjava/DbiBuilderTest.java index 25e622bf..d01f6417 100644 --- a/src/test/java/org/lmdbjava/DbiBuilderTest.java +++ b/src/test/java/org/lmdbjava/DbiBuilderTest.java @@ -133,7 +133,7 @@ public void callback() { final Dbi dbi = env.buildDbi() .setDbName("foo") - .withCallbackComparator(comparator) + .withCallbackComparator(ignored -> comparator) .addDbiFlags(DbiFlags.MDB_CREATE) .open();