A Bloom filter is a data structure that can be used for probabilistic - * set membership.
+ * A Bloom filter is a data structure that can be used for probabilistic + * set membership. * *When querying a Bloom filter, there are no false positives. Specifically: * When querying an item that has already been inserted to the filter, the filter will @@ -100,7 +100,7 @@ public final class BloomFilter { throw new SketchesArgumentException("Provided WritableMemory capacity insufficient to initialize BloomFilter"); } - // we don't resize so initialize wizth non-empty preLongs value + // we don't resize so initialize with non-empty preLongs value // and no empty flag final WritableBuffer wbuf = wmem.asWritableBuffer(); wbuf.putByte((byte) Family.BLOOMFILTER.getMaxPreLongs()); @@ -381,7 +381,7 @@ private void updateInternal(final long h0, final long h1) { /** * Updates the filter with the provided long and - * returns the result from quering that value prior to the update. + * returns the result from querying that value prior to the update. * @param item an item with which to update the filter * @return The query result prior to applying the update */ @@ -393,7 +393,7 @@ public boolean queryAndUpdate(final long item) { /** * Updates the filter with the provided double and - * returns the result from quering that value prior to the update. + * returns the result from querying that value prior to the update. * The double is canonicalized (NaN and +/- infinity) in the call. * @param item an item with which to update the filter * @return The query result prior to applying the update @@ -408,7 +408,7 @@ public boolean queryAndUpdate(final double item) { /** * Updates the filter with the provided String and - * returns the result from quering that value prior to the update. + * returns the result from querying that value prior to the update. * The string is converted to a byte array using UTF8 encoding. * *
Note: this will not produce the same output hash values as the {@link #queryAndUpdate(char[])} @@ -428,7 +428,7 @@ public boolean queryAndUpdate(final String item) { /** * Updates the filter with the provided byte[] and - * returns the result from quering that array prior to the update. + * returns the result from querying that array prior to the update. * @param data an array with which to update the filter * @return The query result prior to applying the update, or false if data is null */ @@ -440,7 +440,7 @@ public boolean queryAndUpdate(final byte[] data) { /** * Updates the filter with the provided char[] and - * returns the result from quering that array prior to the update. + * returns the result from querying that array prior to the update. * @param data an array with which to update the filter * @return The query result prior to applying the update, or false if data is null */ @@ -453,7 +453,7 @@ public boolean queryAndUpdate(final char[] data) { /** * Updates the filter with the provided short[] and - * returns the result from quering that array prior to the update. + * returns the result from querying that array prior to the update. * @param data an array with which to update the filter * @return The query result prior to applying the update, or false if data is null */ @@ -466,7 +466,7 @@ public boolean queryAndUpdate(final short[] data) { /** * Updates the filter with the provided int[] and - * returns the result from quering that array prior to the update. + * returns the result from querying that array prior to the update. * @param data an array with which to update the filter * @return The query result prior to applying the update, or false if data is null */ @@ -479,7 +479,7 @@ public boolean queryAndUpdate(final int[] data) { /** * Updates the filter with the provided long[] and - * returns the result from quering that array prior to the update. + * returns the result from querying that array prior to the update. * @param data an array with which to update the filter * @return The query result prior to applying the update, or false if data is null */ @@ -492,7 +492,7 @@ public boolean queryAndUpdate(final long[] data) { /** * Updates the filter with the provided Memory and - * returns the result from quering that Memory prior to the update. + * returns the result from querying that Memory prior to the update. * @param mem an array with which to update the filter * @return The query result prior to applying the update, or false if mem is null */ @@ -762,7 +762,7 @@ public static long getSerializedSize(final long numBits) { * 3 ||---------------------------------NumBitsSet------------------------------------| * * - * The raw BitArray bits, if non-empty start at byte 24. + * The raw BitArray bits, if non-empty, start at byte 32. */ /** diff --git a/src/main/java/org/apache/datasketches/filters/bloomfilter/BloomFilterBuilder.java b/src/main/java/org/apache/datasketches/filters/bloomfilter/BloomFilterBuilder.java index f865a3350..ee17a9918 100644 --- a/src/main/java/org/apache/datasketches/filters/bloomfilter/BloomFilterBuilder.java +++ b/src/main/java/org/apache/datasketches/filters/bloomfilter/BloomFilterBuilder.java @@ -25,8 +25,8 @@ import org.apache.datasketches.memory.WritableMemory; /** - *
This class provides methods to help estimate the correct parameters when - * creating a Bloom filter, and methods to create the filter using those values.
+ * This class provides methods to help estimate the correct parameters when + * creating a Bloom filter, and methods to create the filter using those values. * * * - *- * MAP: Low significance bytes of this long data structure are on the right. However, the + *
MAP: Low significance bytes of this long data structure are on the right. However, the * multi-byte integers (int and long) are stored in native byte order. The byte - * values are treated as unsigned. - *
+ * values are treated as unsigned. * - *- * An empty FrequentItems only requires 8 bytes. All others require 32 bytes of preamble. - *
+ *An empty FrequentItems only requires 8 bytes. All others require 32 bytes of preamble.
* ** * Long || Start Byte Adr: diff --git a/src/main/java/org/apache/datasketches/hash/MurmurHash3.java b/src/main/java/org/apache/datasketches/hash/MurmurHash3.java index 9fb2ab948..a708e0077 100644 --- a/src/main/java/org/apache/datasketches/hash/MurmurHash3.java +++ b/src/main/java/org/apache/datasketches/hash/MurmurHash3.java @@ -29,10 +29,8 @@ import org.apache.datasketches.memory.Memory; /** - ** The MurmurHash3 is a fast, non-cryptographic, 128-bit hash function that has * excellent avalanche and 2-way bit independence properties. - *
* ** Austin Appleby's C++ diff --git a/src/main/java/org/apache/datasketches/hash/MurmurHash3Adaptor.java b/src/main/java/org/apache/datasketches/hash/MurmurHash3Adaptor.java deleted file mode 100644 index feecdba97..000000000 --- a/src/main/java/org/apache/datasketches/hash/MurmurHash3Adaptor.java +++ /dev/null @@ -1,479 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.datasketches.hash; - -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.datasketches.common.Util.ceilingPowerOf2; -import static org.apache.datasketches.hash.MurmurHash3.hash; - -import java.nio.ByteBuffer; - -import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.common.SketchesStateException; - -/** - * A general purpose wrapper for the MurmurHash3. - *
The integer values produced are only as random as the MurmurHash3 algorithm, which may be - * adequate for many applications. However, if you are looking for high guarantees of randomness - * you should turn to more sophisticated random generators such as Mersenne Twister or Well19937c - * algorithms. - * - * @param data The input data (key) - * @param n The upper exclusive bound of the integers produced. Must be > 1. - * @return deterministic uniform random integer - */ - private static int asInteger(final long[] data, final int n) { - int t; - int cnt = 0; - long seed = 0; - if (n < 2) { - throw new SketchesArgumentException("Given value of n must be > 1."); - } - if (n > (1 << 30)) { - while (++cnt < 10000) { - final long[] h = MurmurHash3.hash(data, seed); - t = (int) (h[0] & INT_MASK); - if (t < n) { - return t; - } - t = (int) ((h[0] >>> 33)); - if (t < n) { - return t; - } - t = (int) (h[1] & INT_MASK); - if (t < n) { - return t; - } - t = (int) ((h[1] >>> 33)); - if (t < n) { - return t; - } - seed += PRIME; - } // end while - throw new SketchesStateException( - "Internal Error: Failed to find integer < n within 10000 iterations."); - } - final long mask = ceilingPowerOf2(n) - 1; - while (++cnt < 10000) { - final long[] h = MurmurHash3.hash(data, seed); - t = (int) (h[0] & mask); - if (t < n) { - return t; - } - t = (int) ((h[0] >>> 33) & mask); - if (t < n) { - return t; - } - t = (int) (h[1] & mask); - if (t < n) { - return t; - } - t = (int) ((h[1] >>> 33) & mask); - if (t < n) { - return t; - } - seed += PRIME; - } // end while - throw new SketchesStateException( - "Internal Error: Failed to find integer < n within 10000 iterations."); - } - - /** - * Returns a uniform random double with a minimum inclusive value of zero and a maximum exclusive - * value of 1.0. - * - *
The double values produced are only as random as the MurmurHash3 algorithm, which may be - * adequate for many applications. However, if you are looking for high guarantees of randomness - * you should turn to more sophisticated random generators such as Mersenne Twister or Well - * algorithms. - * - * @param hash The output of the MurmurHash3. - * @return the uniform random double. - */ - public static double asDouble(final long[] hash) { - return (hash[0] >>> 12) * 0x1.0p-52d; - } - - /** - * Returns the remainder from the modulo division of the 128-bit output of the murmurHash3 by the - * divisor. - * - * @param h0 The lower 64-bits of the 128-bit MurmurHash3 hash. - * @param h1 The upper 64-bits of the 128-bit MurmurHash3 hash. - * @param divisor Must be positive and greater than zero. - * @return the modulo result. - */ - public static int modulo(final long h0, final long h1, final int divisor) { - final long d = divisor; - final long modH0 = (h0 < 0L) ? addRule(mulRule(BIT62, 2L, d), (h0 & MAX_LONG), d) : h0 % d; - final long modH1 = (h1 < 0L) ? addRule(mulRule(BIT62, 2L, d), (h1 & MAX_LONG), d) : h1 % d; - final long modTop = mulRule(mulRule(BIT62, 4L, d), modH1, d); - return (int) addRule(modTop, modH0, d); - } - - /** - * Returns the remainder from the modulo division of the 128-bit output of the murmurHash3 by the - * divisor. - * - * @param hash The size 2 long array from the MurmurHash3. - * @param divisor Must be positive and greater than zero. - * @return the modulo result - */ - public static int modulo(final long[] hash, final int divisor) { - return modulo(hash[0], hash[1], divisor); - } - - private static long addRule(final long a, final long b, final long d) { - return ((a % d) + (b % d)) % d; - } - - private static long mulRule(final long a, final long b, final long d) { - return ((a % d) * (b % d)) % d; - } - - private static byte[] toByteArray(final long[] hash) { //Assumes Big Endian - final byte[] bArr = new byte[16]; - final ByteBuffer bb = ByteBuffer.wrap(bArr); - bb.putLong(hash[0]); - bb.putLong(hash[1]); - return bArr; - } - - private static long[] toLongArray(final byte[] data) { - final int dataLen = data.length; - final int longLen = (dataLen + 7) / 8; - final long[] longArr = new long[longLen]; - for (int bi = 0; bi < dataLen; bi++) { - final int li = bi / 8; - longArr[li] |= (((long)data[bi]) << ((bi * 8) % 64)); - } - return longArr; - } - - private static long[] toLongArray(final int[] data) { - final int dataLen = data.length; - final int longLen = (dataLen + 1) / 2; - final long[] longArr = new long[longLen]; - for (int ii = 0; ii < dataLen; ii++) { - final int li = ii / 2; - longArr[li] |= (((long)data[ii]) << ((ii * 32) % 64)); - } - return longArr; - } - -} diff --git a/src/main/java/org/apache/datasketches/hash/MurmurHash3FFM21.java b/src/main/java/org/apache/datasketches/hash/MurmurHash3FFM21.java new file mode 100644 index 000000000..a94fd3fd9 --- /dev/null +++ b/src/main/java/org/apache/datasketches/hash/MurmurHash3FFM21.java @@ -0,0 +1,386 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.hash; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.Objects; + +import org.apache.datasketches.memory.Memory; + +/** + * The MurmurHash3 is a fast, non-cryptographic, 128-bit hash function that has + * excellent avalanche and 2-way bit independence properties. + * + *
Austin Appleby's C++ + * + * MurmurHash3_x64_128(...), final revision 150, + * which is in the Public Domain, was the inspiration for this implementation in Java.
+ * + *This implementation of the MurmurHash3 allows hashing of a block of on-heap Memory defined by an offset + * and length. The calling API also allows the user to supply the small output array of two longs, + * so that the entire hash function is static and free of object allocations.
+ * + *This implementation produces exactly the same hash result as the + * MurmurHash3 function in datasketches-java given compatible inputs.
+ * + *This FFM21 version of the implementation leverages the java.lang.foreign package (FFM) of JDK-21 in place of + * the Unsafe class. + * + * @author Lee Rhodes + */ +public final class MurmurHash3FFM21 { + private static final long C1 = 0x87c37b91114253d5L; + private static final long C2 = 0x4cf5ad432745937fL; + + /** + * Returns a 128-bit hash of the input. + * Provided for compatibility with older version of MurmurHash3, + * but empty or null input now throws IllegalArgumentException. + * @param in long array + * @param seed A long valued seed. + * @return the hash + * @throws IllegalArgumentException if input is empty or null + */ + public static long[] hash(final long[] in, final long seed) { + if ((in == null) || (in.length == 0)) { + throw new IllegalArgumentException("Input in is empty or null."); + } + return hash(MemorySegment.ofArray(in), 0L, in.length << 3, seed, new long[2]); + } + + /** + * Returns a 128-bit hash of the input. + * Provided for compatibility with older version of MurmurHash3, + * but empty or null input now throws IllegalArgumentException. + * @param in int array + * @param seed A long valued seed. + * @return the hash + * @throws IllegalArgumentException if input is empty or null + */ + public static long[] hash(final int[] in, final long seed) { + if ((in == null) || (in.length == 0)) { + throw new IllegalArgumentException("Input in is empty or null."); + } + return hash(MemorySegment.ofArray(in), 0L, in.length << 2, seed, new long[2]); + } + + /** + * Returns a 128-bit hash of the input. + * Provided for compatibility with older version of MurmurHash3, + * but empty or null input now throws IllegalArgumentException. + * @param in char array + * @param seed A long valued seed. + * @return the hash + * @throws IllegalArgumentException if input is empty or null + */ + public static long[] hash(final char[] in, final long seed) { + if ((in == null) || (in.length == 0)) { + throw new IllegalArgumentException("Input in is empty or null."); + } + return hash(MemorySegment.ofArray(in), 0L, in.length << 1, seed, new long[2]); + } + + /** + * Returns a 128-bit hash of the input. + * Provided for compatibility with older version of MurmurHash3, + * but empty or null input now throws IllegalArgumentException. + * @param in byte array + * @param seed A long valued seed. + * @return the hash + * @throws IllegalArgumentException if input is empty or null + */ + public static long[] hash(final byte[] in, final long seed) { + if ((in == null) || (in.length == 0)) { + throw new IllegalArgumentException("Input in is empty or null."); + } + return hash(MemorySegment.ofArray(in), 0L, in.length, seed, new long[2]); + } + + //Single primitive inputs + + /** + * Returns a 128-bit hash of the input. + * Note the entropy of the resulting hash cannot be more than 64 bits. + * @param in a long + * @param seed A long valued seed. + * @param hashOut A long array of size 2 + * @return the hash + */ + public static long[] hash(final long in, final long seed, final long[] hashOut) { + final long h1 = seed ^ mixK1(in); + final long h2 = seed; + return finalMix128(h1, h2, 8, hashOut); + } + + /** + * Returns a 128-bit hash of the input. + * Note the entropy of the resulting hash cannot be more than 64 bits. + * @param in a double + * @param seed A long valued seed. + * @param hashOut A long array of size 2 + * @return the hash + */ + public static long[] hash(final double in, final long seed, final long[] hashOut) { + final double d = (in == 0.0) ? 0.0 : in; // canonicalize -0.0, 0.0 + final long k1 = Double.doubleToLongBits(d); // canonicalize all NaN forms + final long h1 = seed ^ mixK1(k1); + final long h2 = seed; + return finalMix128(h1, h2, 8, hashOut); + } + + /** + * Returns a 128-bit hash of the input. + * An empty or null input throws IllegalArgumentException. + * @param in a String + * @param seed A long valued seed. + * @param hashOut A long array of size 2 + * @return the hash + * @throws IllegalArgumentException if input is empty or null + */ + public static long[] hash(final String in, final long seed, final long[] hashOut) { + if ((in == null) || (in.length() == 0)) { + throw new IllegalArgumentException("Input in is empty or null."); + } + final byte[] byteArr = in.getBytes(UTF_8); + return hash(MemorySegment.ofArray(byteArr), 0L, byteArr.length, seed, hashOut); + } + + //The main API calls + + /** + * Returns a 128-bit hash of the input as a long array of size 2. + * + * @param mem The input Memory. Must be non-null and non-empty, + * otherwise throws IllegalArgumentException. + * @param offsetBytes the starting point within Memory. + * @param lengthBytes the total number of bytes to be hashed. + * @param seed A long valued seed. + * @param hashOut the size 2 long array for the resulting 128-bit hash + * @return the hash. + */ + public static long[] hash(final Memory mem, final long offsetBytes, final long lengthBytes, + final long seed, final long[] hashOut) { + Objects.requireNonNull(mem, "Input Memory is null"); + final MemorySegment seg = mem.getMemorySegment(); + return hash(seg, offsetBytes, lengthBytes, seed, hashOut); + } + + /** + * Returns a 128-bit hash of the input as a long array of size 2. + * + * @param seg The input MemorySegment. Must be non-null and non-empty, + * otherwise throws IllegalArgumentException. + * @param offsetBytes the starting point within Memory. + * @param lengthBytes the total number of bytes to be hashed. + * @param seed A long valued seed. + * @param hashOut the size 2 long array for the resulting 128-bit hash + * @return the hash. + * @throws IllegalArgumentException if input MemorySegment is empty + */ + public static long[] hash(final MemorySegment seg, final long offsetBytes, final long lengthBytes, + final long seed, final long[] hashOut) { + Objects.requireNonNull(seg, "Input MemorySegment is null"); + if (seg.byteSize() == 0L) { throw new IllegalArgumentException("Input MemorySegment is empty."); } + + long cumOff = offsetBytes; + + long h1 = seed; + long h2 = seed; + long rem = lengthBytes; + + // Process the 128-bit blocks (the body) into the hash + while (rem >= 16L) { + final long k1 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff); //0, 16, 32, ... + final long k2 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff + 8); //8, 24, 40, ... + cumOff += 16L; + rem -= 16L; + + h1 ^= mixK1(k1); + h1 = Long.rotateLeft(h1, 27); + h1 += h2; + h1 = (h1 * 5) + 0x52dce729L; + + h2 ^= mixK2(k2); + h2 = Long.rotateLeft(h2, 31); + h2 += h1; + h2 = (h2 * 5) + 0x38495ab5L; + } + + // Get the tail (if any): 1 to 15 bytes + if (rem > 0L) { + long k1 = 0; + long k2 = 0; + switch ((int) rem) { + case 15: { + k2 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 14) & 0xFFL) << 48; + } + //$FALL-THROUGH$ + case 14: { + k2 ^= (seg.get(ValueLayout.JAVA_SHORT_UNALIGNED, cumOff + 12) & 0xFFFFL) << 32; + k2 ^= seg.get(ValueLayout.JAVA_INT_UNALIGNED, cumOff + 8) & 0xFFFFFFFFL; + k1 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff); + break; + } + + case 13: { + k2 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 12) & 0xFFL) << 32; + } + //$FALL-THROUGH$ + case 12: { + k2 ^= seg.get(ValueLayout.JAVA_INT_UNALIGNED, cumOff + 8) & 0xFFFFFFFFL; + k1 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff); + break; + } + + case 11: { + k2 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 10) & 0xFFL) << 16; + } + //$FALL-THROUGH$ + case 10: { + k2 ^= seg.get(ValueLayout.JAVA_SHORT_UNALIGNED, cumOff + 8) & 0xFFFFL; + k1 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff); + break; + } + + case 9: { + k2 ^= seg.get(ValueLayout.JAVA_BYTE, cumOff + 8) & 0xFFL; + } + //$FALL-THROUGH$ + case 8: { + k1 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff); + break; + } + + case 7: { + k1 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 6) & 0xFFL) << 48; + } + //$FALL-THROUGH$ + case 6: { + k1 ^= (seg.get(ValueLayout.JAVA_SHORT_UNALIGNED, cumOff + 4) & 0xFFFFL) << 32; + k1 ^= seg.get(ValueLayout.JAVA_INT_UNALIGNED, cumOff) & 0xFFFFFFFFL; + break; + } + + case 5: { + k1 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 4) & 0xFFL) << 32; + } + //$FALL-THROUGH$ + case 4: { + k1 ^= seg.get(ValueLayout.JAVA_INT_UNALIGNED, cumOff) & 0xFFFFFFFFL; + break; + } + + case 3: { + k1 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 2) & 0xFFL) << 16; + } + //$FALL-THROUGH$ + case 2: { + k1 ^= seg.get(ValueLayout.JAVA_SHORT_UNALIGNED, cumOff) & 0xFFFFL; + break; + } + + case 1: { + k1 ^= seg.get(ValueLayout.JAVA_BYTE, cumOff) & 0xFFL; + break; + } + default: break; //can't happen + } + + h1 ^= mixK1(k1); + h2 ^= mixK2(k2); + } + return finalMix128(h1, h2, lengthBytes, hashOut); + } + + //--Helper methods---------------------------------------------------- + + /** + * Self mix of k1 + * + * @param k1 input argument + * @return mix + */ + private static long mixK1(long k1) { + k1 *= C1; + k1 = Long.rotateLeft(k1, 31); + k1 *= C2; + return k1; + } + + /** + * Self mix of k2 + * + * @param k2 input argument + * @return mix + */ + private static long mixK2(long k2) { + k2 *= C2; + k2 = Long.rotateLeft(k2, 33); + k2 *= C1; + return k2; + } + + /** + * Final self mix of h*. + * + * @param h input to final mix + * @return mix + */ + private static long finalMix64(long h) { + h ^= h >>> 33; + h *= 0xff51afd7ed558ccdL; + h ^= h >>> 33; + h *= 0xc4ceb9fe1a85ec53L; + h ^= h >>> 33; + return h; + } + + /** + * Finalization: Add the length into the hash and mix + * @param h1 intermediate hash + * @param h2 intermediate hash + * @param lengthBytes the length in bytes + * @param hashOut the output array of 2 longs + * @return hashOut + */ + private static long[] finalMix128(long h1, long h2, final long lengthBytes, final long[] hashOut) { + h1 ^= lengthBytes; + h2 ^= lengthBytes; + + h1 += h2; + h2 += h1; + + h1 = finalMix64(h1); + h2 = finalMix64(h2); + + h1 += h2; + h2 += h1; + + hashOut[0] = h1; + hashOut[1] = h2; + return hashOut; + } + + private MurmurHash3FFM21() { } + +} diff --git a/src/main/java/org/apache/datasketches/hash/package-info.java b/src/main/java/org/apache/datasketches/hash/package-info.java index 2d97afeb6..5744b2776 100644 --- a/src/main/java/org/apache/datasketches/hash/package-info.java +++ b/src/main/java/org/apache/datasketches/hash/package-info.java @@ -18,12 +18,11 @@ */ /** - *
The hash package contains a high-performing and extended Java implementations + * The hash package contains a high-performing and extended Java implementations * of Austin Appleby's 128-bit MurmurHash3 hash function originally coded in C. * This core MurmurHash3.java class is used throughout many of the sketch classes for consistency * and as long as the user specifies the same seed will result in coordinated hash operations. * This package also contains an adaptor class that extends the basic class with more functions * commonly associated with hashing. - *
*/ package org.apache.datasketches.hash; diff --git a/src/main/java/org/apache/datasketches/hll/DirectAuxHashMap.java b/src/main/java/org/apache/datasketches/hll/DirectAuxHashMap.java index 98884f5ea..e6c408f35 100644 --- a/src/main/java/org/apache/datasketches/hll/DirectAuxHashMap.java +++ b/src/main/java/org/apache/datasketches/hll/DirectAuxHashMap.java @@ -195,7 +195,7 @@ private static final void grow(final DirectHllArray host, final int oldLgAuxArrI final WritableMemory newWmem = svr.request(host.wmem, requestBytes); host.wmem.copyTo(0, newWmem, 0, host.auxStart); newWmem.clear(host.auxStart, newAuxBytes); //clear space for new aux data - svr.requestClose(host.wmem, newWmem); //old host.wmem is now invalid + svr.requestClose(host.wmem); //old host.wmem is now invalid host.updateMemory(newWmem); } //rehash into larger aux array diff --git a/src/main/java/org/apache/datasketches/hll/DirectCouponHashSet.java b/src/main/java/org/apache/datasketches/hll/DirectCouponHashSet.java index 52ddf60ba..683c8be96 100644 --- a/src/main/java/org/apache/datasketches/hll/DirectCouponHashSet.java +++ b/src/main/java/org/apache/datasketches/hll/DirectCouponHashSet.java @@ -35,6 +35,7 @@ import static org.apache.datasketches.hll.PreambleUtil.insertLgArr; import org.apache.datasketches.common.SketchesArgumentException; +import org.apache.datasketches.common.SketchesException; import org.apache.datasketches.common.SketchesStateException; import org.apache.datasketches.memory.Memory; import org.apache.datasketches.memory.WritableMemory; @@ -105,13 +106,14 @@ private boolean checkGrowOrPromote() { if (lgCouponArrInts == (getLgConfigK() - 3)) { return true; // promote } - //TODO if direct, ask for more memory insertLgArr(wmem, ++lgCouponArrInts); growHashSet(wmem, lgCouponArrInts); } return false; } + //This could fail if the user has undersized the given WritableMemory + // and not used the public methods for sizing the Memory. See exception. private static final void growHashSet(final WritableMemory wmem, final int tgtLgCouponArrSize) { final int tgtArrSize = 1 << tgtLgCouponArrSize; final int[] tgtCouponIntArr = new int[tgtArrSize]; @@ -128,7 +130,11 @@ private static final void growHashSet(final WritableMemory wmem, final int tgtLg } } wmem.clear(HASH_SET_INT_ARR_START, tgtArrSize << 2); - wmem.putIntArray(HASH_SET_INT_ARR_START, tgtCouponIntArr, 0, tgtArrSize); + try { wmem.putIntArray(HASH_SET_INT_ARR_START, tgtCouponIntArr, 0, tgtArrSize); } + catch (final IndexOutOfBoundsException e) { + throw new SketchesException( + "The WritableMemory is undersized. Use the public methods for properly sizing Memory.", e); + } } //Searches the Coupon hash table (embedded in Memory) for an empty slot diff --git a/src/main/java/org/apache/datasketches/hll/DirectHllArray.java b/src/main/java/org/apache/datasketches/hll/DirectHllArray.java index 5b3e1a4ff..e0c66c0aa 100644 --- a/src/main/java/org/apache/datasketches/hll/DirectHllArray.java +++ b/src/main/java/org/apache/datasketches/hll/DirectHllArray.java @@ -56,7 +56,6 @@ abstract class DirectHllArray extends AbstractHllArray { WritableMemory wmem; Memory mem; Object memObj; - long memAdd; final boolean compact; private static int checkMemCompactFlag(final WritableMemory wmem, final int lgConfigK) { @@ -70,7 +69,6 @@ private static int checkMemCompactFlag(final WritableMemory wmem, final int lgCo this.wmem = wmem; mem = wmem; memObj = wmem.getArray(); - memAdd = wmem.getCumulativeOffset(0L); compact = extractCompactFlag(mem); insertEmptyFlag(wmem, false); } @@ -81,7 +79,6 @@ private static int checkMemCompactFlag(final WritableMemory wmem, final int lgCo wmem = null; this.mem = mem; memObj = ((WritableMemory) mem).getArray(); - memAdd = mem.getCumulativeOffset(0L); compact = extractCompactFlag(mem); } @@ -90,7 +87,6 @@ final void updateMemory(final WritableMemory newWmem) { wmem = newWmem; mem = newWmem; memObj = wmem.getArray(); - memAdd = wmem.getCumulativeOffset(0L); } @Override diff --git a/src/main/java/org/apache/datasketches/hll/HeapAuxHashMap.java b/src/main/java/org/apache/datasketches/hll/HeapAuxHashMap.java index 374dbd624..5724b563c 100644 --- a/src/main/java/org/apache/datasketches/hll/HeapAuxHashMap.java +++ b/src/main/java/org/apache/datasketches/hll/HeapAuxHashMap.java @@ -205,7 +205,6 @@ else if (slotNo == (arrVal & configKmask)) { //Compares only on slotNo private void checkGrow() { if ((RESIZE_DENOM * auxCount) > (RESIZE_NUMER * auxIntArr.length)) { growAuxSpace(); - //TODO if direct, ask for more memory } } diff --git a/src/main/java/org/apache/datasketches/hll/package-info.java b/src/main/java/org/apache/datasketches/hll/package-info.java index 9ffafa7bd..a17a9f646 100644 --- a/src/main/java/org/apache/datasketches/hll/package-info.java +++ b/src/main/java/org/apache/datasketches/hll/package-info.java @@ -18,7 +18,7 @@ */ /** - *Please refer to the documentation in the package-info:
+ * {@link org.apache.datasketches.kll}
Here is what we do for each level:
*The parameter k will not change.
*/ @Override diff --git a/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java b/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java index 50cadeb3e..1796cac4d 100644 --- a/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java +++ b/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java @@ -207,8 +207,11 @@ static void mergeFloatImpl(final KllFloatsSketch mySketch, final KllFloatsSketch //MEMORY SPACE MANAGEMENT if (mySketch.getWritableMemory() != null) { - final WritableMemory wmem = - KllHelper.memorySpaceMgmt(mySketch, myNewLevelsArr.length, myNewFloatItemsArr.length); + final WritableMemory oldWmem = mySketch.getWritableMemory(); + final WritableMemory wmem = KllHelper.memorySpaceMgmt(mySketch, myNewLevelsArr.length, myNewFloatItemsArr.length); + if (!wmem.isSameResource(oldWmem)) { + mySketch.getMemoryRequestServer().requestClose(oldWmem); + } mySketch.setWritableMemory(wmem); } } //end of updating levels above level 0 @@ -312,6 +315,7 @@ private static void randomlyHalveUpFloats(final float[] buf, final int start, fi /** * Compression algorithm used to merge higher levels. + * *Here is what we do for each level:
*The parameter k will not change.
*/ @Override diff --git a/src/main/java/org/apache/datasketches/kll/KllHeapItemsSketch.java b/src/main/java/org/apache/datasketches/kll/KllHeapItemsSketch.java index e58516fd3..87e3a542a 100644 --- a/src/main/java/org/apache/datasketches/kll/KllHeapItemsSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllHeapItemsSketch.java @@ -36,7 +36,7 @@ import org.apache.datasketches.memory.WritableMemory; /** - * This class implements an on-heap doubles KllSketch. + * This class implements an on-heap items KllSketch. * *Please refer to the documentation in the package-info:
* {@link org.apache.datasketches.kll}
Please refer to the documentation in the package-info:
+ * {@link org.apache.datasketches.kll}
Here is what we do for each level:
*Here is what we do for each level:
+ *It can be proved that generalCompress returns a sketch that satisfies the space constraints + * no matter how much data is passed in. + * We are pretty sure that it works correctly when inBuf and outBuf are the same. + * All levels except for level zero must be sorted before calling this, and will still be + * sorted afterwards. + * Level zero is not required to be sorted before, and may not be sorted afterwards.
+ * + *This trashes inBuf and inLevels and modifies outBuf and outLevels.
+ * + * @param k The sketch parameter k + * @param m The minimum level size + * @param numLevelsIn provisional number of number of levels = max(this.numLevels, other.numLevels) + * @param inBuf work buffer of size = this.getNumRetained() + other.getNumRetainedAboveLevelZero(). + * This contains the long[] of the other sketch + * @param inLevels work levels array size = ubOnNumLevels(this.n + other.n) + 2 + * @param outBuf the same array as inBuf + * @param outLevels the same size as inLevels + * @param isLevelZeroSorted true if this.level 0 is sorted + * @param random instance of java.util.Random + * @return int array of: {numLevels, targetItemCount, currentItemCount) + */ + // + private static int[] generalLongsCompress( + final int k, + final int m, + final int numLevelsIn, + final long[] inBuf, + final int[] inLevels, + final long[] outBuf, + final int[] outLevels, + final boolean isLevelZeroSorted, + final Random random) { + assert numLevelsIn > 0; // things are too weird if zero levels are allowed + int numLevels = numLevelsIn; + int currentItemCount = inLevels[numLevels] - inLevels[0]; // decreases with each compaction + int targetItemCount = KllHelper.computeTotalItemCapacity(k, m, numLevels); // increases if we add levels + boolean doneYet = false; + outLevels[0] = 0; + int curLevel = -1; + while (!doneYet) { + curLevel++; // start out at level 0 + + // If we are at the current top level, add an empty level above it for convenience, + // but do not actually increment numLevels until later + if (curLevel == (numLevels - 1)) { + inLevels[curLevel + 2] = inLevels[curLevel + 1]; + } + + final int rawBeg = inLevels[curLevel]; + final int rawLim = inLevels[curLevel + 1]; + final int rawPop = rawLim - rawBeg; + + if ((currentItemCount < targetItemCount) || (rawPop < KllHelper.levelCapacity(k, numLevels, curLevel, m))) { + // copy level over as is + // because inBuf and outBuf could be the same, make sure we are not moving data upwards! + assert (rawBeg >= outLevels[curLevel]); + System.arraycopy(inBuf, rawBeg, outBuf, outLevels[curLevel], rawPop); + outLevels[curLevel + 1] = outLevels[curLevel] + rawPop; + } + else { + // The sketch is too full AND this level is too full, so we compact it + // Note: this can add a level and thus change the sketch's capacity + + final int popAbove = inLevels[curLevel + 2] - rawLim; + final boolean oddPop = isOdd(rawPop); + final int adjBeg = oddPop ? 1 + rawBeg : rawBeg; + final int adjPop = oddPop ? rawPop - 1 : rawPop; + final int halfAdjPop = adjPop / 2; + + if (oddPop) { // copy one guy over + outBuf[outLevels[curLevel]] = inBuf[rawBeg]; + outLevels[curLevel + 1] = outLevels[curLevel] + 1; + } else { // copy zero guys over + outLevels[curLevel + 1] = outLevels[curLevel]; + } + + // level zero might not be sorted, so we must sort it if we wish to compact it + if ((curLevel == 0) && !isLevelZeroSorted) { + Arrays.sort(inBuf, adjBeg, adjBeg + adjPop); + } + + if (popAbove == 0) { // Level above is empty, so halve up + randomlyHalveUpLongs(inBuf, adjBeg, adjPop, random); + } else { // Level above is nonempty, so halve down, then merge up + randomlyHalveDownLongs(inBuf, adjBeg, adjPop, random); + mergeSortedLongArrays(inBuf, adjBeg, halfAdjPop, inBuf, rawLim, popAbove, inBuf, adjBeg + halfAdjPop); + } + + // track the fact that we just eliminated some data + currentItemCount -= halfAdjPop; + + // Adjust the boundaries of the level above + inLevels[curLevel + 1] = inLevels[curLevel + 1] - halfAdjPop; + + // Increment numLevels if we just compacted the old top level + // This creates some more capacity (the size of the new bottom level) + if (curLevel == (numLevels - 1)) { + numLevels++; + targetItemCount += KllHelper.levelCapacity(k, numLevels, 0, m); + } + } // end of code for compacting a level + + // determine whether we have processed all levels yet (including any new levels that we created) + if (curLevel == (numLevels - 1)) { doneYet = true; } + } // end of loop over levels + + assert (outLevels[numLevels] - outLevels[0]) == currentItemCount; + return new int[] {numLevels, targetItemCount, currentItemCount}; + } + + private static void populateLongWorkArrays( //workBuf and workLevels are modified + final long[] workBuf, final int[] workLevels, final int provisionalNumLevels, + final int myCurNumLevels, final int[] myCurLevelsArr, final long[] myCurLongItemsArr, + final int otherNumLevels, final int[] otherLevelsArr, final long[] otherLongItemsArr) { + + workLevels[0] = 0; + + // Note: the level zero data from "other" was already inserted into "self". + // This copies into workbuf. + final int selfPopZero = KllHelper.currentLevelSizeItems(0, myCurNumLevels, myCurLevelsArr); + System.arraycopy(myCurLongItemsArr, myCurLevelsArr[0], workBuf, workLevels[0], selfPopZero); + workLevels[1] = workLevels[0] + selfPopZero; + + for (int lvl = 1; lvl < provisionalNumLevels; lvl++) { + final int selfPop = KllHelper.currentLevelSizeItems(lvl, myCurNumLevels, myCurLevelsArr); + final int otherPop = KllHelper.currentLevelSizeItems(lvl, otherNumLevels, otherLevelsArr); + workLevels[lvl + 1] = workLevels[lvl] + selfPop + otherPop; + assert selfPop >= 0 && otherPop >= 0; + if (selfPop == 0 && otherPop == 0) { continue; } + if (selfPop > 0 && otherPop == 0) { + System.arraycopy(myCurLongItemsArr, myCurLevelsArr[lvl], workBuf, workLevels[lvl], selfPop); + } + else if (selfPop == 0 && otherPop > 0) { + System.arraycopy(otherLongItemsArr, otherLevelsArr[lvl], workBuf, workLevels[lvl], otherPop); + } + else if (selfPop > 0 && otherPop > 0) { + mergeSortedLongArrays( //only workBuf is modified + myCurLongItemsArr, myCurLevelsArr[lvl], selfPop, + otherLongItemsArr, otherLevelsArr[lvl], otherPop, + workBuf, workLevels[lvl]); + } + } + } + + /* + * Validation Method. + * The following must be enabled for use with the KllDoublesValidationTest, + * which is only enabled for manual testing. In addition, two Validation Methods + * above need to be modified. + */ //NOTE Validation Method: Need to uncomment to use + // static int nextOffset = 0; + // + // private static int deterministicOffset() { + // final int result = nextOffset; + // nextOffset = 1 - nextOffset; + // return result; + // } + +} diff --git a/src/main/java/org/apache/datasketches/kll/KllLongsSketch.java b/src/main/java/org/apache/datasketches/kll/KllLongsSketch.java new file mode 100644 index 000000000..827f825f8 --- /dev/null +++ b/src/main/java/org/apache/datasketches/kll/KllLongsSketch.java @@ -0,0 +1,670 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.kll; + +import static java.lang.Math.max; +import static java.lang.Math.min; +import static org.apache.datasketches.common.ByteArrayUtil.putLongLE; +import static org.apache.datasketches.kll.KllSketch.SketchStructure.UPDATABLE; +import static org.apache.datasketches.kll.KllSketch.SketchType.LONGS_SKETCH; + +import java.util.Arrays; +import java.util.Objects; + +import org.apache.datasketches.common.ArrayOfItemsSerDe; +import org.apache.datasketches.common.SketchesArgumentException; +import org.apache.datasketches.common.SuppressFBWarnings; +import org.apache.datasketches.kll.KllDirectLongsSketch.KllDirectCompactLongsSketch; +import org.apache.datasketches.memory.DefaultMemoryRequestServer; +import org.apache.datasketches.memory.Memory; +import org.apache.datasketches.memory.MemoryRequestServer; +import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.quantilescommon.LongsSketchSortedView; +import org.apache.datasketches.quantilescommon.QuantileSearchCriteria; +import org.apache.datasketches.quantilescommon.QuantilesLongsAPI; +import org.apache.datasketches.quantilescommon.QuantilesLongsSketchIterator; + +/** + * This variation of the KllSketch implements primitive longs. + * + * @see org.apache.datasketches.kll.KllSketch + */ +public abstract class KllLongsSketch extends KllSketch implements QuantilesLongsAPI { + private LongsSketchSortedView longsSV = null; + final static int ITEM_BYTES = Long.BYTES; + + KllLongsSketch( + final SketchStructure sketchStructure) { + super(SketchType.LONGS_SKETCH, sketchStructure); + } + + //Factories for new heap instances. + + /** + * Create a new heap instance of this sketch with the default k = 200. + * The default k = 200 results in a normalized rank error of about + * 1.65%. Larger K will have smaller error but the sketch will be larger (and slower). + * @return new KllLongsSketch on the Java heap. + */ + public static KllLongsSketch newHeapInstance() { + return newHeapInstance(DEFAULT_K); + } + + /** + * Create a new heap instance of this sketch with a given parameter k. + * k can be between 8, inclusive, and 65535, inclusive. + * The default k = 200 results in a normalized rank error of about + * 1.65%. Larger K will have smaller error but the sketch will be larger (and slower). + * @param k parameter that controls size of the sketch and accuracy of estimates. + * @return new KllLongsSketch on the Java heap. + */ + public static KllLongsSketch newHeapInstance(final int k) { + return new KllHeapLongsSketch(k, DEFAULT_M); + } + + //Factories for new direct instances. + + /** + * Create a new direct updatable instance of this sketch with the default k. + * The default k = 200 results in a normalized rank error of about + * 1.65%. Larger k will have smaller error but the sketch will be larger (and slower). + * @param dstMem the given destination WritableMemory object for use by the sketch + * @param memReqSvr the given MemoryRequestServer to request a larger WritableMemory + * @return a new direct instance of this sketch + */ + public static KllLongsSketch newDirectInstance( + final WritableMemory dstMem, + final MemoryRequestServer memReqSvr) { + return newDirectInstance(DEFAULT_K, dstMem, memReqSvr); + } + + /** + * Create a new direct updatable instance of this sketch with a given k. + * @param k parameter that controls size of the sketch and accuracy of estimates. + * @param dstMem the given destination WritableMemory object for use by the sketch + * @param memReqSvr the given MemoryRequestServer to request a larger WritableMemory + * @return a new direct instance of this sketch + */ + public static KllLongsSketch newDirectInstance( + final int k, + final WritableMemory dstMem, + final MemoryRequestServer memReqSvr) { + Objects.requireNonNull(dstMem, "Parameter 'dstMem' must not be null"); + Objects.requireNonNull(memReqSvr, "Parameter 'memReqSvr' must not be null"); + return KllDirectLongsSketch.newDirectUpdatableInstance(k, DEFAULT_M, dstMem, memReqSvr); + } + + //Factory to create an heap instance from a Memory image + + /** + * Factory heapify takes a compact sketch image in Memory and instantiates an on-heap sketch. + * The resulting sketch will not retain any link to the source Memory. + * @param srcMem a compact Memory image of a sketch serialized by this sketch. + * See Memory + * @return a heap-based sketch based on the given Memory. + */ + public static KllLongsSketch heapify(final Memory srcMem) { + Objects.requireNonNull(srcMem, "Parameter 'srcMem' must not be null"); + return KllHeapLongsSketch.heapifyImpl(srcMem); + } + + //Factory to wrap a Read-Only Memory + + /** + * Wrap a sketch around the given read only compact source Memory containing sketch data + * that originated from this sketch. + * @param srcMem the read only source Memory + * @return instance of this sketch + */ + public static KllLongsSketch wrap(final Memory srcMem) { + Objects.requireNonNull(srcMem, "Parameter 'srcMem' must not be null"); + final KllMemoryValidate memVal = new KllMemoryValidate(srcMem, LONGS_SKETCH, null); + if (memVal.sketchStructure == UPDATABLE) { + final MemoryRequestServer memReqSvr = new DefaultMemoryRequestServer(); + return new KllDirectLongsSketch(memVal.sketchStructure, (WritableMemory)srcMem, memReqSvr, memVal); + } else { + return new KllDirectCompactLongsSketch(memVal.sketchStructure, srcMem, memVal); + } + } + + //Factory to wrap a WritableMemory image + + /** + * Wrap a sketch around the given source Writable Memory containing sketch data + * that originated from this sketch. + * @param srcMem a WritableMemory that contains data. + * @param memReqSvr the given MemoryRequestServer to request a larger WritableMemory + * @return instance of this sketch + */ + public static KllLongsSketch writableWrap( + final WritableMemory srcMem, + final MemoryRequestServer memReqSvr) { + Objects.requireNonNull(srcMem, "Parameter 'srcMem' must not be null"); + Objects.requireNonNull(memReqSvr, "Parameter 'memReqSvr' must not be null"); + final KllMemoryValidate memVal = new KllMemoryValidate(srcMem, LONGS_SKETCH); + if (memVal.sketchStructure == UPDATABLE) { + return new KllDirectLongsSketch(UPDATABLE, srcMem, memReqSvr, memVal); + } else { + return new KllDirectCompactLongsSketch(memVal.sketchStructure, srcMem, memVal); + } + } + + //END of Constructors + + @Override + public double[] getCDF(final long[] splitPoints, final QuantileSearchCriteria searchCrit) { + if (isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); } + refreshSortedView(); + return longsSV.getCDF(splitPoints, searchCrit); + } + + @Override + public double[] getPMF(final long[] splitPoints, final QuantileSearchCriteria searchCrit) { + if (isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); } + refreshSortedView(); + return longsSV.getPMF(splitPoints, searchCrit); + } + + @Override + public long getQuantile(final double rank, final QuantileSearchCriteria searchCrit) { + if (isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); } + refreshSortedView(); + return longsSV.getQuantile(rank, searchCrit); + } + + @Override + public long[] getQuantiles(final double[] ranks, final QuantileSearchCriteria searchCrit) { + if (isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); } + refreshSortedView(); + final int len = ranks.length; + final long[] quantiles = new long[len]; + for (int i = 0; i < len; i++) { + quantiles[i] = longsSV.getQuantile(ranks[i], searchCrit); + } + return quantiles; + } + + /** + * {@inheritDoc} + * The approximate probability that the true quantile is within the confidence interval + * specified by the upper and lower quantile bounds for this sketch is 0.99. + */ + @Override + public long getQuantileLowerBound(final double rank) { + return getQuantile(max(0, rank - KllHelper.getNormalizedRankError(getMinK(), false))); + } + + /** + * {@inheritDoc} + * The approximate probability that the true quantile is within the confidence interval + * specified by the upper and lower quantile bounds for this sketch is 0.99. + */ + @Override + public long getQuantileUpperBound(final double rank) { + return getQuantile(min(1.0, rank + KllHelper.getNormalizedRankError(getMinK(), false))); + } + + @Override + public double getRank(final long quantile, final QuantileSearchCriteria searchCrit) { + if (isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); } + refreshSortedView(); + return longsSV.getRank(quantile, searchCrit); + } + + /** + * {@inheritDoc} + * The approximate probability that the true rank is within the confidence interval + * specified by the upper and lower rank bounds for this sketch is 0.99. + */ + @Override + public double getRankLowerBound(final double rank) { + return max(0.0, rank - KllHelper.getNormalizedRankError(getMinK(), false)); + } + + /** + * {@inheritDoc} + * The approximate probability that the true rank is within the confidence interval + * specified by the upper and lower rank bounds for this sketch is 0.99. + */ + @Override + public double getRankUpperBound(final double rank) { + return min(1.0, rank + KllHelper.getNormalizedRankError(getMinK(), false)); + } + + @Override + public double[] getRanks(final long[] quantiles, final QuantileSearchCriteria searchCrit) { + if (isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); } + refreshSortedView(); + final int len = quantiles.length; + final double[] ranks = new double[len]; + for (int i = 0; i < len; i++) { + ranks[i] = longsSV.getRank(quantiles[i], searchCrit); + } + return ranks; + } + + @Override + public QuantilesLongsSketchIterator iterator() { + return new KllLongsSketchIterator( + getLongItemsArray(), getLevelsArray(SketchStructure.UPDATABLE), getNumLevels()); + } + + @Override + public final void merge(final KllSketch other) { + if (readOnly || sketchStructure != UPDATABLE) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } + if (this == other) { throw new SketchesArgumentException(SELF_MERGE_MSG); } + final KllLongsSketch otherLngSk = (KllLongsSketch)other; + if (otherLngSk.isEmpty()) { return; } + KllLongsHelper.mergeLongsImpl(this, otherLngSk); + longsSV = null; + } + + /** + * {@inheritDoc} + * + *The parameter k will not change.
+ */ + @Override + public final void reset() { + if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } + final int k = getK(); + setN(0); + setMinK(k); + setNumLevels(1); + setLevelZeroSorted(false); + setLevelsArray(new int[] {k, k}); + setMinItem(Long.MAX_VALUE); + setMaxItem(Long.MIN_VALUE); + setLongItemsArray(new long[k]); + longsSV = null; + } + + @Override + public byte[] toByteArray() { + return KllHelper.toByteArray(this, false); + } + + @Override + public String toString(final boolean withLevels, final boolean withLevelsAndItems) { + KllSketch sketch = this; + if (withLevelsAndItems && sketchStructure != UPDATABLE) { + final Memory mem = getWritableMemory(); + assert mem != null; + sketch = KllLongsSketch.heapify(getWritableMemory()); + } + return KllHelper.toStringImpl(sketch, withLevels, withLevelsAndItems, getSerDe()); + } + + //SINGLE UPDATE + + @Override + public void update(final long item) { + // Align with KllDoublesSketch + if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } + updateLong(this, item); + longsSV = null; + } + + //Also Called from KllLongsHelper::merge + static void updateLong(final KllLongsSketch lngSk, final long item) { + lngSk.updateMinMax(item); + int freeSpace = lngSk.levelsArr[0]; + assert (freeSpace >= 0); + if (freeSpace == 0) { + KllLongsHelper.compressWhileUpdatingSketch(lngSk); + freeSpace = lngSk.levelsArr[0]; + assert (freeSpace > 0); + } + lngSk.incN(1); + lngSk.setLevelZeroSorted(false); + final int nextPos = freeSpace - 1; + lngSk.setLevelsArrayAt(0, nextPos); + lngSk.setLongItemsArrayAt(nextPos, item); + } + + /** + * Single update of min and max + * @param item the source item, it must not be a NaN. + */ + final void updateMinMax(final long item) { + if (isEmpty()) { + setMinItem(item); + setMaxItem(item); + } else { + setMinItem(min(getMinItemInternal(), item)); + setMaxItem(max(getMaxItemInternal(), item)); + } + } + + //WEIGHTED UPDATE + + /** + * Weighted update. Updates this sketch with the given item the number of times specified by the given integer weight. + * @param item the item to be repeated. NaNs are ignored. + * @param weight the number of times the update of item is to be repeated. It must be ≥ one. + */ + public void update(final long item, final long weight) { + // + if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } + if (weight < 1L) { throw new SketchesArgumentException("Weight is less than one."); } + if (weight == 1L) { updateLong(this, item); } + else { + if (weight < levelsArr[0]) { + for (int i = 0; i < (int)weight; i++) { updateLong(this, item); } + } else { + final KllHeapLongsSketch tmpSk = new KllHeapLongsSketch(getK(), DEFAULT_M, item, weight); + merge(tmpSk); + } + } + longsSV = null; + } + + // VECTOR UPDATE + + /** + * Vector update. Updates this sketch with the given array (vector) of items, starting at the items + * offset for a length number of items. This is not supported for direct sketches. + * @param items the vector of items + * @param offset the starting index of the items[] array + * @param length the number of items + */ + public void update(final long[] items, final int offset, final int length) { + if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } + if (length == 0) { return; } + updateLong(items, offset, length); + longsSV = null; + } + /* Align with KllDoublesSketch + + + + + + + + + + + + + */ + private void updateLong(final long[] srcItems, final int srcOffset, final int length) { + if (isEmpty()) { + setMinItem(srcItems[srcOffset]); //initialize with a real value + setMaxItem(srcItems[srcOffset]); + } + + int count = 0; + while (count < length) { + if (levelsArr[0] == 0) { + KllLongsHelper.compressWhileUpdatingSketch(this); + } + final int spaceNeeded = length - count; + final int freeSpace = levelsArr[0]; + assert (freeSpace > 0); + final int numItemsToCopy = min(spaceNeeded, freeSpace); + final int dstOffset = freeSpace - numItemsToCopy; + final int localSrcOffset = srcOffset + count; + setLongItemsArrayAt(dstOffset, srcItems, localSrcOffset, numItemsToCopy); + updateMinMax(srcItems, localSrcOffset, numItemsToCopy); + count += numItemsToCopy; + incN(numItemsToCopy); + setLevelsArrayAt(0, dstOffset); + } + setLevelZeroSorted(false); + } + + /** + * Vector update of min and max. + * @param srcItems the input source array of values, no NaNs allowed. + * @param srcOffset the starting offset in srcItems + * @param length the number of items to update min and max + */ + private void updateMinMax(final long[] srcItems, final int srcOffset, final int length) { + final int end = srcOffset + length; + for (int i = srcOffset; i < end; i++) { + setMinItem(min(getMinItemInternal(), srcItems[i])); + setMaxItem(max(getMaxItemInternal(), srcItems[i])); + } + } + /* Align with KllDoublesSketch + + + + + + + + + */ + // END ALL UPDATE METHODS + + /** + * @return full size of internal items array including empty space at bottom. + */ + abstract long[] getLongItemsArray(); + + /** + * @return items array of retained items. + */ + abstract long[] getLongRetainedItemsArray(); + + abstract long getLongSingleItem(); + + // Min & Max Methods + + abstract long getMaxItemInternal(); + + abstract void setMaxItem(long item); + + abstract long getMinItemInternal(); + + abstract void setMinItem(long item); + + @Override + abstract byte[] getMinMaxByteArr(); + + @Override + int getMinMaxSizeBytes() { + return Long.BYTES * 2; + } + + //END Min & Max Methods + + @Override + abstract byte[] getRetainedItemsByteArr(); + + @Override + int getRetainedItemsSizeBytes() { + return getNumRetained() * Long.BYTES; + } + + @Override + ArrayOfItemsSerDe getSerDe() { return null; } + + @Override + final byte[] getSingleItemByteArr() { + final byte[] bytes = new byte[ITEM_BYTES]; + putLongLE(bytes, 0, getLongSingleItem()); + return bytes; + } + + @Override + int getSingleItemSizeBytes() { + return Long.BYTES; + } + + @Override + abstract byte[] getTotalItemsByteArr(); + + @Override + int getTotalItemsNumBytes() { + return levelsArr[getNumLevels()] * Long.BYTES; + } + + abstract void setLongItemsArray(long[] longItems); + + abstract void setLongItemsArrayAt(int index, long item); + + abstract void setLongItemsArrayAt(int dstIndex, long[] srcItems, int srcOffset, int length); + + // SORTED VIEW + + @Override + @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "OK in this case.") + public LongsSketchSortedView getSortedView() { + refreshSortedView(); + return longsSV; + } + + private final LongsSketchSortedView refreshSortedView() { + if (longsSV == null) { + final CreateSortedView csv = new CreateSortedView(); + longsSV = csv.getSV(); + } + return longsSV; + } + + private final class CreateSortedView { + long[] quantiles; + long[] cumWeights; + + LongsSketchSortedView getSV() { + if (isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); } + final long[] srcQuantiles = getLongItemsArray(); + final int[] srcLevels = levelsArr; + final int srcNumLevels = getNumLevels(); + + if (!isLevelZeroSorted()) { + Arrays.sort(srcQuantiles, srcLevels[0], srcLevels[1]); + if (!hasMemory()) { setLevelZeroSorted(true); } + //we don't sort level0 in Memory, only our copy. + } + final int numQuantiles = getNumRetained(); + quantiles = new long[numQuantiles]; + cumWeights = new long[numQuantiles]; + populateFromSketch(srcQuantiles, srcLevels, srcNumLevels, numQuantiles); + return new LongsSketchSortedView( + quantiles, cumWeights, KllLongsSketch.this); + } + + private void populateFromSketch(final long[] srcQuantiles, final int[] srcLevels, + final int srcNumLevels, final int numItems) { + final int[] myLevels = new int[srcNumLevels + 1]; + final int offset = srcLevels[0]; + System.arraycopy(srcQuantiles, offset, quantiles, 0, numItems); + int srcLevel = 0; + int dstLevel = 0; + long weight = 1; + while (srcLevel < srcNumLevels) { + final int fromIndex = srcLevels[srcLevel] - offset; + final int toIndex = srcLevels[srcLevel + 1] - offset; // exclusive + if (fromIndex < toIndex) { // if equal, skip empty level + Arrays.fill(cumWeights, fromIndex, toIndex, weight); + myLevels[dstLevel] = fromIndex; + myLevels[dstLevel + 1] = toIndex; + dstLevel++; + } + srcLevel++; + weight *= 2; + } + final int numLevels = dstLevel; + blockyTandemMergeSort(quantiles, cumWeights, myLevels, numLevels); //create unit weights + KllHelper.convertToCumulative(cumWeights); + } + } //End of class CreateSortedView + + private static void blockyTandemMergeSort(final long[] quantiles, final long[] weights, + final int[] levels, final int numLevels) { + if (numLevels == 1) { return; } + + // duplicate the input in preparation for the "ping-pong" copy reduction strategy. + final long[] quantilesTmp = Arrays.copyOf(quantiles, quantiles.length); + final long[] weightsTmp = Arrays.copyOf(weights, quantiles.length); // don't need the extra one + + blockyTandemMergeSortRecursion(quantilesTmp, weightsTmp, quantiles, weights, levels, 0, numLevels); + } + + private static void blockyTandemMergeSortRecursion( + final long[] quantilesSrc, final long[] weightsSrc, + final long[] quantilesDst, final long[] weightsDst, + final int[] levels, final int startingLevel, final int numLevels) { + if (numLevels == 1) { return; } + final int numLevels1 = numLevels / 2; + final int numLevels2 = numLevels - numLevels1; + assert numLevels1 >= 1; + assert numLevels2 >= numLevels1; + final int startingLevel1 = startingLevel; + final int startingLevel2 = startingLevel + numLevels1; + // swap roles of src and dst + blockyTandemMergeSortRecursion( + quantilesDst, weightsDst, + quantilesSrc, weightsSrc, + levels, startingLevel1, numLevels1); + blockyTandemMergeSortRecursion( + quantilesDst, weightsDst, + quantilesSrc, weightsSrc, + levels, startingLevel2, numLevels2); + tandemMerge( + quantilesSrc, weightsSrc, + quantilesDst, weightsDst, + levels, + startingLevel1, numLevels1, + startingLevel2, numLevels2); + } + + private static void tandemMerge( + final long[] quantilesSrc, final long[] weightsSrc, + final long[] quantilesDst, final long[] weightsDst, + final int[] levelStarts, + final int startingLevel1, final int numLevels1, + final int startingLevel2, final int numLevels2) { + final int fromIndex1 = levelStarts[startingLevel1]; + final int toIndex1 = levelStarts[startingLevel1 + numLevels1]; // exclusive + final int fromIndex2 = levelStarts[startingLevel2]; + final int toIndex2 = levelStarts[startingLevel2 + numLevels2]; // exclusive + int iSrc1 = fromIndex1; + int iSrc2 = fromIndex2; + int iDst = fromIndex1; + + while (iSrc1 < toIndex1 && iSrc2 < toIndex2) { + if (quantilesSrc[iSrc1] < quantilesSrc[iSrc2]) { + quantilesDst[iDst] = quantilesSrc[iSrc1]; + weightsDst[iDst] = weightsSrc[iSrc1]; + iSrc1++; + } else { + quantilesDst[iDst] = quantilesSrc[iSrc2]; + weightsDst[iDst] = weightsSrc[iSrc2]; + iSrc2++; + } + iDst++; + } + if (iSrc1 < toIndex1) { + System.arraycopy(quantilesSrc, iSrc1, quantilesDst, iDst, toIndex1 - iSrc1); + System.arraycopy(weightsSrc, iSrc1, weightsDst, iDst, toIndex1 - iSrc1); + } else if (iSrc2 < toIndex2) { + System.arraycopy(quantilesSrc, iSrc2, quantilesDst, iDst, toIndex2 - iSrc2); + System.arraycopy(weightsSrc, iSrc2, weightsDst, iDst, toIndex2 - iSrc2); + } + } + + // END SORTED VIEW + +} diff --git a/src/main/java/org/apache/datasketches/kll/KllLongsSketchIterator.java b/src/main/java/org/apache/datasketches/kll/KllLongsSketchIterator.java new file mode 100644 index 000000000..8922ec70e --- /dev/null +++ b/src/main/java/org/apache/datasketches/kll/KllLongsSketchIterator.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.kll; + +import org.apache.datasketches.quantilescommon.QuantilesLongsSketchIterator; + +/** + * Iterator over KllLongsSketch. The order is not defined. + */ +public final class KllLongsSketchIterator extends KllSketchIterator implements QuantilesLongsSketchIterator { + private final long[] quantiles; + + KllLongsSketchIterator(final long[] quantiles, final int[] levelsArr, final int numLevels) { + super(levelsArr, numLevels); + this.quantiles = quantiles; + } + + @Override + public long getQuantile() { + return quantiles[index]; + } + +} diff --git a/src/main/java/org/apache/datasketches/kll/KllPreambleUtil.java b/src/main/java/org/apache/datasketches/kll/KllPreambleUtil.java index 18e4de3f0..f2c3847ae 100644 --- a/src/main/java/org/apache/datasketches/kll/KllPreambleUtil.java +++ b/src/main/java/org/apache/datasketches/kll/KllPreambleUtil.java @@ -28,6 +28,7 @@ import static org.apache.datasketches.kll.KllSketch.SketchType.DOUBLES_SKETCH; import static org.apache.datasketches.kll.KllSketch.SketchType.FLOATS_SKETCH; import static org.apache.datasketches.kll.KllSketch.SketchType.ITEMS_SKETCH; +import static org.apache.datasketches.kll.KllSketch.SketchType.LONGS_SKETCH; import java.util.Objects; @@ -210,7 +211,7 @@ static String toString(final Memory mem, final SketchType sketchType, final bool * Used primarily in testing. * * @param mem the given Memory - * @param sketchType the sketch type: FLOATS_SKETCH, DOUBLES_SKETCH, or ITEMS_SKETCH. + * @param sketchType the sketch type: FLOATS_SKETCH, DOUBLES_SKETCH, LONGS_SKETCH, or ITEMS_SKETCH. * @param includeData if true, includes detail of retained data. * @param serDe must be supplied for KllItemsSketch, otherwise can be null. * @return the summary string. @@ -226,10 +227,9 @@ staticThe parameter k will not change.
*/ @Override diff --git a/src/main/java/org/apache/datasketches/quantiles/UpdateDoublesSketch.java b/src/main/java/org/apache/datasketches/quantiles/UpdateDoublesSketch.java index 155001c87..f7191402d 100644 --- a/src/main/java/org/apache/datasketches/quantiles/UpdateDoublesSketch.java +++ b/src/main/java/org/apache/datasketches/quantiles/UpdateDoublesSketch.java @@ -23,6 +23,7 @@ import org.apache.datasketches.memory.WritableMemory; /** + * Extends DoubleSketch * @author Jon Malkin */ public abstract class UpdateDoublesSketch extends DoublesSketch { @@ -49,11 +50,19 @@ public static UpdateDoublesSketch wrap(final WritableMemory srcMem) { @Override public abstract void update(double item); + /** + * Factory heapify takes a compact sketch image in Memory and instantiates an on-heap sketch. + * The resulting sketch will not retain any link to the source Memory. + * @param srcMem a compact Memory image of a sketch serialized by this sketch. + * See Memory + * @return a heap-based sketch based on the given Memory. + */ public static UpdateDoublesSketch heapify(final Memory srcMem) { return HeapUpdateDoublesSketch.heapifyInstance(srcMem); } /** + * Returns a CompactDoublesSketch of this class * @return a CompactDoublesSketch of this class */ public CompactDoublesSketch compact() { diff --git a/src/main/java/org/apache/datasketches/quantiles/package-info.java b/src/main/java/org/apache/datasketches/quantiles/package-info.java index c44e2e316..8767757b7 100644 --- a/src/main/java/org/apache/datasketches/quantiles/package-info.java +++ b/src/main/java/org/apache/datasketches/quantiles/package-info.java @@ -18,9 +18,8 @@ */ /** - *The quantiles package contains stochastic streaming algorithms that enable single-pass + * The quantiles package contains stochastic streaming algorithms that enable single-pass * analysis of the distribution of a stream of quantiles. - *
* * @see org.apache.datasketches.quantiles.DoublesSketch * @see org.apache.datasketches.quantiles.ItemsSketch diff --git a/src/main/java/org/apache/datasketches/quantilescommon/DoublesSortedView.java b/src/main/java/org/apache/datasketches/quantilescommon/DoublesSortedView.java index bdc3cc75c..1427f6279 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/DoublesSortedView.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/DoublesSortedView.java @@ -38,7 +38,7 @@ public interface DoublesSortedView extends SortedView { * @param splitPoints an array of m unique, monotonically increasing items * (of the same type as the input items) * that divide the item input domain into m+1 overlapping intervals. - * + **The start of each interval is below the lowest item retained by the sketch * corresponding to a zero rank or zero probability, and the end of the interval * is the rank or cumulative probability corresponding to the split point.
@@ -55,7 +55,7 @@ public interface DoublesSortedView extends SortedView { *
It is not recommended to include either the minimum or maximum items of the input stream.
- * + * * @param searchCrit the desired search criteria. * @return a discrete CDF array of m+1 double ranks (or cumulative probabilities) on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. @@ -100,7 +100,7 @@ default double[] getCDF(double[] splitPoints, QuantileSearchCriteria searchCrit) * @param splitPoints an array of m unique, monotonically increasing items * (of the same type as the input items) * that divide the item input domain into m+1 consecutive, non-overlapping intervals. - * + **Each interval except for the end intervals starts with a split point and ends with the next split * point in sequence.
* @@ -124,7 +124,7 @@ default double[] getCDF(double[] splitPoints, QuantileSearchCriteria searchCrit) *
It is not recommended to include either the minimum or maximum items of the input stream.
- * + * * @param searchCrit the desired search criteria. * @return a PMF array of m+1 probability masses as doubles on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. @@ -148,7 +148,7 @@ default double[] getPMF(double[] splitPoints, QuantileSearchCriteria searchCrit * the quantile directly corresponding to the given rank. * @return the approximate quantile given the normalized rank. * @throws IllegalArgumentException if sketch is empty. - * @see org.apache.datasketches.quantilescommon.QuantileSearchCriteria + * @see QuantileSearchCriteria */ double getQuantile(double rank, QuantileSearchCriteria searchCrit); @@ -165,7 +165,7 @@ default double[] getPMF(double[] splitPoints, QuantileSearchCriteria searchCrit * @param searchCrit if INCLUSIVE the given quantile is included into the rank. * @return the normalized rank corresponding to the given quantile. * @throws IllegalArgumentException if sketch is empty. - * @see org.apache.datasketches.quantilescommon.QuantileSearchCriteria + * @see QuantileSearchCriteria */ double getRank(double quantile, QuantileSearchCriteria searchCrit); @@ -173,4 +173,3 @@ default double[] getPMF(double[] splitPoints, QuantileSearchCriteria searchCrit DoublesSortedViewIterator iterator(); } - diff --git a/src/main/java/org/apache/datasketches/quantilescommon/FloatsSortedView.java b/src/main/java/org/apache/datasketches/quantilescommon/FloatsSortedView.java index 0a0c54b5a..eec699d94 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/FloatsSortedView.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/FloatsSortedView.java @@ -38,7 +38,7 @@ public interface FloatsSortedView extends SortedView { * @param splitPoints an array of m unique, monotonically increasing items * (of the same type as the input items) * that divide the item input domain into m+1 overlapping intervals. - * + **The start of each interval is below the lowest item retained by the sketch * corresponding to a zero rank or zero probability, and the end of the interval * is the rank or cumulative probability corresponding to the split point.
@@ -55,7 +55,7 @@ public interface FloatsSortedView extends SortedView { *
It is not recommended to include either the minimum or maximum items of the input stream.
- * + * * @param searchCrit the desired search criteria. * @return a discrete CDF array of m+1 double ranks (or cumulative probabilities) on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. @@ -100,7 +100,7 @@ default double[] getCDF(float[] splitPoints, QuantileSearchCriteria searchCrit) * @param splitPoints an array of m unique, monotonically increasing items * (of the same type as the input items) * that divide the item input domain into m+1 consecutive, non-overlapping intervals. - * + *** @param searchCrit the desired search criteria. * @return a PMF array of m+1 probability masses as doubles on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. diff --git a/src/main/java/org/apache/datasketches/quantilescommon/GenericInequalitySearch.java b/src/main/java/org/apache/datasketches/quantilescommon/GenericInequalitySearch.java index e650fef9b..7b9d6d665 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/GenericInequalitySearch.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/GenericInequalitySearch.java @@ -41,7 +41,7 @@ *Each interval except for the end intervals starts with a split point and ends with the next split * point in sequence.
* @@ -124,7 +124,7 @@ default double[] getCDF(float[] splitPoints, QuantileSearchCriteria searchCrit) * * *It is not recommended to include either the minimum or maximum items of the input stream.
- * + *
Given a sorted array of values arr[] and a search key value v, the algorithms for * the searching criteria are given with each enum criterion.
* - * @see + * @see * Sketching Quantiles and Ranks Tutorial * @author Lee Rhodes */ diff --git a/src/main/java/org/apache/datasketches/quantilescommon/GenericSortedView.java b/src/main/java/org/apache/datasketches/quantilescommon/GenericSortedView.java index 0d27ce78c..f0dc81151 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/GenericSortedView.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/GenericSortedView.java @@ -47,7 +47,7 @@ public interface GenericSortedView** @param searchCrit the desired search criteria. * @return a discrete CDF array of m+1 double ranks (or cumulative probabilities) on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. @@ -116,7 +116,7 @@ default double[] getCDF(final T[] splitPoints, final QuantileSearchCriteria sear * @param splitPoints an array of m unique, monotonically increasing items * (of the same type as the input items) * that divide the item input domain into m+1 consecutive, non-overlapping intervals. - * + *The start of each interval is below the lowest item retained by the sketch * corresponding to a zero rank or zero probability, and the end of the interval * is the rank or cumulative probability corresponding to the split point.
@@ -64,7 +64,7 @@ public interface GenericSortedViewextends PartitioningFeature , SketchPar * * * It is not recommended to include either the minimum or maximum items of the input stream.
- * + *
** @param searchCrit the desired search criteria. * @return a PMF array of m+1 probability masses as doubles on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. diff --git a/src/main/java/org/apache/datasketches/quantilescommon/IncludeMinMax.java b/src/main/java/org/apache/datasketches/quantilescommon/IncludeMinMax.java index 203e338d2..1edfc2054 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/IncludeMinMax.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/IncludeMinMax.java @@ -63,6 +63,24 @@ public FloatsPair(final float[] quantiles, final long[] cumWeights) { } } + /** A simple structure to hold a pair of arrays */ + public static class LongsPair { + /** the array of quantiles */ + public long[] quantiles; + /** the array of associated cumulative weights */ + public long[] cumWeights; + + /** + * Constructor. + * @param quantiles the array of quantiles + * @param cumWeights the array of associated cumulative weights + */ + public LongsPair(final long[] quantiles, final long[] cumWeights) { + this.quantiles = quantiles; + this.cumWeights = cumWeights; + } + } + /** * A simple structure to hold a pair of arrays * @paramEach interval except for the end intervals starts with a split point and ends with the next split * point in sequence.
* @@ -140,7 +140,7 @@ default double[] getCDF(final T[] splitPoints, final QuantileSearchCriteria sear * * *It is not recommended to include either the minimum or maximum items of the input stream.
- * + *
Given a sorted array of values arr[] and a search key value v, the algorithms for * the searching criteria are given with each enum criterion.
* - * @see + * @see * Sketching Quantiles and Ranks Tutorial * @author Lee Rhodes */ diff --git a/src/main/java/org/apache/datasketches/quantilescommon/LongsSketchSortedView.java b/src/main/java/org/apache/datasketches/quantilescommon/LongsSketchSortedView.java new file mode 100644 index 000000000..efb4006f6 --- /dev/null +++ b/src/main/java/org/apache/datasketches/quantilescommon/LongsSketchSortedView.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.quantilescommon; + +import static org.apache.datasketches.quantilescommon.QuantileSearchCriteria.INCLUSIVE; +import static org.apache.datasketches.quantilescommon.QuantilesAPI.EMPTY_MSG; +import static org.apache.datasketches.quantilescommon.QuantilesUtil.getNaturalRank; + +import org.apache.datasketches.common.SketchesArgumentException; +import org.apache.datasketches.quantilescommon.IncludeMinMax.LongsPair; + +/** + * The SortedView of the KllLongsSketch. + * @author Lee Rhodes + * @author Zac Blanco + */ +public final class LongsSketchSortedView implements LongsSortedView { + private final long[] quantiles; + private final long[] cumWeights; //cumulative natural weights + private final long totalN; + + /** + * Construct from elements, also used in testing. + * @param quantiles sorted array of quantiles + * @param cumWeights sorted, monotonically increasing cumulative weights. + * @param sk the underlying quantile sketch. + */ + public LongsSketchSortedView( + final long[] quantiles, + final long[] cumWeights, + final QuantilesLongsAPI sk) { + final LongsPair dPair = + IncludeMinMax.includeLongsMinMax(quantiles, cumWeights, sk.getMaxItem(), sk.getMinItem()); + this.quantiles = dPair.quantiles; + this.cumWeights = dPair.cumWeights; + this.totalN = sk.getN(); + } + + //Used for testing + LongsSketchSortedView( + final long[] quantiles, + final long[] cumWeights, + final long totalN, + final long maxItem, + final long minItem) { + final LongsPair dPair = + IncludeMinMax.includeLongsMinMax(quantiles, cumWeights, maxItem, minItem); + this.quantiles = dPair.quantiles; + this.cumWeights = dPair.cumWeights; + this.totalN = totalN; + } + + @Override + public long[] getCumulativeWeights() { + return cumWeights.clone(); + } + + @Override + public long getMaxItem() { + final int top = quantiles.length - 1; + return quantiles[top]; + } + + @Override + public long getMinItem() { + return quantiles[0]; + } + + @Override + public long getN() { + return totalN; + } + + @Override + public int getNumRetained() { + return quantiles.length; + } + + @Override + public long getQuantile(final double rank, final QuantileSearchCriteria searchCrit) { + if (isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); } + QuantilesUtil.checkNormalizedRankBounds(rank); + final int len = cumWeights.length; + final double naturalRank = getNaturalRank(rank, totalN, searchCrit); + final InequalitySearch crit = (searchCrit == INCLUSIVE) ? InequalitySearch.GE : InequalitySearch.GT; + final int index = InequalitySearch.find(cumWeights, 0, len - 1, naturalRank, crit); + if (index == -1) { + return quantiles[len - 1]; //EXCLUSIVE (GT) case: normRank == 1.0; + } + return quantiles[index]; + } + + @Override + public long[] getQuantiles() { + return quantiles.clone(); + } + + @Override + public double getRank(final long quantile, final QuantileSearchCriteria searchCrit) { + if (isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); } + final int len = quantiles.length; + final InequalitySearch crit = (searchCrit == INCLUSIVE) ? InequalitySearch.LE : InequalitySearch.LT; + final int index = InequalitySearch.find(quantiles, 0, len - 1, quantile, crit); + if (index == -1) { + return 0; //EXCLUSIVE (LT) case: quantile <= minQuantile; INCLUSIVE (LE) case: quantile < minQuantile + } + return (double)cumWeights[index] / totalN; + } + + @Override + public boolean isEmpty() { + return totalN == 0; + } + + @Override + public LongsSortedViewIterator iterator() { + return new LongsSortedViewIterator(quantiles, cumWeights); + } + +} diff --git a/src/main/java/org/apache/datasketches/quantilescommon/LongsSortedView.java b/src/main/java/org/apache/datasketches/quantilescommon/LongsSortedView.java new file mode 100644 index 000000000..e7e3521c7 --- /dev/null +++ b/src/main/java/org/apache/datasketches/quantilescommon/LongsSortedView.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.quantilescommon; + +/** + * The Sorted View for quantile sketches of primitive type long. + * @see SortedView + * @author Lee Rhodes + * @author Zac Blanco + */ +public interface LongsSortedView extends SortedView { + + /** + * Returns an approximation to the Cumulative Distribution Function (CDF) of the input stream + * as a monotonically increasing array of double ranks (or cumulative probabilities) on the interval [0.0, 1.0], + * given a set of splitPoints. + * + *The resulting approximations have a probabilistic guarantee that can be obtained from the + * getNormalizedRankError(false) function.
+ * + * @param splitPoints an array of m unique, monotonically increasing items + * (of the same type as the input items) + * that divide the item input domain into m+1 overlapping intervals. + *+ *+ * @param searchCrit the desired search criteria. + * @return a discrete CDF array of m+1 double ranks (or cumulative probabilities) on the interval [0.0, 1.0]. + * @throws IllegalArgumentException if sketch is empty. + */ + default double[] getCDF(long[] splitPoints, QuantileSearchCriteria searchCrit) { + QuantilesUtil.checkLongsSplitPointsOrder(splitPoints); + final int len = splitPoints.length + 1; + final double[] buckets = new double[len]; + for (int i = 0; i < len - 1; i++) { + buckets[i] = getRank(splitPoints[i], searchCrit); + } + buckets[len - 1] = 1; + return buckets; + } + + /** + * Returns the maximum item of the stream. This may be distinct from the largest item retained by the + * sketch algorithm. + * + * @return the maximum item of the stream + * @throws IllegalArgumentException if sketch is empty. + */ + long getMaxItem(); + + /** + * Returns the minimum item of the stream. This may be distinct from the smallest item retained by the + * sketch algorithm. + * + * @return the minimum item of the stream + * @throws IllegalArgumentException if sketch is empty. + */ + long getMinItem(); + + /** + * Returns an approximation to the Probability Mass Function (PMF) of the input stream + * as an array of probability masses as doubles on the interval [0.0, 1.0], + * given a set of splitPoints. + * + *The start of each interval is below the lowest item retained by the sketch + * corresponding to a zero rank or zero probability, and the end of the interval + * is the rank or cumulative probability corresponding to the split point.
+ * + *The (m+1)th interval represents 100% of the distribution represented by the sketch + * and consistent with the definition of a cumulative probability distribution, thus the (m+1)th + * rank or probability in the returned array is always 1.0.
+ * + *If a split point exactly equals a retained item of the sketch and the search criterion is:
+ * + *+ *
+ * + *- INCLUSIVE, the resulting cumulative probability will include that item.
+ *- EXCLUSIVE, the resulting cumulative probability will not include the weight of that split point.
+ *It is not recommended to include either the minimum or maximum items of the input stream.
+ *
The resulting approximations have a probabilistic guarantee that can be obtained from the + * getNormalizedRankError(true) function.
+ * + * @param splitPoints an array of m unique, monotonically increasing items + * (of the same type as the input items) + * that divide the item input domain into m+1 consecutive, non-overlapping intervals. + *+ *+ * @param searchCrit the desired search criteria. + * @return a PMF array of m+1 probability masses as doubles on the interval [0.0, 1.0]. + * @throws IllegalArgumentException if sketch is empty. + */ + default double[] getPMF(long[] splitPoints, QuantileSearchCriteria searchCrit) { + final double[] buckets = getCDF(splitPoints, searchCrit); + final int len = buckets.length; + for (int i = len; i-- > 1; ) { + buckets[i] -= buckets[i - 1]; + } + return buckets; + } + + /** + * Gets the approximate quantile of the given normalized rank and the given search criterion. + * + * @param rank the given normalized rank, a double in the range [0.0, 1.0]. + * @param searchCrit If INCLUSIVE, the given rank includes all quantiles ≤ + * the quantile directly corresponding to the given rank. + * If EXCLUSIVE, he given rank includes all quantiles < + * the quantile directly corresponding to the given rank. + * @return the approximate quantile given the normalized rank. + * @throws IllegalArgumentException if sketch is empty. + * @see QuantileSearchCriteria + */ + long getQuantile(double rank, QuantileSearchCriteria searchCrit); + + /** + * Returns an array of all retained quantiles by the sketch. + * @return an array of all retained quantiles by the sketch. + */ + long[] getQuantiles(); + + /** + * Gets the normalized rank corresponding to the given a quantile. + * + * @param quantile the given quantile + * @param searchCrit if INCLUSIVE the given quantile is included into the rank. + * @return the normalized rank corresponding to the given quantile. + * @throws IllegalArgumentException if sketch is empty. + * @see QuantileSearchCriteria + */ + double getRank(long quantile, QuantileSearchCriteria searchCrit); + + @Override + LongsSortedViewIterator iterator(); + +} diff --git a/src/main/java/org/apache/datasketches/quantilescommon/LongsSortedViewIterator.java b/src/main/java/org/apache/datasketches/quantilescommon/LongsSortedViewIterator.java new file mode 100644 index 000000000..77510cd5a --- /dev/null +++ b/src/main/java/org/apache/datasketches/quantilescommon/LongsSortedViewIterator.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.quantilescommon; + +/** + * Iterator over quantile sketches of primitive type long. + */ +public final class LongsSortedViewIterator extends SortedViewIterator { + private final long[] quantiles; + + /** + * Constructor. + * @param quantiles the given array of quantiles, which must be ordered. + * @param cumWeights the given array of cumulative weights, which must be ordered, start with the value one, and + * the last value must be equal to N, the total number of items updated to the sketch. + */ + public LongsSortedViewIterator(final long[] quantiles, final long[] cumWeights) { + super(cumWeights); + this.quantiles = quantiles; //SpotBugs EI_EXPOSE_REP2 suppressed by FindBugsExcludeFilter + } + + /** + * Gets the quantile at the current index. + * + *Each interval except for the end intervals starts with a split point and ends with the next split + * point in sequence.
+ * + *The first interval starts below the lowest item retained by the sketch + * corresponding to a zero rank or zero probability, and ends with the first split point
+ * + *The last (m+1)th interval starts with the last split point and ends after the last + * item retained by the sketch corresponding to a rank or probability of 1.0.
+ * + *The sum of the probability masses of all (m+1) intervals is 1.0.
+ * + *If the search criterion is:
+ * + *+ *
+ * + *- INCLUSIVE, and the upper split point of an interval equals an item retained by the sketch, the interval + * will include that item. If the lower split point equals an item retained by the sketch, the interval will exclude + * that item.
+ *- EXCLUSIVE, and the upper split point of an interval equals an item retained by the sketch, the interval + * will exclude that item. If the lower split point equals an item retained by the sketch, the interval will include + * that item.
+ *It is not recommended to include either the minimum or maximum items of the input stream.
+ *
Don't call this before calling next() for the first time + * or after getting false from next().
+ * + * @return the quantile at the current index. + */ + public long getQuantile() { + return quantiles[index]; + } + +} diff --git a/src/main/java/org/apache/datasketches/quantilescommon/QuantileSearchCriteria.java b/src/main/java/org/apache/datasketches/quantilescommon/QuantileSearchCriteria.java index 8ea3c3415..e7b9e6ef6 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/QuantileSearchCriteria.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/QuantileSearchCriteria.java @@ -22,7 +22,7 @@ /** * These search criteria are used by the KLL, REQ and Classic Quantiles sketches in the DataSketches library. * - * @see + * @see * Sketching Quantiles and Ranks Tutorial * * @author Lee Rhodes diff --git a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java index b70843bb4..471ee5570 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java @@ -20,12 +20,12 @@ package org.apache.datasketches.quantilescommon; /** - *This is a stochastic streaming sketch that enables near-real time analysis of the + * This is a stochastic streaming sketch that enables near-real time analysis of the * approximate distribution of items from a very large stream in a single pass, requiring only * that the items are comparable. * The analysis is obtained using the getQuantile() function or the * inverse functions getRank(), getPMF() (the Probability Mass Function), and getCDF() - * (the Cumulative Distribution Function).
+ * (the Cumulative Distribution Function). * *Given an input stream of N items, the natural rank of any specific * item is defined as its index (1 to N) in the hypothetical sorted stream of all @@ -194,7 +194,7 @@ *
[*] Note that obtaining epsilon may require using a similar function but with more parameters * based on the specific sketch implementation.
* - * @see + * @see * Sketching Quantiles and Ranks, Tutorial * @see org.apache.datasketches.quantilescommon.QuantileSearchCriteria * @@ -202,14 +202,25 @@ * @author Kevin Lang * @author Alexander Saydakov */ -@SuppressWarnings("javadoc") + public interface QuantilesAPI { + /** The sketch must not be empty for this operation. */ static String EMPTY_MSG = "The sketch must not be empty for this operation. "; + + /** Unsupported operation for this Sketch Type. */ static String UNSUPPORTED_MSG = "Unsupported operation for this Sketch Type. "; + + /** Sketch does not have just one item. */ static String NOT_SINGLE_ITEM_MSG = "Sketch does not have just one item. "; + + /** MemoryRequestServer must not be null. */ static String MEM_REQ_SVR_NULL_MSG = "MemoryRequestServer must not be null. "; + + /** Target sketch is Read Only, cannot write. */ static String TGT_IS_READ_ONLY_MSG = "Target sketch is Read Only, cannot write. "; + + /** A sketch cannot merge with itself. */ static String SELF_MERGE_MSG = "A sketch cannot merge with itself. "; /** diff --git a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesDoublesAPI.java b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesDoublesAPI.java index e8e5310f5..8c4f6620f 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesDoublesAPI.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesDoublesAPI.java @@ -50,7 +50,7 @@ default double[] getCDF(double[] splitPoints) { * @param splitPoints an array of m unique, monotonically increasing items * (of the same type as the input items) * that divide the item input domain into m+1 overlapping intervals. - * + *** @param searchCrit the desired search criteria. * @return a discrete CDF array of m+1 double ranks (or cumulative probabilities) on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. @@ -113,7 +113,7 @@ default double[] getPMF(double[] splitPoints) { * @param splitPoints an array of m unique, monotonically increasing items * (of the same type as the input items) * that divide the item input domain into m+1 consecutive, non-overlapping intervals. - * + *The start of each interval is below the lowest item retained by the sketch * corresponding to a zero rank or zero probability, and the end of the interval * is the rank or cumulative probability corresponding to the split point.
@@ -67,7 +67,7 @@ default double[] getCDF(double[] splitPoints) { * * *It is not recommended to include either the minimum or maximum items of the input stream.
- * + *
** @param searchCrit the desired search criteria. * @return a PMF array of m+1 probability masses as doubles on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. diff --git a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesFloatsAPI.java b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesFloatsAPI.java index 986780444..8b8a91bdd 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesFloatsAPI.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesFloatsAPI.java @@ -49,7 +49,7 @@ default double[] getCDF(float[] splitPoints) { * @param splitPoints an array of m unique, monotonically increasing items * (of the same type as the input items) * that divide the item input domain into m+1 overlapping intervals. - * + *Each interval except for the end intervals starts with a split point and ends with the next split * point in sequence.
* @@ -137,7 +137,7 @@ default double[] getPMF(double[] splitPoints) { * * *It is not recommended to include either the minimum or maximum items of the input stream.
- * + *
** @param searchCrit the desired search criteria. * @return a discrete CDF array of m+1 double ranks (or cumulative probabilities) on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. @@ -112,7 +112,7 @@ default double[] getPMF(float[] splitPoints) { * @param splitPoints an array of m unique, monotonically increasing items * (of the same type as the input items) * that divide the item input domain into m+1 consecutive, non-overlapping intervals. - * + *The start of each interval is below the lowest item retained by the sketch * corresponding to a zero rank or zero probability, and the end of the interval * is the rank or cumulative probability corresponding to the split point.
@@ -66,7 +66,7 @@ default double[] getCDF(float[] splitPoints) { * * *It is not recommended to include either the minimum or maximum items of the input stream.
- * + *
** @param searchCrit the desired search criteria. * @return a PMF array of m+1 probability masses as doubles on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. diff --git a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesGenericAPI.java b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesGenericAPI.java index 459e58cdd..c6a05ffa1 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesGenericAPI.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesGenericAPI.java @@ -53,7 +53,7 @@ default double[] getCDF(T[] splitPoints) { * @param splitPoints an array of m unique, monotonically increasing items * (of the same type as the input items) * that divide the item input domain into m+1 overlapping intervals. - * + *Each interval except for the end intervals starts with a split point and ends with the next split * point in sequence.
* @@ -136,7 +136,7 @@ default double[] getPMF(float[] splitPoints) { * * *It is not recommended to include either the minimum or maximum items of the input stream.
- * + *
** @param searchCrit the desired search criteria. * @return a discrete CDF array of m+1 double ranks (or cumulative probabilities) on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. @@ -78,6 +78,7 @@ default double[] getCDF(T[] splitPoints) { double[] getCDF(T[] splitPoints, QuantileSearchCriteria searchCrit); /** + * Returns the sketch item class * @return the sketch item class */ ClassThe start of each interval is below the lowest item retained by the sketch * corresponding to a zero rank or zero probability, and the end of the interval * is the rank or cumulative probability corresponding to the split point.
@@ -70,7 +70,7 @@ default double[] getCDF(T[] splitPoints) { * * *It is not recommended to include either the minimum or maximum items of the input stream.
- * + *
** @param searchCrit the desired search criteria. * @return a PMF array of m+1 probability masses as doubles on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. diff --git a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesLongsAPI.java b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesLongsAPI.java new file mode 100644 index 000000000..fb1ca5817 --- /dev/null +++ b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesLongsAPI.java @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.quantilescommon; + +import static org.apache.datasketches.quantilescommon.QuantileSearchCriteria.INCLUSIVE; + +/** + * The Quantiles API for item type long. + * @see QuantilesAPI + * @author Lee Rhodes + * @author Zac Blanco + */ +public interface QuantilesLongsAPI extends QuantilesAPI { + + /** + * This is equivalent to {@link #getCDF(long[], QuantileSearchCriteria) getCDF(splitPoints, INCLUSIVE)} + * @param splitPoints an array of m unique, monotonically increasing items. + * @return a discrete CDF array of m+1 double ranks (or cumulative probabilities) on the interval [0.0, 1.0]. + * @throws IllegalArgumentException if sketch is empty. + */ + default double[] getCDF(long[] splitPoints) { + return getCDF(splitPoints, INCLUSIVE); + } + + /** + * Returns an approximation to the Cumulative Distribution Function (CDF) of the input stream + * as a monotonically increasing array of double ranks (or cumulative probabilities) on the interval [0.0, 1.0], + * given a set of splitPoints. + * + *Each interval except for the end intervals starts with a split point and ends with the next split * point in sequence.
* @@ -156,7 +157,7 @@ default double[] getPMF(T[] splitPoints) { * * *It is not recommended to include either the minimum or maximum items of the input stream.
- * + *
The resulting approximations have a probabilistic guarantee that can be obtained from the + * getNormalizedRankError(false) function.
+ * + * @param splitPoints an array of m unique, monotonically increasing items + * (of the same type as the input items) + * that divide the item input domain into m+1 overlapping intervals. + *+ *+ * @param searchCrit the desired search criteria. + * @return a discrete CDF array of m+1 double ranks (or cumulative probabilities) on the interval [0.0, 1.0]. + * @throws IllegalArgumentException if sketch is empty. + */ + double[] getCDF(long[] splitPoints, QuantileSearchCriteria searchCrit); + + /** + * Returns the maximum item of the stream. This is provided for convenience and may be different from the + * item returned by getQuantile(1.0). + * + * @return the maximum item of the stream + * @throws IllegalArgumentException if sketch is empty. + */ + long getMaxItem(); + + /** + * Returns the minimum item of the stream. This is provided for convenience and may be different from the + * item returned by getQuantile(0.0). + * + * @return the minimum item of the stream + * @throws IllegalArgumentException if sketch is empty. + */ + long getMinItem(); + + /** + * This is equivalent to {@link #getPMF(long[], QuantileSearchCriteria) getPMF(splitPoints, INCLUSIVE)} + * @param splitPoints an array of m unique, monotonically increasing items. + * @return a PMF array of m+1 probability masses as doubles on the interval [0.0, 1.0]. + * @throws IllegalArgumentException if sketch is empty. + */ + default double[] getPMF(long[] splitPoints) { + return getPMF(splitPoints, INCLUSIVE); + } + + /** + * Returns an approximation to the Probability Mass Function (PMF) of the input stream + * as an array of probability masses as doubles on the interval [0.0, 1.0], + * given a set of splitPoints. + * + *The start of each interval is below the lowest item retained by the sketch + * corresponding to a zero rank or zero probability, and the end of the interval + * is the rank or cumulative probability corresponding to the split point.
+ * + *The (m+1)th interval represents 100% of the distribution represented by the sketch + * and consistent with the definition of a cumulative probability distribution, thus the (m+1)th + * rank or probability in the returned array is always 1.0.
+ * + *If a split point exactly equals a retained item of the sketch and the search criterion is:
+ * + *+ *
+ * + *- INCLUSIVE, the resulting cumulative probability will include that item.
+ *- EXCLUSIVE, the resulting cumulative probability will not include the weight of that split point.
+ *It is not recommended to include either the minimum or maximum items of the input stream.
+ *
The resulting approximations have a probabilistic guarantee that can be obtained from the + * getNormalizedRankError(true) function.
+ * + * @param splitPoints an array of m unique, monotonically increasing items + * (of the same type as the input items) + * that divide the item input domain into m+1 consecutive, non-overlapping intervals. + *+ *+ * @param searchCrit the desired search criteria. + * @return a PMF array of m+1 probability masses as doubles on the interval [0.0, 1.0]. + * @throws IllegalArgumentException if sketch is empty. + */ + double[] getPMF(long[] splitPoints, QuantileSearchCriteria searchCrit); + + /** + * This is equivalent to {@link #getQuantile(double, QuantileSearchCriteria) getQuantile(rank, INCLUSIVE)} + * @param rank the given normalized rank, a double in the range [0.0, 1.0]. + * @return the approximate quantile given the normalized rank. + * @throws IllegalArgumentException if sketch is empty. + */ + default long getQuantile(double rank) { + return getQuantile(rank, INCLUSIVE); + } + + /** + * Gets the approximate quantile of the given normalized rank and the given search criterion. + * + * @param rank the given normalized rank, a double in the range [0.0, 1.0]. + * @param searchCrit If INCLUSIVE, the given rank includes all quantiles ≤ + * the quantile directly corresponding to the given rank. + * If EXCLUSIVE, he given rank includes all quantiles < + * the quantile directly corresponding to the given rank. + * @return the approximate quantile given the normalized rank. + * @throws IllegalArgumentException if sketch is empty. + * @see org.apache.datasketches.quantilescommon.QuantileSearchCriteria + */ + long getQuantile(double rank, QuantileSearchCriteria searchCrit); + + /** + * Gets the lower bound of the quantile confidence interval in which the quantile of the + * given rank exists. + * + *Each interval except for the end intervals starts with a split point and ends with the next split + * point in sequence.
+ * + *The first interval starts below the lowest item retained by the sketch + * corresponding to a zero rank or zero probability, and ends with the first split point
+ * + *The last (m+1)th interval starts with the last split point and ends after the last + * item retained by the sketch corresponding to a rank or probability of 1.0.
+ * + *The sum of the probability masses of all (m+1) intervals is 1.0.
+ * + *If the search criterion is:
+ * + *+ *
+ * + *- INCLUSIVE, and the upper split point of an interval equals an item retained by the sketch, the interval + * will include that item. If the lower split point equals an item retained by the sketch, the interval will exclude + * that item.
+ *- EXCLUSIVE, and the upper split point of an interval equals an item retained by the sketch, the interval + * will exclude that item. If the lower split point equals an item retained by the sketch, the interval will include + * that item.
+ *It is not recommended to include either the minimum or maximum items of the input stream.
+ *
Although it is possible to estimate the probability that the true quantile + * exists within the quantile confidence interval specified by the upper and lower quantile bounds, + * it is not possible to guarantee the width of the quantile confidence interval + * as an additive or multiplicative percent of the true quantile.
+ * + * @param rank the given normalized rank + * @return the lower bound of the quantile confidence interval in which the quantile of the + * given rank exists. + * @throws IllegalArgumentException if sketch is empty. + */ + long getQuantileLowerBound(double rank); + + /** + * Gets the upper bound of the quantile confidence interval in which the true quantile of the + * given rank exists. + * + *Although it is possible to estimate the probability that the true quantile + * exists within the quantile confidence interval specified by the upper and lower quantile bounds, + * it is not possible to guarantee the width of the quantile interval + * as an additive or multiplicative percent of the true quantile.
+ * + * @param rank the given normalized rank + * @return the upper bound of the quantile confidence interval in which the true quantile of the + * given rank exists. + * @throws IllegalArgumentException if sketch is empty. + */ + long getQuantileUpperBound(double rank); + + /** + * This is equivalent to {@link #getQuantiles(double[], QuantileSearchCriteria) getQuantiles(ranks, INCLUSIVE)} + * @param ranks the given array of normalized ranks, each of which must be + * in the interval [0.0,1.0]. + * @return an array of quantiles corresponding to the given array of normalized ranks. + * @throws IllegalArgumentException if sketch is empty. + */ + default long[] getQuantiles(double[] ranks) { + return getQuantiles(ranks, INCLUSIVE); + } + + /** + * Gets an array of quantiles from the given array of normalized ranks. + * + * @param ranks the given array of normalized ranks, each of which must be + * in the interval [0.0,1.0]. + * @param searchCrit if INCLUSIVE, the given ranks include all quantiles ≤ + * the quantile directly corresponding to each rank. + * @return an array of quantiles corresponding to the given array of normalized ranks. + * @throws IllegalArgumentException if sketch is empty. + * @see org.apache.datasketches.quantilescommon.QuantileSearchCriteria + */ + long[] getQuantiles(double[] ranks, QuantileSearchCriteria searchCrit); + + /** + * This is equivalent to {@link #getRank(long, QuantileSearchCriteria) getRank(quantile, INCLUSIVE)} + * @param quantile the given quantile + * @return the normalized rank corresponding to the given quantile + * @throws IllegalArgumentException if sketch is empty. + */ + default double getRank(long quantile) { + return getRank(quantile, INCLUSIVE); + } + + /** + * Gets the normalized rank corresponding to the given a quantile. + * + * @param quantile the given quantile + * @param searchCrit if INCLUSIVE the given quantile is included into the rank. + * @return the normalized rank corresponding to the given quantile + * @throws IllegalArgumentException if sketch is empty. + * @see org.apache.datasketches.quantilescommon.QuantileSearchCriteria + */ + double getRank(long quantile, QuantileSearchCriteria searchCrit); + + /** + * This is equivalent to {@link #getRanks(long[], QuantileSearchCriteria) getRanks(quantiles, INCLUSIVE)} + * @param quantiles the given array of quantiles + * @return an array of normalized ranks corresponding to the given array of quantiles. + * @throws IllegalArgumentException if sketch is empty. + */ + default double[] getRanks(long[] quantiles) { + return getRanks(quantiles, INCLUSIVE); + } + + /** + * Gets an array of normalized ranks corresponding to the given array of quantiles and the given + * search criterion. + * + * @param quantiles the given array of quantiles + * @param searchCrit if INCLUSIVE, the given quantiles include the rank directly corresponding to each quantile. + * @return an array of normalized ranks corresponding to the given array of quantiles. + * @throws IllegalArgumentException if sketch is empty. + * @see org.apache.datasketches.quantilescommon.QuantileSearchCriteria + */ + double[] getRanks(long[] quantiles, QuantileSearchCriteria searchCrit); + + /** + * Returns the current number of bytes this Sketch would require if serialized. + * @return the number of bytes this sketch would require if serialized. + */ + int getSerializedSizeBytes(); + + /** + * Gets the sorted view of this sketch + * @return the sorted view of this sketch + */ + LongsSortedView getSortedView(); + + /** + * Gets the iterator for this sketch, which is not sorted. + * @return the iterator for this sketch + */ + QuantilesLongsSketchIterator iterator(); + + /** + * Returns a byte array representation of this sketch. + * @return a byte array representation of this sketch. + */ + byte[] toByteArray(); + + /** + * Updates this sketch with the given item. + * @param item from a stream of items. NaNs are ignored. + */ + void update(long item); + +} diff --git a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesLongsSketchIterator.java b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesLongsSketchIterator.java new file mode 100644 index 000000000..7ed0d9805 --- /dev/null +++ b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesLongsSketchIterator.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.quantilescommon; + +/** + * The quantiles sketch iterator for primitive type long. + * @see QuantilesSketchIterator + * @author Zac Blanco + */ +public interface QuantilesLongsSketchIterator extends QuantilesSketchIterator { + + /** + * Gets the long quantile at the current index. + * + *Don't call this before calling next() for the first time + * or after getting false from next().
+ * + * @return the long quantile at the current index. + */ + long getQuantile(); + +} + diff --git a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesUtil.java b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesUtil.java index 75798c20f..529fd386d 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesUtil.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesUtil.java @@ -67,6 +67,21 @@ public static final void checkDoublesSplitPointsOrder(final double[] values) { } } + /** + * Checks the sequential validity of the given array of double values. + * They must be unique and monotonically increasing. + * @param values the given array of double values + */ + public static void checkLongsSplitPointsOrder(final long[] values) { + Objects.requireNonNull(values); + final int len = values.length; + for (int j = 0; j < len - 1; j++) { + if (values[j] < values[j + 1]) { continue; } + throw new SketchesArgumentException( + "Values must be unique and monotonically increasing."); + } + } + /** * Checks the sequential validity of the given array of float values. * They must be unique, monotonically increasing and not NaN. diff --git a/src/main/java/org/apache/datasketches/req/BaseReqSketch.java b/src/main/java/org/apache/datasketches/req/BaseReqSketch.java index e587cd633..feeba7739 100644 --- a/src/main/java/org/apache/datasketches/req/BaseReqSketch.java +++ b/src/main/java/org/apache/datasketches/req/BaseReqSketch.java @@ -28,7 +28,7 @@ * This abstract class provides a single place to define and document the public API * for the Relative Error Quantiles Sketch. * - * @see + * @see * Sketching Quantiles and Ranks Tutorial * * @author Lee Rhodes @@ -89,11 +89,23 @@ public static double getRSE(final int k, final double rank, final boolean hra, f @Override public abstract float getQuantileLowerBound(double rank); + /** + * Gets an approximate lower bound of the quantile associated with the given rank. + * @param rank the given normalized rank, a number between 0 and 1.0. + * @param numStdDev the number of standard deviations. Must be 1, 2, or 3. + * @return an approximate lower bound quantile, if it exists. + */ public abstract float getQuantileLowerBound(double rank, int numStdDev); @Override public abstract float getQuantileUpperBound(double rank); + /** + * Gets an approximate upper bound of the quantile associated with the given rank. + * @param rank the given normalized rank, a number between 0 and 1.0. + * @param numStdDev the number of standard deviations. Must be 1, 2, or 3. + * @return an approximate upper bound quantile, if it exists. + */ public abstract float getQuantileUpperBound(double rank, int numStdDev); @Override @@ -101,7 +113,7 @@ public static double getRSE(final int k, final double rank, final boolean hra, f /** * Gets an approximate lower bound rank of the given normalized rank. - * @param rank the given rank, a number between 0 and 1.0. + * @param rank the given normalized rank, a number between 0 and 1.0. * @param numStdDev the number of standard deviations. Must be 1, 2, or 3. * @return an approximate lower bound rank. */ @@ -160,6 +172,7 @@ public boolean isReadOnly() { /** * {@inheritDoc} + * *The parameters k, highRankAccuracy, and reqDebug will not change.
*/ @Override diff --git a/src/main/java/org/apache/datasketches/req/ReqSerDe.java b/src/main/java/org/apache/datasketches/req/ReqSerDe.java index 079c9f282..952749deb 100644 --- a/src/main/java/org/apache/datasketches/req/ReqSerDe.java +++ b/src/main/java/org/apache/datasketches/req/ReqSerDe.java @@ -26,6 +26,7 @@ import java.util.ArrayList; import java.util.List; +import org.apache.datasketches.common.Family; import org.apache.datasketches.memory.Buffer; import org.apache.datasketches.memory.Memory; import org.apache.datasketches.memory.WritableBuffer; @@ -109,7 +110,8 @@ * 0 || (empty)| 0 | K | Flags |FamID=17| SerVer | PreInts = 2 | * *- ** - *Flags:
+ * + * Flags: * Bit 0 : Endianness, reserved * Bit 1 : ReadOnly, reserved * Bit 2 : Empty @@ -126,7 +128,7 @@ class ReqSerDe { enum SerDeFormat { EMPTY, RAWITEMS, EXACT, ESTIMATION } private static final byte SER_VER = 1; - private static final byte FAMILY_ID = 17; + private static final byte FAMILY_ID = (byte) Family.REQ.getID(); static ReqSketch heapify(final Memory mem) { final Buffer buff = mem.asBuffer(); diff --git a/src/main/java/org/apache/datasketches/sampling/ReservoirLongsUnion.java b/src/main/java/org/apache/datasketches/sampling/ReservoirLongsUnion.java index c3ef33957..b58317a9a 100644 --- a/src/main/java/org/apache/datasketches/sampling/ReservoirLongsUnion.java +++ b/src/main/java/org/apache/datasketches/sampling/ReservoirLongsUnion.java @@ -132,10 +132,9 @@ public static ReservoirLongsUnion heapify(final Memory srcMem) { /** * Union the given sketch. - *- * This method can be repeatedly called. If the given sketch is null it is interpreted as an empty - * sketch. - *
+ * + *This method can be repeatedly called. If the given sketch is null it is interpreted as an empty + * sketch.
* * @param sketchIn The incoming sketch. */ diff --git a/src/main/java/org/apache/datasketches/sampling/ReservoirSize.java b/src/main/java/org/apache/datasketches/sampling/ReservoirSize.java index e6f361955..e12d31aa9 100644 --- a/src/main/java/org/apache/datasketches/sampling/ReservoirSize.java +++ b/src/main/java/org/apache/datasketches/sampling/ReservoirSize.java @@ -25,6 +25,7 @@ /** * This class provides a compact representation of reservoir size by encoding it into a * fixed-point 16-bit value. + * *The value itself is a fractional power of 2, with 5 bits of exponent and 11 bits of * mantissa. The exponent allows a choice of anywhere from 0-30, and there are 2048 possible * reservoir size values within each octave. Because reservoir size must be an integer, this diff --git a/src/main/java/org/apache/datasketches/sampling/SampleSubsetSummary.java b/src/main/java/org/apache/datasketches/sampling/SampleSubsetSummary.java index b7758b148..1504542f7 100644 --- a/src/main/java/org/apache/datasketches/sampling/SampleSubsetSummary.java +++ b/src/main/java/org/apache/datasketches/sampling/SampleSubsetSummary.java @@ -41,13 +41,15 @@ public class SampleSubsetSummary { } /** - * @return the Lower Bound + * Returns the lower bound + * @return the lower bound */ public double getLowerBound() { return lowerBound; } /** + * Returns the total sketch weight * @return the total sketch weight */ public double getTotalSketchWeight() { @@ -55,13 +57,15 @@ public double getTotalSketchWeight() { } /** - * @return the Upper Bound + * Returns the upper bound + * @return the upper bound */ public double getUpperBound() { return upperBound; } /** + * Returns the unique count estimate * @return the unique count estimate */ public double getEstimate() { diff --git a/src/main/java/org/apache/datasketches/sampling/package-info.java b/src/main/java/org/apache/datasketches/sampling/package-info.java index edfaa20a8..bbe446914 100644 --- a/src/main/java/org/apache/datasketches/sampling/package-info.java +++ b/src/main/java/org/apache/datasketches/sampling/package-info.java @@ -18,8 +18,8 @@ */ /** - *
This package is dedicated to streaming algorithms that enable fixed size, uniform sampling of - * weighted and unweighted items from a stream.
+ * This package is dedicated to streaming algorithms that enable fixed size, uniform sampling of + * weighted and unweighted items from a stream. * *These sketches are mergeable and can be serialized and deserialized to/from a compact * form.
diff --git a/src/main/java/org/apache/datasketches/tdigest/TDigestDouble.java b/src/main/java/org/apache/datasketches/tdigest/TDigestDouble.java index 1e3408511..d44a439d1 100644 --- a/src/main/java/org/apache/datasketches/tdigest/TDigestDouble.java +++ b/src/main/java/org/apache/datasketches/tdigest/TDigestDouble.java @@ -32,6 +32,7 @@ import org.apache.datasketches.memory.WritableBuffer; import org.apache.datasketches.memory.WritableMemory; import org.apache.datasketches.quantilescommon.QuantilesAPI; +import org.apache.datasketches.quantilescommon.QuantilesUtil; /** * t-Digest for estimating quantiles and ranks. @@ -85,6 +86,7 @@ public TDigestDouble(final short k) { } /** + * Returns parameter k (compression) that was used to configure this TDigest * @return parameter k (compression) that was used to configure this TDigest */ public short getK() { @@ -125,7 +127,7 @@ public void merge(final TDigestDouble other) { /** * Process buffered values and merge centroids if needed */ - public void compress() { + private void compress() { if (numBuffered_ == 0) { return; } final int num = numBuffered_ + numCentroids_; final double[] values = new double[num]; @@ -136,6 +138,7 @@ public void compress() { } /** + * Returns true if TDigest has not seen any data * @return true if TDigest has not seen any data */ public boolean isEmpty() { @@ -143,6 +146,7 @@ public boolean isEmpty() { } /** + * Returns minimum value seen by TDigest * @return minimum value seen by TDigest */ public double getMinValue() { @@ -151,6 +155,7 @@ public double getMinValue() { } /** + * Returns maximum value seen by TDigest * @return maximum value seen by TDigest */ public double getMaxValue() { @@ -159,6 +164,7 @@ public double getMaxValue() { } /** + * Returns total weight * @return total weight */ public long getTotalWeight() { @@ -277,6 +283,51 @@ public double getQuantile(final double rank) { return weightedAverage(centroidWeights_[numCentroids_ - 1], w1, maxValue_, w2); } + /** + * Returns an approximation to the Probability Mass Function (PMF) of the input stream + * given a set of split points. + * + * @param splitPoints an array of m unique, monotonically increasing values + * that divide the input domain into m+1 consecutive disjoint intervals (bins). + * + * @return an array of m+1 doubles each of which is an approximation + * to the fraction of the input stream values (the mass) that fall into one of those intervals. + * @throws SketchesStateException if sketch is empty. + */ + public double[] getPMF(final double[] splitPoints) { + final double[] buckets = getCDF(splitPoints); + for (int i = buckets.length; i-- > 1; ) { + buckets[i] -= buckets[i - 1]; + } + return buckets; + } + + /** + * Returns an approximation to the Cumulative Distribution Function (CDF), which is the + * cumulative analog of the PMF, of the input stream given a set of split points. + * + * @param splitPoints an array of m unique, monotonically increasing values + * that divide the input domain into m+1 consecutive disjoint intervals. + * + * @return an array of m+1 doubles, which are a consecutive approximation to the CDF + * of the input stream given the splitPoints. The value at array position j of the returned + * CDF array is the sum of the returned values in positions 0 through j of the returned PMF + * array. This can be viewed as array of ranks of the given split points plus one more value + * that is always 1. + * @throws SketchesStateException if sketch is empty. + */ + public double[] getCDF(final double[] splitPoints) { + if (isEmpty()) { throw new SketchesStateException(QuantilesAPI.EMPTY_MSG); } + QuantilesUtil.checkDoublesSplitPointsOrder(splitPoints); + final int len = splitPoints.length + 1; + final double[] ranks = new double[len]; + for (int i = 0; i < len - 1; i++) { + ranks[i] = getRank(splitPoints[i]); + } + ranks[len - 1] = 1.0; + return ranks; + } + /** * Computes size needed to serialize the current state. * @return size in bytes needed to serialize this tdigest diff --git a/src/main/java/org/apache/datasketches/theta/AnotB.java b/src/main/java/org/apache/datasketches/theta/AnotB.java index 322e66ef7..d9ab37416 100644 --- a/src/main/java/org/apache/datasketches/theta/AnotB.java +++ b/src/main/java/org/apache/datasketches/theta/AnotB.java @@ -56,6 +56,11 @@ */ public abstract class AnotB extends SetOperation { + /** + * Constructor + */ + AnotB() {} + @Override public Family getFamily() { return Family.A_NOT_B; diff --git a/src/main/java/org/apache/datasketches/theta/AnotBimpl.java b/src/main/java/org/apache/datasketches/theta/AnotBimpl.java index d2161c995..cc076fd85 100644 --- a/src/main/java/org/apache/datasketches/theta/AnotBimpl.java +++ b/src/main/java/org/apache/datasketches/theta/AnotBimpl.java @@ -20,13 +20,15 @@ package org.apache.datasketches.theta; import static org.apache.datasketches.common.Util.exactLog2OfLong; -import static org.apache.datasketches.thetacommon.HashOperations.convertToHashTable; +import static org.apache.datasketches.thetacommon.HashOperations.checkThetaCorruption; +import static org.apache.datasketches.thetacommon.HashOperations.continueCondition; import static org.apache.datasketches.thetacommon.HashOperations.hashSearch; +import static org.apache.datasketches.thetacommon.HashOperations.hashSearchOrInsert; +import static org.apache.datasketches.thetacommon.HashOperations.minLgHashTableSize; import java.util.Arrays; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; import org.apache.datasketches.memory.WritableMemory; import org.apache.datasketches.thetacommon.ThetaUtil; @@ -125,7 +127,7 @@ public CompactSketch aNotB(final Sketch skA, final Sketch skB, final boolean dst if (skB.isEmpty()) { return skA.compact(dstOrdered, dstMem); - } + } ThetaUtil.checkSeedHashes(skB.getSeedHash(), seedHash_); //Both skA & skB are not empty @@ -148,11 +150,6 @@ int getRetainedEntries() { return curCount_; } - @Override - public boolean isSameResource(final Memory that) { - return false; - } - //restricted private static long[] getHashArrA(final Sketch skA) { //returns a new array @@ -168,14 +165,12 @@ private static long[] getResultHashArr( //returns a new array final long[] hashArrA, final Sketch skB) { - //Rebuild/get hashtable of skB + // Rebuild or get hashtable of skB final long[] hashTableB; //read only - final long[] thetaCache = skB.getCache(); - final int countB = skB.getRetainedEntries(true); if (skB instanceof CompactSketch) { - hashTableB = convertToHashTable(thetaCache, countB, minThetaLong, ThetaUtil.REBUILD_THRESHOLD); + hashTableB = convertToHashTable(skB, minThetaLong, ThetaUtil.REBUILD_THRESHOLD); } else { - hashTableB = thetaCache; + hashTableB = skB.getCache(); } //build temporary result arrays of skA @@ -197,6 +192,25 @@ private static long[] getResultHashArr( //returns a new array return Arrays.copyOfRange(tmpHashArrA, 0, nonMatches); } + private static long[] convertToHashTable( + final Sketch sketch, + final long thetaLong, + final double rebuildThreshold) { + final int lgArrLongs = minLgHashTableSize(sketch.getRetainedEntries(true), rebuildThreshold); + final int arrLongs = 1 << lgArrLongs; + final long[] hashTable = new long[arrLongs]; + checkThetaCorruption(thetaLong); + final HashIterator it = sketch.iterator(); + while (it.next()) { + final long hash = it.get(); + if (continueCondition(thetaLong, hash) ) { + continue; + } + hashSearchOrInsert(hashTable, lgArrLongs, hash); + } + return hashTable; + } + private void reset() { thetaLong_ = Long.MAX_VALUE; empty_ = true; diff --git a/src/main/java/org/apache/datasketches/theta/BitPacking.java b/src/main/java/org/apache/datasketches/theta/BitPacking.java index ca70dafa2..99bcfb105 100644 --- a/src/main/java/org/apache/datasketches/theta/BitPacking.java +++ b/src/main/java/org/apache/datasketches/theta/BitPacking.java @@ -152,7 +152,7 @@ static void packBitsBlock8(final long[] values, final int i, final byte[] buf, f case 61: packBits61(values, i, buf, off); break; case 62: packBits62(values, i, buf, off); break; case 63: packBits63(values, i, buf, off); break; - default: throw new SketchesArgumentException("wrong number of bits " + bits); + default: throw new SketchesArgumentException("wrong number of bits in packBitsBlock8: " + bits); } } @@ -221,7 +221,7 @@ static void unpackBitsBlock8(final long[] values, final int i, final byte[] buf, case 61: unpackBits61(values, i, buf, off); break; case 62: unpackBits62(values, i, buf, off); break; case 63: unpackBits63(values, i, buf, off); break; - default: throw new SketchesArgumentException("wrong number of bits " + bits); + default: throw new SketchesArgumentException("wrong number of bits unpackBitsBlock8: " + bits); } } @@ -482,7 +482,7 @@ static void packBits13(final long[] values, final int i, final byte[] buf, int o buf[off++] = (byte) (values[i + 3] >>> 4); - buf[off] = (byte) (values[i + 3] >>> 4); + buf[off] = (byte) (values[i + 3] << 4); buf[off++] |= values[i + 4] >>> 9; buf[off++] = (byte) (values[i + 4] >>> 1); @@ -4449,7 +4449,7 @@ static void unpackBits35(final long[] values, final int i, final byte[] buf, int values[i + 1] |= Byte.toUnsignedLong(buf[off++]) << 6; values[i + 1] |= Byte.toUnsignedLong(buf[off]) >>> 2; - values[i + 2] = (Byte.toUnsignedLong(buf[off++]) & 2) << 33; + values[i + 2] = (Byte.toUnsignedLong(buf[off++]) & 3) << 33; values[i + 2] |= (Byte.toUnsignedLong(buf[off++])) << 25; values[i + 2] |= Byte.toUnsignedLong(buf[off++]) << 17; values[i + 2] |= Byte.toUnsignedLong(buf[off++]) << 9; diff --git a/src/main/java/org/apache/datasketches/theta/BytesCompactCompressedHashIterator.java b/src/main/java/org/apache/datasketches/theta/BytesCompactCompressedHashIterator.java new file mode 100644 index 000000000..97792da26 --- /dev/null +++ b/src/main/java/org/apache/datasketches/theta/BytesCompactCompressedHashIterator.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.theta; + +/* + * This is to uncompress serial version 4 sketch incrementally + */ +class BytesCompactCompressedHashIterator implements HashIterator { + private byte[] bytes; + private int offset; + private int entryBits; + private int numEntries; + private int index; + private long previous; + private int offsetBits; + private long[] buffer; + private boolean isBlockMode; + + BytesCompactCompressedHashIterator( + final byte[] bytes, + final int offset, + final int entryBits, + final int numEntries + ) { + this.bytes = bytes; + this.offset = offset; + this.entryBits = entryBits; + this.numEntries = numEntries; + index = -1; + previous = 0; + offsetBits = 0; + buffer = new long[8]; + isBlockMode = numEntries >= 8; + } + + @Override + public long get() { + return buffer[index & 7]; + } + + @Override + public boolean next() { + if (++index == numEntries) { return false; } + if (isBlockMode) { + if ((index & 7) == 0) { + if (numEntries - index >= 8) { + unpack8(); + } else { + isBlockMode = false; + unpack1(); + } + } + } else { + unpack1(); + } + return true; + } + + private void unpack1() { + final int i = index & 7; + BitPacking.unpackBits(buffer, i, entryBits, bytes, offset, offsetBits); + offset += (offsetBits + entryBits) >>> 3; + offsetBits = (offsetBits + entryBits) & 7; + buffer[i] += previous; + previous = buffer[i]; + } + + private void unpack8() { + BitPacking.unpackBitsBlock8(buffer, 0, bytes, offset, entryBits); + offset += entryBits; + for (int i = 0; i < 8; i++) { + buffer[i] += previous; + previous = buffer[i]; + } + } +} diff --git a/src/main/java/org/apache/datasketches/theta/BytesCompactHashIterator.java b/src/main/java/org/apache/datasketches/theta/BytesCompactHashIterator.java new file mode 100644 index 000000000..20e21da11 --- /dev/null +++ b/src/main/java/org/apache/datasketches/theta/BytesCompactHashIterator.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.theta; + +import org.apache.datasketches.common.ByteArrayUtil; + +/* + * This is to iterate over serial version 3 sketch representation + */ +class BytesCompactHashIterator implements HashIterator { + final private byte[] bytes; + final private int offset; + final private int numEntries; + private int index; + + BytesCompactHashIterator( + final byte[] bytes, + final int offset, + final int numEntries + ) { + this.bytes = bytes; + this.offset = offset; + this.numEntries = numEntries; + index = -1; + } + + @Override + public long get() { + return ByteArrayUtil.getLongLE(bytes, offset + index * Long.BYTES); + } + + @Override + public boolean next() { + return ++index < numEntries; + } +} diff --git a/src/main/java/org/apache/datasketches/theta/CompactOperations.java b/src/main/java/org/apache/datasketches/theta/CompactOperations.java index a8066314d..2b52f59fa 100644 --- a/src/main/java/org/apache/datasketches/theta/CompactOperations.java +++ b/src/main/java/org/apache/datasketches/theta/CompactOperations.java @@ -161,7 +161,7 @@ static CompactSketch memoryToCompact( final long hash = srcMem.getLong(srcPreLongs << 3); final SingleItemSketch sis = new SingleItemSketch(hash, srcSeedHash); if (dstMem != null) { - dstMem.putByteArray(0, sis.toByteArray(),0, 16); + dstMem.putByteArray(0, sis.toByteArray(), 0, 16); return new DirectCompactSketch(dstMem); } else { //heap return sis; diff --git a/src/main/java/org/apache/datasketches/theta/CompactSketch.java b/src/main/java/org/apache/datasketches/theta/CompactSketch.java index 1426368f1..630b2e135 100644 --- a/src/main/java/org/apache/datasketches/theta/CompactSketch.java +++ b/src/main/java/org/apache/datasketches/theta/CompactSketch.java @@ -19,11 +19,15 @@ package org.apache.datasketches.theta; +import static org.apache.datasketches.common.ByteArrayUtil.getShortLE; import static org.apache.datasketches.common.Family.idToFamily; import static org.apache.datasketches.theta.PreambleUtil.COMPACT_FLAG_MASK; import static org.apache.datasketches.theta.PreambleUtil.EMPTY_FLAG_MASK; +import static org.apache.datasketches.theta.PreambleUtil.FLAGS_BYTE; import static org.apache.datasketches.theta.PreambleUtil.ORDERED_FLAG_MASK; +import static org.apache.datasketches.theta.PreambleUtil.PREAMBLE_LONGS_BYTE; import static org.apache.datasketches.theta.PreambleUtil.READ_ONLY_FLAG_MASK; +import static org.apache.datasketches.theta.PreambleUtil.SEED_HASH_SHORT; import static org.apache.datasketches.theta.PreambleUtil.extractFamilyID; import static org.apache.datasketches.theta.PreambleUtil.extractFlags; import static org.apache.datasketches.theta.PreambleUtil.extractPreLongs; @@ -32,6 +36,7 @@ import static org.apache.datasketches.theta.PreambleUtil.extractEntryBitsV4; import static org.apache.datasketches.theta.PreambleUtil.extractNumEntriesBytesV4; import static org.apache.datasketches.theta.PreambleUtil.extractThetaLongV4; +import static org.apache.datasketches.theta.PreambleUtil.wholeBytesToHoldBits; import static org.apache.datasketches.theta.SingleItemSketch.otherCheckForSingleItem; import org.apache.datasketches.common.Family; @@ -187,9 +192,8 @@ private static CompactSketch wrap(final Memory srcMem, final long seed, final bo final short seedHash = ThetaUtil.computeSeedHash(seed); if (serVer == 4) { - // not wrapping the compressed format since currently we cannot take advantage of - // decompression during iteration because set operations reach into memory directly - return heapifyV4(srcMem, seed, enforceSeed); + return DirectCompactCompressedSketch.wrapInstance(srcMem, + enforceSeed ? seedHash : (short) extractSeedHash(srcMem)); } else if (serVer == 3) { if (PreambleUtil.isEmptyFlag(srcMem)) { @@ -224,6 +228,103 @@ else if (serVer == 2) { "Corrupted: Serialization Version " + serVer + " not recognized."); } + /** + * Wrap takes the sketch image in the given Memory and refers to it directly. + * There is no data copying onto the java heap. + * The wrap operation enables fast read-only merging and access to all the public read-only API. + * + *Only "Direct" Serialization Version 3 (i.e, OpenSource) sketches that have + * been explicitly stored as direct sketches can be wrapped. + * Wrapping earlier serial version sketches will result in a heapify operation. + * These early versions were never designed to "wrap".
+ * + *Wrapping any subclass of this class that is empty or contains only a single item will + * result in heapified forms of empty and single item sketch respectively. + * This is actually faster and consumes less overall memory.
+ * + *This method checks if the DEFAULT_UPDATE_SEED was used to create the source Memory image. + * Note that SerialVersion 1 sketches cannot be checked as they don't have a seedHash field, + * so the resulting heapified CompactSketch will be given the hash of DEFAULT_UPDATE_SEED.
+ * + * @param bytes a byte array image of a Sketch that was created using the DEFAULT_UPDATE_SEED. + * See Memory + * + * @return a CompactSketch backed by the given Memory except as above. + */ + public static CompactSketch wrap(final byte[] bytes) { + return wrap(bytes, ThetaUtil.DEFAULT_UPDATE_SEED, false); + } + + /** + * Wrap takes the sketch image in the given Memory and refers to it directly. + * There is no data copying onto the java heap. + * The wrap operation enables fast read-only merging and access to all the public read-only API. + * + *Only "Direct" Serialization Version 3 (i.e, OpenSource) sketches that have + * been explicitly stored as direct sketches can be wrapped. + * Wrapping earlier serial version sketches will result in a heapify operation. + * These early versions were never designed to "wrap".
+ * + *Wrapping any subclass of this class that is empty or contains only a single item will + * result in heapified forms of empty and single item sketch respectively. + * This is actually faster and consumes less overall memory.
+ * + *This method checks if the given expectedSeed was used to create the source Memory image. + * Note that SerialVersion 1 sketches cannot be checked as they don't have a seedHash field, + * so the resulting heapified CompactSketch will be given the hash of the expectedSeed.
+ * + * @param bytes a byte array image of a Sketch that was created using the given expectedSeed. + * See Memory + * @param expectedSeed the seed used to validate the given Memory image. + * See Update Hash Seed. + * @return a CompactSketch backed by the given Memory except as above. + */ + public static CompactSketch wrap(final byte[] bytes, final long expectedSeed) { + return wrap(bytes, expectedSeed, true); + } + + private static CompactSketch wrap(final byte[] bytes, final long seed, final boolean enforceSeed) { + final int serVer = bytes[PreambleUtil.SER_VER_BYTE]; + final int familyId = bytes[PreambleUtil.FAMILY_BYTE]; + final Family family = Family.idToFamily(familyId); + if (family != Family.COMPACT) { + throw new IllegalArgumentException("Corrupted: " + family + " is not Compact!"); + } + final short seedHash = ThetaUtil.computeSeedHash(seed); + if (serVer == 4) { + return WrappedCompactCompressedSketch.wrapInstance(bytes, seedHash); + } else if (serVer == 3) { + final int flags = bytes[FLAGS_BYTE]; + if ((flags & EMPTY_FLAG_MASK) > 0) { + return EmptyCompactSketch.getHeapInstance(Memory.wrap(bytes)); + } + final int preLongs = bytes[PREAMBLE_LONGS_BYTE]; + if (otherCheckForSingleItem(preLongs, serVer, familyId, flags)) { + return SingleItemSketch.heapify(Memory.wrap(bytes), enforceSeed ? seedHash : getShortLE(bytes, SEED_HASH_SHORT)); + } + //not empty & not singleItem + final boolean compactFlag = (flags & COMPACT_FLAG_MASK) > 0; + if (!compactFlag) { + throw new SketchesArgumentException( + "Corrupted: COMPACT family sketch image must have compact flag set"); + } + final boolean readOnly = (flags & READ_ONLY_FLAG_MASK) > 0; + if (!readOnly) { + throw new SketchesArgumentException( + "Corrupted: COMPACT family sketch image must have Read-Only flag set"); + } + return WrappedCompactSketch.wrapInstance(bytes, + enforceSeed ? seedHash : getShortLE(bytes, SEED_HASH_SHORT)); + } else if (serVer == 1) { + return ForwardCompatibility.heapify1to3(Memory.wrap(bytes), seedHash); + } else if (serVer == 2) { + return ForwardCompatibility.heapify2to3(Memory.wrap(bytes), + enforceSeed ? seedHash : getShortLE(bytes, SEED_HASH_SHORT)); + } + throw new SketchesArgumentException( + "Corrupted: Serialization Version " + serVer + " not recognized."); + } + //Sketch Overrides @Override @@ -249,6 +350,11 @@ public boolean isCompact() { return true; } + @Override + public double getEstimate() { + return Sketch.estimate(getThetaLong(), getRetainedEntries()); + } + /** * gets the sketch as a compressed byte array * @return the sketch as a compressed byte array @@ -274,10 +380,6 @@ private int computeMinLeadingZeros() { return Long.numberOfLeadingZeros(ored); } - private static int wholeBytesToHoldBits(final int bits) { - return (bits >>> 3) + ((bits & 7) > 0 ? 1 : 0); - } - private byte[] toByteArrayV4() { final int preambleLongs = isEstimationMode() ? 2 : 1; final int entryBits = 64 - computeMinLeadingZeros(); @@ -286,8 +388,8 @@ private byte[] toByteArrayV4() { // store num_entries as whole bytes since whole-byte blocks will follow (most probably) final int numEntriesBytes = wholeBytesToHoldBits(32 - Integer.numberOfLeadingZeros(getRetainedEntries())); - final int size = preambleLongs * Long.BYTES + numEntriesBytes + wholeBytesToHoldBits(compressedBits); - final byte[] bytes = new byte[size]; + final int sizeBytes = preambleLongs * Long.BYTES + numEntriesBytes + wholeBytesToHoldBits(compressedBits); + final byte[] bytes = new byte[sizeBytes]; final WritableMemory mem = WritableMemory.writableWrap(bytes); int offsetBytes = 0; mem.putByte(offsetBytes++, (byte) preambleLongs); @@ -334,12 +436,10 @@ private byte[] toByteArrayV4() { private static CompactSketch heapifyV4(final Memory srcMem, final long seed, final boolean enforceSeed) { final int preLongs = extractPreLongs(srcMem); - final int flags = extractFlags(srcMem); final int entryBits = extractEntryBitsV4(srcMem); final int numEntriesBytes = extractNumEntriesBytesV4(srcMem); final short seedHash = (short) extractSeedHash(srcMem); - final boolean isEmpty = (flags & EMPTY_FLAG_MASK) > 0; - if (enforceSeed && !isEmpty) { PreambleUtil.checkMemorySeedHash(srcMem, seed); } + if (enforceSeed) { PreambleUtil.checkMemorySeedHash(srcMem, seed); } int offsetBytes = 8; long theta = Long.MAX_VALUE; if (preLongs > 1) { @@ -374,7 +474,7 @@ private static CompactSketch heapifyV4(final Memory srcMem, final long seed, fin entries[i] += previous; previous = entries[i]; } - return new HeapCompactSketch(entries, isEmpty, seedHash, numEntries, theta, true); + return new HeapCompactSketch(entries, false, seedHash, numEntries, theta, true); } } diff --git a/src/main/java/org/apache/datasketches/theta/ConcurrentDirectQuickSelectSketch.java b/src/main/java/org/apache/datasketches/theta/ConcurrentDirectQuickSelectSketch.java index 946603194..dbdedebd5 100644 --- a/src/main/java/org/apache/datasketches/theta/ConcurrentDirectQuickSelectSketch.java +++ b/src/main/java/org/apache/datasketches/theta/ConcurrentDirectQuickSelectSketch.java @@ -199,7 +199,7 @@ public void awaitBgPropagationTermination() { @Override public final void initBgPropagationService() { - executorService_ = ConcurrentPropagationService.getExecutorService(Thread.currentThread().getId()); + executorService_ = ConcurrentPropagationService.getExecutorService(Thread.currentThread().threadId()); } @Override @@ -258,7 +258,7 @@ public boolean validateEpoch(final long epoch) { private void advanceEpoch() { awaitBgPropagationTermination(); startEagerPropagation(); - ConcurrentPropagationService.resetExecutorService(Thread.currentThread().getId()); + ConcurrentPropagationService.resetExecutorService(Thread.currentThread().threadId()); //no inspection NonAtomicOperationOnVolatileField // this increment of a volatile field is done within the scope of the propagation // synchronization and hence is done by a single thread. diff --git a/src/main/java/org/apache/datasketches/theta/ConcurrentHeapQuickSelectSketch.java b/src/main/java/org/apache/datasketches/theta/ConcurrentHeapQuickSelectSketch.java index 1ce3b4ecc..e4cc3a157 100644 --- a/src/main/java/org/apache/datasketches/theta/ConcurrentHeapQuickSelectSketch.java +++ b/src/main/java/org/apache/datasketches/theta/ConcurrentHeapQuickSelectSketch.java @@ -194,7 +194,7 @@ public void awaitBgPropagationTermination() { @Override public void initBgPropagationService() { - executorService_ = ConcurrentPropagationService.getExecutorService(Thread.currentThread().getId()); + executorService_ = ConcurrentPropagationService.getExecutorService(Thread.currentThread().threadId()); } @Override @@ -253,7 +253,7 @@ public boolean validateEpoch(final long epoch) { private void advanceEpoch() { awaitBgPropagationTermination(); startEagerPropagation(); - ConcurrentPropagationService.resetExecutorService(Thread.currentThread().getId()); + ConcurrentPropagationService.resetExecutorService(Thread.currentThread().threadId()); //no inspection NonAtomicOperationOnVolatileField // this increment of a volatile field is done within the scope of the propagation // synchronization and hence is done by a single thread diff --git a/src/main/java/org/apache/datasketches/theta/ConcurrentHeapThetaBuffer.java b/src/main/java/org/apache/datasketches/theta/ConcurrentHeapThetaBuffer.java index 741db2f72..e7b3ddaac 100644 --- a/src/main/java/org/apache/datasketches/theta/ConcurrentHeapThetaBuffer.java +++ b/src/main/java/org/apache/datasketches/theta/ConcurrentHeapThetaBuffer.java @@ -94,7 +94,7 @@ private static int computeLogBufferSize(final int lgNomLongs, final long exactSi * @param hash to be propagated */ private boolean propagateToSharedSketch(final long hash) { - //noinspection StatementWithEmptyBody + //no inspection StatementWithEmptyBody while (localPropagationInProgress.get()) { } //busy wait until previous propagation completed localPropagationInProgress.set(true); @@ -108,7 +108,7 @@ private boolean propagateToSharedSketch(final long hash) { * Propagates the content of the buffer as a sketch to the shared sketch */ private void propagateToSharedSketch() { - //noinspection StatementWithEmptyBody + //no inspection StatementWithEmptyBody while (localPropagationInProgress.get()) { } //busy wait until previous propagation completed diff --git a/src/main/java/org/apache/datasketches/theta/ConcurrentSharedThetaSketch.java b/src/main/java/org/apache/datasketches/theta/ConcurrentSharedThetaSketch.java index 49a8140c3..cdc843f8b 100644 --- a/src/main/java/org/apache/datasketches/theta/ConcurrentSharedThetaSketch.java +++ b/src/main/java/org/apache/datasketches/theta/ConcurrentSharedThetaSketch.java @@ -21,6 +21,7 @@ import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.datasketches.common.MemoryStatus; import org.apache.datasketches.memory.WritableMemory; /** @@ -30,7 +31,7 @@ * * @author eshcar */ -interface ConcurrentSharedThetaSketch { +interface ConcurrentSharedThetaSketch extends MemoryStatus { long NOT_SINGLE_HASH = -1L; double MIN_ERROR = 0.0000001; @@ -63,8 +64,8 @@ static long computeExactLimit(long k, double error) { void endPropagation(AtomicBoolean localPropagationInProgress, boolean isEager); /** - * Returns the value of the volatile theta manged by the shared sketch - * @return the value of the volatile theta manged by the shared sketch + * Returns the value of the volatile theta managed by the shared sketch + * @return the value of the volatile theta managed by the shared sketch */ long getVolatileTheta(); @@ -124,10 +125,10 @@ boolean propagate(final AtomicBoolean localPropagationInProgress, final Sketch s // //For the external user all of the below methods can be obtained by casting the shared //sketch to UpdateSketch. However, these methods here also act as an alias so that an - //attempt to access these methods from the local buffer will be divered to the shared + //attempt to access these methods from the local buffer will be diverted to the shared //sketch. - //From Sketch + //From Sketch and MemoryStatus int getCompactBytes(); @@ -139,10 +140,6 @@ boolean propagate(final AtomicBoolean localPropagationInProgress, final Sketch s double getUpperBound(int numStdDev); - boolean hasMemory(); - - boolean isDirect(); - boolean isEmpty(); boolean isEstimationMode(); diff --git a/src/main/java/org/apache/datasketches/theta/DirectCompactCompressedSketch.java b/src/main/java/org/apache/datasketches/theta/DirectCompactCompressedSketch.java new file mode 100644 index 000000000..64c0fafd4 --- /dev/null +++ b/src/main/java/org/apache/datasketches/theta/DirectCompactCompressedSketch.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.theta; + +import static org.apache.datasketches.theta.PreambleUtil.extractEntryBitsV4; +import static org.apache.datasketches.theta.PreambleUtil.extractNumEntriesBytesV4; +import static org.apache.datasketches.theta.PreambleUtil.extractPreLongs; +import static org.apache.datasketches.theta.PreambleUtil.extractSeedHash; +import static org.apache.datasketches.theta.PreambleUtil.extractThetaLongV4; +import static org.apache.datasketches.theta.PreambleUtil.wholeBytesToHoldBits; + +import org.apache.datasketches.memory.Memory; +import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.thetacommon.ThetaUtil; + +/** + * An off-heap (Direct), compact, compressed, read-only sketch. It is not empty, not a single item and ordered. + * + *This sketch can only be associated with a Serialization Version 4 format binary image.
+ * + *This implementation uses data in a given Memory that is owned and managed by the caller. + * This Memory can be off-heap, which if managed properly will greatly reduce the need for + * the JVM to perform garbage collection.
+ */ +class DirectCompactCompressedSketch extends DirectCompactSketch { + /** + * Construct this sketch with the given memory. + * @param mem Read-only Memory object. + */ + DirectCompactCompressedSketch(final Memory mem) { + super(mem); + } + + /** + * Wraps the given Memory, which must be a SerVer 4 compressed CompactSketch image. + * Must check the validity of the Memory before calling. + * @param srcMem See Memory + * @param seedHash The update seedHash. + * See Seed Hash. + * @return this sketch + */ + static DirectCompactCompressedSketch wrapInstance(final Memory srcMem, final short seedHash) { + ThetaUtil.checkSeedHashes((short) extractSeedHash(srcMem), seedHash); + return new DirectCompactCompressedSketch(srcMem); + } + + //Sketch Overrides + + @Override + public CompactSketch compact(final boolean dstOrdered, final WritableMemory dstMem) { + if (dstMem != null) { + mem_.copyTo(0, dstMem, 0, getCurrentBytes()); + return new DirectCompactSketch(dstMem); + } + return CompactSketch.heapify(mem_); + } + + @Override + public int getCurrentBytes() { + final int preLongs = extractPreLongs(mem_); + final int entryBits = extractEntryBitsV4(mem_); + final int numEntriesBytes = extractNumEntriesBytesV4(mem_); + return preLongs * Long.BYTES + numEntriesBytes + wholeBytesToHoldBits(getRetainedEntries() * entryBits); + } + + private static final int START_PACKED_DATA_EXACT_MODE = 8; + private static final int START_PACKED_DATA_ESTIMATION_MODE = 16; + + @Override + public int getRetainedEntries(final boolean valid) { //compact is always valid + // number of entries is stored using variable length encoding + // most significant bytes with all zeros are not stored + // one byte in the preamble has the number of non-zero bytes used + final int preLongs = extractPreLongs(mem_); // if > 1 then the second long has theta + final int numEntriesBytes = extractNumEntriesBytesV4(mem_); + int offsetBytes = preLongs > 1 ? START_PACKED_DATA_ESTIMATION_MODE : START_PACKED_DATA_EXACT_MODE; + int numEntries = 0; + for (int i = 0; i < numEntriesBytes; i++) { + numEntries |= Byte.toUnsignedInt(mem_.getByte(offsetBytes++)) << (i << 3); + } + return numEntries; + } + + @Override + public long getThetaLong() { + final int preLongs = extractPreLongs(mem_); + return (preLongs > 1) ? extractThetaLongV4(mem_) : Long.MAX_VALUE; + } + + @Override + public boolean isEmpty() { + return false; + } + + @Override + public boolean isOrdered() { + return true; + } + + @Override + public HashIterator iterator() { + return new MemoryCompactCompressedHashIterator( + mem_, + (extractPreLongs(mem_) > 1 ? START_PACKED_DATA_ESTIMATION_MODE : START_PACKED_DATA_EXACT_MODE) + + extractNumEntriesBytesV4(mem_), + extractEntryBitsV4(mem_), + getRetainedEntries() + ); + } + + //restricted methods + + @Override + long[] getCache() { + final int numEntries = getRetainedEntries(); + final long[] cache = new long[numEntries]; + int i = 0; + final HashIterator it = iterator(); + while (it.next()) { + cache[i++] = it.get(); + } + return cache; + } +} diff --git a/src/main/java/org/apache/datasketches/theta/DirectCompactSketch.java b/src/main/java/org/apache/datasketches/theta/DirectCompactSketch.java index de0e6e43b..15b03311b 100644 --- a/src/main/java/org/apache/datasketches/theta/DirectCompactSketch.java +++ b/src/main/java/org/apache/datasketches/theta/DirectCompactSketch.java @@ -35,7 +35,7 @@ /** * An off-heap (Direct), compact, read-only sketch. The internal hash array can be either ordered - * or unordered. + * or unordered. It is not empty, not a single item. * *This sketch can only be associated with a Serialization Version 3 format binary image.
* @@ -57,7 +57,7 @@ class DirectCompactSketch extends CompactSketch { } /** - * Wraps the given Memory, which must be a SerVer 3, ordered, CompactSketch image. + * Wraps the given Memory, which must be a SerVer 3, CompactSketch image. * Must check the validity of the Memory before calling. The order bit must be set properly. * @param srcMem See Memory * @param seedHash The update seedHash. @@ -84,15 +84,6 @@ public int getCurrentBytes() { return (preLongs + curCount) << 3; } - @Override - public double getEstimate() { - if (otherCheckForSingleItem(mem_)) { return 1; } - final int preLongs = extractPreLongs(mem_); - final int curCount = (preLongs == 1) ? 0 : extractCurCount(mem_); - final long thetaLong = (preLongs > 2) ? extractThetaLong(mem_) : Long.MAX_VALUE; - return Sketch.estimate(thetaLong, curCount); - } - @Override public int getRetainedEntries(final boolean valid) { //compact is always valid if (otherCheckForSingleItem(mem_)) { return 1; } @@ -109,12 +100,12 @@ public long getThetaLong() { @Override public boolean hasMemory() { - return true; + return mem_ != null; } @Override public boolean isDirect() { - return mem_.isDirect(); + return hasMemory() ? mem_.isDirect() : false; } @Override @@ -132,7 +123,7 @@ public boolean isOrdered() { @Override public boolean isSameResource(final Memory that) { - return mem_.isSameResource(that); + return hasMemory() ? mem_.isSameResource(that) : false; } @Override @@ -142,10 +133,8 @@ public HashIterator iterator() { @Override public byte[] toByteArray() { - final int curCount = getRetainedEntries(true); - checkIllegalCurCountAndEmpty(isEmpty(), curCount); - final int preLongs = extractPreLongs(mem_); - final int outBytes = (curCount + preLongs) << 3; + checkIllegalCurCountAndEmpty(isEmpty(), getRetainedEntries()); + final int outBytes = getCurrentBytes(); final byte[] byteArrOut = new byte[outBytes]; mem_.getByteArray(0, byteArrOut, 0, outBytes); return byteArrOut; diff --git a/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketch.java b/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketch.java index 8397c130b..af073a5ee 100644 --- a/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketch.java +++ b/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketch.java @@ -169,7 +169,7 @@ private DirectQuickSelectSketch( //clear hash table area dstMem.clear(preambleLongs << 3, 8 << lgArrLongs); - hashTableThreshold_ = setHashTableThreshold(lgNomLongs, lgArrLongs); + hashTableThreshold_ = getOffHeapHashTableThreshold(lgNomLongs, lgArrLongs); memReqSvr_ = memReqSvr; } @@ -210,7 +210,7 @@ static DirectQuickSelectSketch writableWrap(final WritableMemory srcMem, final l final DirectQuickSelectSketch dqss = new DirectQuickSelectSketch(seed, srcMem); - dqss.hashTableThreshold_ = setHashTableThreshold(lgNomLongs, lgArrLongs); + dqss.hashTableThreshold_ = getOffHeapHashTableThreshold(lgNomLongs, lgArrLongs); return dqss; } @@ -228,7 +228,7 @@ static DirectQuickSelectSketch fastWritableWrap(final WritableMemory srcMem, fin final DirectQuickSelectSketch dqss = new DirectQuickSelectSketch(seed, srcMem); - dqss.hashTableThreshold_ = setHashTableThreshold(lgNomLongs, lgArrLongs); + dqss.hashTableThreshold_ = getOffHeapHashTableThreshold(lgNomLongs, lgArrLongs); return dqss; } @@ -310,7 +310,7 @@ UpdateReturnState hashUpdate(final long hash) { if (actLgRF > 0) { //Expand in current Memory //lgArrLongs will change; thetaLong, curCount will not resize(wmem_, preambleLongs, lgArrLongs, tgtLgArrLongs); - hashTableThreshold_ = setHashTableThreshold(lgNomLongs, tgtLgArrLongs); + hashTableThreshold_ = getOffHeapHashTableThreshold(lgNomLongs, tgtLgArrLongs); return InsertedCountIncrementedResized; } //end of Expand in current memory, exit. @@ -323,14 +323,18 @@ UpdateReturnState hashUpdate(final long hash) { memReqSvr_ = (memReqSvr_ == null) ? wmem_.getMemoryRequestServer() : memReqSvr_; - final WritableMemory newDstMem = memReqSvr_.request(wmem_,reqBytes); + if (memReqSvr_ == null) { //in case the MRS is not enabled or null. + throw new SketchesArgumentException("Out of Memory, MemoryRequestServer is null, cannot expand."); + } + + final WritableMemory newDstMem = memReqSvr_.request(wmem_, reqBytes); moveAndResize(wmem_, preambleLongs, lgArrLongs, newDstMem, tgtLgArrLongs, thetaLong); - memReqSvr_.requestClose(wmem_, newDstMem); + memReqSvr_.requestClose(wmem_); wmem_ = newDstMem; - hashTableThreshold_ = setHashTableThreshold(lgNomLongs, tgtLgArrLongs); + hashTableThreshold_ = getOffHeapHashTableThreshold(lgNomLongs, tgtLgArrLongs); return InsertedCountIncrementedResized; } //end of Request more memory to resize } //end of resize diff --git a/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketchR.java b/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketchR.java index c593f52e3..fb2aed2a5 100644 --- a/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketchR.java +++ b/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketchR.java @@ -86,7 +86,7 @@ static DirectQuickSelectSketchR readOnlyWrap(final Memory srcMem, final long see final DirectQuickSelectSketchR dqssr = new DirectQuickSelectSketchR(seed, (WritableMemory) srcMem); - dqssr.hashTableThreshold_ = setHashTableThreshold(lgNomLongs, lgArrLongs); + dqssr.hashTableThreshold_ = getOffHeapHashTableThreshold(lgNomLongs, lgArrLongs); return dqssr; } @@ -104,7 +104,7 @@ static DirectQuickSelectSketchR fastReadOnlyWrap(final Memory srcMem, final long final DirectQuickSelectSketchR dqss = new DirectQuickSelectSketchR(seed, (WritableMemory) srcMem); - dqss.hashTableThreshold_ = setHashTableThreshold(lgNomLongs, lgArrLongs); + dqss.hashTableThreshold_ = getOffHeapHashTableThreshold(lgNomLongs, lgArrLongs); return dqss; } @@ -144,12 +144,12 @@ public long getThetaLong() { @Override public boolean hasMemory() { - return true; + return wmem_ != null; } @Override public boolean isDirect() { - return wmem_.isDirect(); + return hasMemory() ? wmem_.isDirect() : false; } @Override @@ -159,7 +159,7 @@ public boolean isEmpty() { @Override public boolean isSameResource(final Memory that) { - return wmem_.isSameResource(that); + return hasMemory() ? wmem_.isSameResource(that) : false; } @Override @@ -276,11 +276,11 @@ UpdateReturnState hashUpdate(final long hash) { * @return the hash table threshold */ @SuppressFBWarnings(value = "DB_DUPLICATE_BRANCHES", justification = "False Positive, see the code comments") - static final int setHashTableThreshold(final int lgNomLongs, final int lgArrLongs) { + protected static final int getOffHeapHashTableThreshold(final int lgNomLongs, final int lgArrLongs) { //SpotBugs may complain (DB_DUPLICATE_BRANCHES) if DQS_RESIZE_THRESHOLD == REBUILD_THRESHOLD, //but this allows us to tune these constants for different sketches. final double fraction = (lgArrLongs <= lgNomLongs) ? DQS_RESIZE_THRESHOLD : ThetaUtil.REBUILD_THRESHOLD; - return (int) Math.floor(fraction * (1 << lgArrLongs)); + return (int) (fraction * (1 << lgArrLongs)); } } diff --git a/src/main/java/org/apache/datasketches/theta/EmptyCompactSketch.java b/src/main/java/org/apache/datasketches/theta/EmptyCompactSketch.java index ae481a425..8f6e4972a 100644 --- a/src/main/java/org/apache/datasketches/theta/EmptyCompactSketch.java +++ b/src/main/java/org/apache/datasketches/theta/EmptyCompactSketch.java @@ -91,16 +91,6 @@ public long getThetaLong() { return Long.MAX_VALUE; } - @Override - public boolean hasMemory() { - return false; - } - - @Override - public boolean isDirect() { - return false; - } - @Override public boolean isEmpty() { return true; diff --git a/src/main/java/org/apache/datasketches/theta/HeapCompactSketch.java b/src/main/java/org/apache/datasketches/theta/HeapCompactSketch.java index 479aa3eeb..2572ce5d5 100644 --- a/src/main/java/org/apache/datasketches/theta/HeapCompactSketch.java +++ b/src/main/java/org/apache/datasketches/theta/HeapCompactSketch.java @@ -87,11 +87,6 @@ public int getCurrentBytes() { return (preLongs_ + curCount_) << 3; } - @Override - public double getEstimate() { - return Sketch.estimate(thetaLong_, curCount_); - } - @Override public int getRetainedEntries(final boolean valid) { return curCount_; @@ -102,16 +97,6 @@ public long getThetaLong() { return thetaLong_; } - @Override - public boolean hasMemory() { - return false; - } - - @Override - public boolean isDirect() { - return false; - } - @Override public boolean isEmpty() { return empty_; diff --git a/src/main/java/org/apache/datasketches/theta/HeapQuickSelectSketch.java b/src/main/java/org/apache/datasketches/theta/HeapQuickSelectSketch.java index 37b615456..b9d4dc9e1 100644 --- a/src/main/java/org/apache/datasketches/theta/HeapQuickSelectSketch.java +++ b/src/main/java/org/apache/datasketches/theta/HeapQuickSelectSketch.java @@ -92,7 +92,7 @@ private HeapQuickSelectSketch(final int lgNomLongs, final long seed, final float } lgArrLongs_ = ThetaUtil.startingSubMultiple(lgNomLongs + 1, rf.lg(), ThetaUtil.MIN_LG_ARR_LONGS); - hashTableThreshold_ = setHashTableThreshold(lgNomLongs, lgArrLongs_); + hashTableThreshold_ = getHashTableThreshold(lgNomLongs, lgArrLongs_); curCount_ = 0; thetaLong_ = (long)(p * LONG_MAX_VALUE_AS_DOUBLE); empty_ = true; //other flags: bigEndian = readOnly = compact = ordered = false; @@ -128,7 +128,7 @@ static HeapQuickSelectSketch heapifyInstance(final Memory srcMem, final long see final HeapQuickSelectSketch hqss = new HeapQuickSelectSketch(lgNomLongs, seed, p, memRF, preambleLongs, family); hqss.lgArrLongs_ = lgArrLongs; - hqss.hashTableThreshold_ = setHashTableThreshold(lgNomLongs, lgArrLongs); + hqss.hashTableThreshold_ = getHashTableThreshold(lgNomLongs, lgArrLongs); hqss.curCount_ = extractCurCount(srcMem); hqss.thetaLong_ = extractThetaLong(srcMem); hqss.empty_ = PreambleUtil.isEmptyFlag(srcMem); @@ -197,7 +197,7 @@ public void reset() { cache_ = new long[1 << lgArrLongsSM]; lgArrLongs_ = lgArrLongsSM; } - hashTableThreshold_ = setHashTableThreshold(lgNomLongs_, lgArrLongs_); + hashTableThreshold_ = getHashTableThreshold(lgNomLongs_, lgArrLongs_); empty_ = true; curCount_ = 0; thetaLong_ = (long)(getP() * LONG_MAX_VALUE_AS_DOUBLE); @@ -293,7 +293,7 @@ private final void resizeCache() { curCount_ = newCount; cache_ = tgtArr; - hashTableThreshold_ = setHashTableThreshold(lgNomLongs_, lgArrLongs_); + hashTableThreshold_ = getHashTableThreshold(lgNomLongs_, lgArrLongs_); } //array stays the same size. Changes theta and thus count @@ -318,9 +318,9 @@ private final void quickSelectAndRebuild() { * @param lgArrLongs See lgArrLongs. * @return the hash table threshold */ - static final int setHashTableThreshold(final int lgNomLongs, final int lgArrLongs) { + private static final int getHashTableThreshold(final int lgNomLongs, final int lgArrLongs) { final double fraction = (lgArrLongs <= lgNomLongs) ? ThetaUtil.RESIZE_THRESHOLD : ThetaUtil.REBUILD_THRESHOLD; - return (int) Math.floor(fraction * (1 << lgArrLongs)); + return (int) (fraction * (1 << lgArrLongs)); } } diff --git a/src/main/java/org/apache/datasketches/theta/HeapUpdateSketch.java b/src/main/java/org/apache/datasketches/theta/HeapUpdateSketch.java index 1cc6d75cd..49734a9e8 100644 --- a/src/main/java/org/apache/datasketches/theta/HeapUpdateSketch.java +++ b/src/main/java/org/apache/datasketches/theta/HeapUpdateSketch.java @@ -66,16 +66,6 @@ public int getCurrentBytes() { return (preLongs + dataLongs) << 3; } - @Override - public boolean isDirect() { - return false; - } - - @Override - public boolean hasMemory() { - return false; - } - //UpdateSketch @Override diff --git a/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java b/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java index 509ec2f93..772480fea 100644 --- a/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java +++ b/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java @@ -288,7 +288,7 @@ else if (curCount_ < 0 && sketchInEntries > 0) { else { //On the heap, allocate a HT hashTable_ = new long[1 << lgArrLongs_]; } - moveDataToTgt(sketchIn.getCache(), curCount_); + moveDataToTgt(sketchIn); } //end of state 5 //state 7 @@ -336,14 +336,24 @@ public CompactSketch getResult(final boolean dstOrdered, final WritableMemory ds dstMem, compactCache); } + @Override + public boolean hasMemory() { + return wmem_ != null; + } + @Override public boolean hasResult() { - return wmem_ != null ? wmem_.getInt(RETAINED_ENTRIES_INT) >= 0 : curCount_ >= 0; + return hasMemory() ? wmem_.getInt(RETAINED_ENTRIES_INT) >= 0 : curCount_ >= 0; + } + + @Override + public boolean isDirect() { + return hasMemory() ? wmem_.isDirect() : false; } @Override public boolean isSameResource(final Memory that) { - return wmem_ != null ? wmem_.isSameResource(that) : false; + return hasMemory() ? wmem_.isSameResource(that) : false; } @Override @@ -424,8 +434,6 @@ long getThetaLong() { private void performIntersect(final Sketch sketchIn) { // curCount and input data are nonzero, match against HT assert curCount_ > 0 && !empty_; - final long[] cacheIn = sketchIn.getCache(); - final int arrLongsIn = cacheIn.length; final long[] hashTable; if (wmem_ != null) { final int htLen = 1 << lgArrLongs_; @@ -438,27 +446,17 @@ private void performIntersect(final Sketch sketchIn) { final long[] matchSet = new long[ min(curCount_, sketchIn.getRetainedEntries(true)) ]; int matchSetCount = 0; - if (sketchIn.isOrdered()) { - //ordered compact, which enables early stop - for (int i = 0; i < arrLongsIn; i++ ) { - final long hashIn = cacheIn[i]; - //if (hashIn <= 0L) continue; //<= 0 should not happen - if (hashIn >= thetaLong_) { - break; //early stop assumes that hashes in input sketch are ordered! - } + final boolean isOrdered = sketchIn.isOrdered(); + final HashIterator it = sketchIn.iterator(); + while (it.next()) { + final long hashIn = it.get(); + if (hashIn < thetaLong_) { final int foundIdx = hashSearch(hashTable, lgArrLongs_, hashIn); - if (foundIdx == -1) { continue; } - matchSet[matchSetCount++] = hashIn; - } - } - else { - //either unordered compact or hash table - for (int i = 0; i < arrLongsIn; i++ ) { - final long hashIn = cacheIn[i]; - if (hashIn <= 0L || hashIn >= thetaLong_) { continue; } - final int foundIdx = hashSearch(hashTable, lgArrLongs_, hashIn); - if (foundIdx == -1) { continue; } - matchSet[matchSetCount++] = hashIn; + if (foundIdx != -1) { + matchSet[matchSetCount++] = hashIn; + } + } else { + if (isOrdered) { break; } // early stop } } //reduce effective array size to minimum @@ -505,6 +503,32 @@ private void moveDataToTgt(final long[] arr, final int count) { assert tmpCnt == count : "Intersection Count Check: got: " + tmpCnt + ", expected: " + count; } + private void moveDataToTgt(final Sketch sketch) { + final int count = sketch.getRetainedEntries(); + int tmpCnt = 0; + if (wmem_ != null) { //Off Heap puts directly into mem + final int preBytes = CONST_PREAMBLE_LONGS << 3; + final int lgArrLongs = lgArrLongs_; + final long thetaLong = thetaLong_; + final HashIterator it = sketch.iterator(); + while (it.next()) { + final long hash = it.get(); + if (continueCondition(thetaLong, hash)) { continue; } + hashInsertOnlyMemory(wmem_, lgArrLongs, hash, preBytes); + tmpCnt++; + } + } else { //On Heap. Assumes HT exists and is large enough + final HashIterator it = sketch.iterator(); + while (it.next()) { + final long hash = it.get(); + if (continueCondition(thetaLong_, hash)) { continue; } + hashInsertOnly(hashTable_, lgArrLongs_, hash); + tmpCnt++; + } + } + assert tmpCnt == count : "Intersection Count Check: got: " + tmpCnt + ", expected: " + count; + } + private void hardReset() { resetCommon(); if (wmem_ != null) { diff --git a/src/main/java/org/apache/datasketches/theta/MemoryCompactCompressedHashIterator.java b/src/main/java/org/apache/datasketches/theta/MemoryCompactCompressedHashIterator.java new file mode 100644 index 000000000..d5f37de96 --- /dev/null +++ b/src/main/java/org/apache/datasketches/theta/MemoryCompactCompressedHashIterator.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.theta; + +import static org.apache.datasketches.theta.PreambleUtil.wholeBytesToHoldBits; + +import org.apache.datasketches.memory.Memory; + +/* + * This is to uncompress serial version 4 sketch incrementally + */ +class MemoryCompactCompressedHashIterator implements HashIterator { + private Memory mem; + private int offset; + private int entryBits; + private int numEntries; + private int index; + private long previous; + private int offsetBits; + private long[] buffer; + private byte[] bytes; + private boolean isBlockMode; + private boolean isFirstUnpack1; + + MemoryCompactCompressedHashIterator( + final Memory mem, + final int offset, + final int entryBits, + final int numEntries + ) { + this.mem = mem; + this.offset = offset; + this.entryBits = entryBits; + this.numEntries = numEntries; + index = -1; + previous = 0; + offsetBits = 0; + buffer = new long[8]; + bytes = new byte[entryBits]; + isBlockMode = numEntries >= 8; + isFirstUnpack1 = true; + } + + @Override + public long get() { + return buffer[index & 7]; + } + + @Override + public boolean next() { + if (++index == numEntries) { return false; } + if (isBlockMode) { + if ((index & 7) == 0) { + if (numEntries - index >= 8) { + unpack8(); + } else { + isBlockMode = false; + unpack1(); + } + } + } else { + unpack1(); + } + return true; + } + + private void unpack1() { + if (isFirstUnpack1) { + mem.getByteArray(offset, bytes, 0, wholeBytesToHoldBits((numEntries - index) * entryBits)); + offset = 0; + isFirstUnpack1 = false; + } + final int i = index & 7; + BitPacking.unpackBits(buffer, i, entryBits, bytes, offset, offsetBits); + offset += (offsetBits + entryBits) >>> 3; + offsetBits = (offsetBits + entryBits) & 7; + buffer[i] += previous; + previous = buffer[i]; + } + + private void unpack8() { + mem.getByteArray(offset, bytes, 0, entryBits); + BitPacking.unpackBitsBlock8(buffer, 0, bytes, 0, entryBits); + offset += entryBits; + for (int i = 0; i < 8; i++) { + buffer[i] += previous; + previous = buffer[i]; + } + } +} diff --git a/src/main/java/org/apache/datasketches/theta/PreambleUtil.java b/src/main/java/org/apache/datasketches/theta/PreambleUtil.java index 562be982c..ec0bc1268 100644 --- a/src/main/java/org/apache/datasketches/theta/PreambleUtil.java +++ b/src/main/java/org/apache/datasketches/theta/PreambleUtil.java @@ -37,6 +37,7 @@ /** * This class defines the preamble data structure and provides basic utilities for some of the key * fields. + * *The intent of the design of this class was to isolate the detailed knowledge of the bit and * byte layout of the serialized form of the sketches derived from the Sketch class into one place. * This allows the possibility of the introduction of different serialization @@ -126,7 +127,7 @@ * 3 ||----------------------Start of Hash Table of longs---------------------------------| *
Union objects require 32 bytes of preamble plus a non-compact array of longs representing a + *
Union objects require 32 bytes of preamble plus a non-compact array of longs representing a * hash table.
* *
@@ -523,4 +524,7 @@ private static void throwNotBigEnough(final long cap, final int required) {
+ ", Required: " + required);
}
+ static int wholeBytesToHoldBits(final int bits) {
+ return (bits >>> 3) + ((bits & 7) > 0 ? 1 : 0);
+ }
}
diff --git a/src/main/java/org/apache/datasketches/theta/Rebuilder.java b/src/main/java/org/apache/datasketches/theta/Rebuilder.java
index 07093f652..b6e3de342 100644
--- a/src/main/java/org/apache/datasketches/theta/Rebuilder.java
+++ b/src/main/java/org/apache/datasketches/theta/Rebuilder.java
@@ -84,7 +84,7 @@ static final void quickSelectAndRebuild(final WritableMemory mem, final int prea
}
/**
- * Moves me (the entire sketch) to a new larger Memory location and rebuilds the hash table.
+ * Moves me (the entire updatable sketch) to a new larger Memory location and rebuilds the hash table.
* This assumes a Memory preamble of standard form with the correct value of thetaLong.
* Afterwards, the caller must update the local Memory reference, lgArrLongs
* and hashTableThreshold from the dstMemory and free the source Memory.
diff --git a/src/main/java/org/apache/datasketches/theta/SetOperation.java b/src/main/java/org/apache/datasketches/theta/SetOperation.java
index 4d8ebf0c4..c198dceb7 100644
--- a/src/main/java/org/apache/datasketches/theta/SetOperation.java
+++ b/src/main/java/org/apache/datasketches/theta/SetOperation.java
@@ -25,6 +25,7 @@
import static org.apache.datasketches.theta.PreambleUtil.SER_VER_BYTE;
import org.apache.datasketches.common.Family;
+import org.apache.datasketches.common.MemoryStatus;
import org.apache.datasketches.common.SketchesArgumentException;
import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.memory.WritableMemory;
@@ -35,9 +36,12 @@
*
* @author Lee Rhodes
*/
-public abstract class SetOperation {
+public abstract class SetOperation implements MemoryStatus {
static final int CONST_PREAMBLE_LONGS = 3;
+ /**
+ * Constructor
+ */
SetOperation() {}
/**
@@ -237,20 +241,6 @@ public static int getMaxAnotBResultBytes(final int nomEntries) {
*/
public abstract Family getFamily();
- /**
- * Returns true if the backing resource of this is identical with the backing resource
- * of that. The capacities must be the same. If this is a region,
- * the region offset must also be the same.
- *
- * Note: Only certain set operators during stateful operations can be serialized.
- * Only when they are stored into Memory will this be relevant.
- *
- * @param that A different non-null object
- * @return true if the backing resource of this is the same as the backing resource
- * of that.
- */
- public abstract boolean isSameResource(Memory that);
-
//restricted
/**
diff --git a/src/main/java/org/apache/datasketches/theta/SetOperationBuilder.java b/src/main/java/org/apache/datasketches/theta/SetOperationBuilder.java
index d788418dc..4a35cf67d 100644
--- a/src/main/java/org/apache/datasketches/theta/SetOperationBuilder.java
+++ b/src/main/java/org/apache/datasketches/theta/SetOperationBuilder.java
@@ -66,7 +66,7 @@ public SetOperationBuilder() {
* Sets the Maximum Nominal Entries (max K) for this set operation. The effective value of K of the result of a
* Set Operation can be less than max K, but never greater.
* The minimum value is 16 and the maximum value is 67,108,864, which is 2^26.
- * @param nomEntries Nominal Entres
+ * @param nomEntries Nominal Entries
* This will become the ceiling power of 2 if it is not a power of 2.
* @return this SetOperationBuilder
*/
diff --git a/src/main/java/org/apache/datasketches/theta/SingleItemSketch.java b/src/main/java/org/apache/datasketches/theta/SingleItemSketch.java
index a4bac21c9..3cfc13b1e 100644
--- a/src/main/java/org/apache/datasketches/theta/SingleItemSketch.java
+++ b/src/main/java/org/apache/datasketches/theta/SingleItemSketch.java
@@ -343,16 +343,6 @@ public double getUpperBound(final int numStdDev) {
return 1.0;
}
- @Override
- public boolean hasMemory() {
- return false;
- }
-
- @Override
- public boolean isDirect() {
- return false;
- }
-
@Override
public boolean isEmpty() {
return false;
diff --git a/src/main/java/org/apache/datasketches/theta/Sketch.java b/src/main/java/org/apache/datasketches/theta/Sketch.java
index 888116512..cb202a189 100644
--- a/src/main/java/org/apache/datasketches/theta/Sketch.java
+++ b/src/main/java/org/apache/datasketches/theta/Sketch.java
@@ -32,6 +32,7 @@
import static org.apache.datasketches.thetacommon.HashOperations.count;
import org.apache.datasketches.common.Family;
+import org.apache.datasketches.common.MemoryStatus;
import org.apache.datasketches.common.SketchesArgumentException;
import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.memory.WritableMemory;
@@ -44,7 +45,7 @@
*
* @author Lee Rhodes
*/
-public abstract class Sketch {
+public abstract class Sketch implements MemoryStatus {
static final int DEFAULT_LG_RESIZE_FACTOR = 3; //Unique to Heap
Sketch() {}
@@ -292,11 +293,10 @@ public double getLowerBound(final int numStdDev) {
/**
* Returns the maximum number of storage bytes required for a CompactSketch with the given
- * number of actual entries. Note that this assumes the worse case of the sketch in
- * estimation mode, which requires storing theta and count.
- * @param numberOfEntries the actual number of entries stored with the CompactSketch.
+ * number of actual entries.
+ * @param numberOfEntries the actual number of retained entries stored in the sketch.
* @return the maximum number of storage bytes required for a CompactSketch with the given number
- * of entries.
+ * of retained entries.
*/
public static int getMaxCompactSketchBytes(final int numberOfEntries) {
if (numberOfEntries == 0) { return 8; }
@@ -304,6 +304,18 @@ public static int getMaxCompactSketchBytes(final int numberOfEntries) {
return (numberOfEntries << 3) + 24;
}
+ /**
+ * Returns the maximum number of storage bytes required for a CompactSketch given the configured
+ * log_base2 of the number of nominal entries, which is a power of 2.
+ * @param lgNomEntries Nominal Entries
+ * @return the maximum number of storage bytes required for a CompactSketch with the given
+ * lgNomEntries.
+ */
+ public static int getCompactSketchMaxBytes(final int lgNomEntries) {
+ return (int)((2 << lgNomEntries) * ThetaUtil.REBUILD_THRESHOLD
+ + Family.QUICKSELECT.getMaxPreLongs()) * Long.BYTES;
+ }
+
/**
* Returns the maximum number of storage bytes required for an UpdateSketch with the given
* number of nominal entries (power of 2).
@@ -372,26 +384,12 @@ public double getUpperBound(final int numStdDev) {
: getRetainedEntries(true);
}
- /**
- * Returns true if this sketch's data structure is backed by Memory or WritableMemory.
- * @return true if this sketch's data structure is backed by Memory or WritableMemory.
- */
- public abstract boolean hasMemory();
-
/**
* Returns true if this sketch is in compact form.
* @return true if this sketch is in compact form.
*/
public abstract boolean isCompact();
- /**
- * Returns true if the this sketch's internal data structure is backed by direct (off-heap)
- * Memory.
- * @return true if the this sketch's internal data structure is backed by direct (off-heap)
- * Memory.
- */
- public abstract boolean isDirect();
-
/**
* See Empty
* @return true if empty.
@@ -413,18 +411,6 @@ public boolean isEstimationMode() {
*/
public abstract boolean isOrdered();
- /**
- * Returns true if the backing resource of this is identical with the backing resource
- * of that. The capacities must be the same. If this is a region,
- * the region offset must also be the same.
- * @param that A different non-null object
- * @return true if the backing resource of this is the same as the backing resource
- * of that.
- */
- public boolean isSameResource(final Memory that) {
- return false;
- }
-
/**
* Returns a HashIterator that can be used to iterate over the retained hash values of the
* Theta sketch.
@@ -465,9 +451,8 @@ public String toString(final boolean sketchSummary, final boolean dataDetail, fi
final boolean hexMode) {
final StringBuilder sb = new StringBuilder();
- final long[] cache = getCache();
int nomLongs = 0;
- int arrLongs = cache.length;
+ int arrLongs = 0;
float p = 0;
int rf = 0;
final boolean updateSketch = this instanceof UpdateSketch;
@@ -487,12 +472,10 @@ public String toString(final boolean sketchSummary, final boolean dataDetail, fi
final int w = width > 0 ? width : 8; // default is 8 wide
if (curCount > 0) {
sb.append("### SKETCH DATA DETAIL");
- for (int i = 0, j = 0; i < arrLongs; i++ ) {
- final long h;
- h = cache[i];
- if (h <= 0 || h >= thetaLong) {
- continue;
- }
+ final HashIterator it = iterator();
+ int j = 0;
+ while (it.next()) {
+ final long h = it.get();
if (j % w == 0) {
sb.append(LS).append(String.format(" %6d", j + 1));
}
@@ -591,8 +574,8 @@ public static String toString(final Memory mem) {
abstract int getCurrentPreambleLongs();
/**
- * Returns the Memory object if it exists, otherwise null.
- * @return the Memory object if it exists, otherwise null.
+ * Returns the backing Memory object if it exists, otherwise null.
+ * @return the backing Memory object if it exists, otherwise null.
*/
abstract Memory getMemory();
diff --git a/src/main/java/org/apache/datasketches/theta/Sketches.java b/src/main/java/org/apache/datasketches/theta/Sketches.java
index a5862e4a4..2e7fa0915 100644
--- a/src/main/java/org/apache/datasketches/theta/Sketches.java
+++ b/src/main/java/org/apache/datasketches/theta/Sketches.java
@@ -79,15 +79,28 @@ public static int getMaxAnotBResultBytes(final int maxNomEntries) {
}
/**
- * Ref: {@link Sketch#getMaxCompactSketchBytes(int)}
- * @param numberOfEntries Ref: {@link Sketch#getMaxCompactSketchBytes(int)},
- * {@code numberOfEntries}
- * @return Ref: {@link Sketch#getMaxCompactSketchBytes(int)}
+ * Returns the maximum number of storage bytes required for a CompactSketch with the given
+ * number of actual entries.
+ * @param numberOfEntries the actual number of retained entries stored in the sketch.
+ * @return the maximum number of storage bytes required for a CompactSketch with the given number
+ * of retained entries.
*/
public static int getMaxCompactSketchBytes(final int numberOfEntries) {
return Sketch.getMaxCompactSketchBytes(numberOfEntries);
}
+ /**
+ * Returns the maximum number of storage bytes required for a CompactSketch given the configured
+ * log_base2 of the number of nominal entries, which is a power of 2.
+ * @param lgNomEntries Nominal Entries
+ * @return the maximum number of storage bytes required for a CompactSketch with the given
+ * lgNomEntries.
+ * @see Sketch#getCompactSketchMaxBytes(int)
+ */
+ public static int getCompactSketchMaxBytes(final int lgNomEntries) {
+ return Sketch.getCompactSketchMaxBytes(lgNomEntries);
+ }
+
/**
* Ref: {@link SetOperation#getMaxIntersectionBytes(int)}
* @param nomEntries Ref: {@link SetOperation#getMaxIntersectionBytes(int)}, {@code nomEntries}
diff --git a/src/main/java/org/apache/datasketches/theta/UnionImpl.java b/src/main/java/org/apache/datasketches/theta/UnionImpl.java
index 3ea5ca557..8c5b2f8f0 100644
--- a/src/main/java/org/apache/datasketches/theta/UnionImpl.java
+++ b/src/main/java/org/apache/datasketches/theta/UnionImpl.java
@@ -20,29 +20,17 @@
package org.apache.datasketches.theta;
import static java.lang.Math.min;
-import static org.apache.datasketches.theta.PreambleUtil.COMPACT_FLAG_MASK;
-import static org.apache.datasketches.theta.PreambleUtil.ORDERED_FLAG_MASK;
-import static org.apache.datasketches.theta.PreambleUtil.PREAMBLE_LONGS_BYTE;
import static org.apache.datasketches.theta.PreambleUtil.UNION_THETA_LONG;
import static org.apache.datasketches.theta.PreambleUtil.clearEmpty;
-import static org.apache.datasketches.theta.PreambleUtil.extractCurCount;
import static org.apache.datasketches.theta.PreambleUtil.extractFamilyID;
-import static org.apache.datasketches.theta.PreambleUtil.extractFlags;
-import static org.apache.datasketches.theta.PreambleUtil.extractLgArrLongs;
-import static org.apache.datasketches.theta.PreambleUtil.extractPreLongs;
-import static org.apache.datasketches.theta.PreambleUtil.extractSeedHash;
-import static org.apache.datasketches.theta.PreambleUtil.extractSerVer;
-import static org.apache.datasketches.theta.PreambleUtil.extractThetaLong;
import static org.apache.datasketches.theta.PreambleUtil.extractUnionThetaLong;
import static org.apache.datasketches.theta.PreambleUtil.insertUnionThetaLong;
-import static org.apache.datasketches.theta.SingleItemSketch.otherCheckForSingleItem;
import static org.apache.datasketches.thetacommon.QuickSelect.selectExcludingZeros;
import java.nio.ByteBuffer;
import org.apache.datasketches.common.Family;
import org.apache.datasketches.common.ResizeFactor;
-import org.apache.datasketches.common.SketchesArgumentException;
import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.memory.MemoryRequestServer;
import org.apache.datasketches.memory.WritableMemory;
@@ -262,10 +250,22 @@ public CompactSketch getResult(final boolean dstOrdered, final WritableMemory ds
minThetaLong, curCountOut, seedHash, empty, true, dstOrdered, dstOrdered, dstMem, compactCacheOut);
}
+ @Override
+ public boolean hasMemory() {
+ return gadget_ instanceof DirectQuickSelectSketchR
+ ? gadget_.hasMemory() : false;
+ }
+
+ @Override
+ public boolean isDirect() {
+ return gadget_ instanceof DirectQuickSelectSketchR
+ ? gadget_.isDirect() : false;
+ }
+
@Override
public boolean isSameResource(final Memory that) {
return gadget_ instanceof DirectQuickSelectSketchR
- ? gadget_.getMemory().isSameResource(that) : false;
+ ? gadget_.isSameResource(that) : false;
}
@Override
@@ -316,38 +316,14 @@ public void union(final Sketch sketchIn) {
unionThetaLong_ = min(min(unionThetaLong_, sketchIn.getThetaLong()), gadget_.getThetaLong()); //Theta rule
unionEmpty_ = false;
- final int curCountIn = sketchIn.getRetainedEntries(true);
- if (curCountIn > 0) {
- if (sketchIn.isOrdered() && (sketchIn instanceof CompactSketch)) { //Use early stop
- //Ordered, thus compact
- if (sketchIn.hasMemory()) {
- final Memory skMem = ((CompactSketch) sketchIn).getMemory();
- final int preambleLongs = skMem.getByte(PREAMBLE_LONGS_BYTE) & 0X3F;
- for (int i = 0; i < curCountIn; i++ ) {
- final int offsetBytes = preambleLongs + i << 3;
- final long hashIn = skMem.getLong(offsetBytes);
- if (hashIn >= unionThetaLong_) { break; } // "early stop"
- gadget_.hashUpdate(hashIn); //backdoor update, hash function is bypassed
- }
- }
- else { //sketchIn is on the Java Heap or has array
- final long[] cacheIn = sketchIn.getCache(); //not a copy!
- for (int i = 0; i < curCountIn; i++ ) {
- final long hashIn = cacheIn[i];
- if (hashIn >= unionThetaLong_) { break; } // "early stop"
- gadget_.hashUpdate(hashIn); //backdoor update, hash function is bypassed
- }
- }
- } //End ordered, compact
- else { //either not-ordered compact or Hash Table form. A HT may have dirty values.
- final long[] cacheIn = sketchIn.getCache(); //if off-heap this will be a copy
- final int arrLongs = cacheIn.length;
- for (int i = 0, c = 0; i < arrLongs && c < curCountIn; i++ ) {
- final long hashIn = cacheIn[i];
- if (hashIn <= 0L || hashIn >= unionThetaLong_) { continue; } //rejects dirty values
- gadget_.hashUpdate(hashIn); //backdoor update, hash function is bypassed
- c++; //ensures against invalid state inside the incoming sketch
- }
+ final boolean isOrdered = sketchIn.isOrdered();
+ final HashIterator it = sketchIn.iterator();
+ while (it.next()) {
+ final long hash = it.get();
+ if (hash < unionThetaLong_ && hash < gadget_.getThetaLong()) {
+ gadget_.hashUpdate(hash); // backdoor update, hash function is bypassed
+ } else {
+ if (isOrdered) { break; }
}
}
unionThetaLong_ = min(unionThetaLong_, gadget_.getThetaLong()); //Theta rule with gadget
@@ -360,105 +336,8 @@ public void union(final Sketch sketchIn) {
@Override
public void union(final Memory skMem) {
- if (skMem == null) { return; }
- final int cap = (int) skMem.getCapacity();
- if (cap < 16) { return; } //empty or garbage
- final int serVer = extractSerVer(skMem);
- final int fam = extractFamilyID(skMem);
-
- if (serVer == 4) { // compressed ordered compact
- // performance can be improved by decompression while performing the union
- // potentially only partial decompression might be needed
- ThetaUtil.checkSeedHashes(expectedSeedHash_, (short) extractSeedHash(skMem));
- final CompactSketch csk = CompactSketch.wrap(skMem);
- union(csk);
- return;
- }
- if (serVer == 3) { //The OpenSource sketches (Aug 4, 2015) starts with serVer = 3
- if (fam < 1 || fam > 3) {
- throw new SketchesArgumentException(
- "Family must be Alpha, QuickSelect, or Compact: " + Family.idToFamily(fam));
- }
- processVer3(skMem);
- return;
- }
- if (serVer == 2) { //older Sketch, which is compact and ordered
- ThetaUtil.checkSeedHashes(expectedSeedHash_, (short)extractSeedHash(skMem));
- final CompactSketch csk = ForwardCompatibility.heapify2to3(skMem, expectedSeedHash_);
- union(csk);
- return;
- }
- if (serVer == 1) { //much older Sketch, which is compact and ordered, no seedHash
- final CompactSketch csk = ForwardCompatibility.heapify1to3(skMem, expectedSeedHash_);
- union(csk);
- return;
- }
-
- throw new SketchesArgumentException("SerVer is unknown: " + serVer);
- }
-
- //Has seedHash, p, could have 0 entries & theta < 1.0,
- //could be unordered, ordered, compact, or not compact,
- //could be Alpha, QuickSelect, or Compact.
- private void processVer3(final Memory skMem) {
- final int preLongs = extractPreLongs(skMem);
-
- if (preLongs == 1) {
- if (otherCheckForSingleItem(skMem)) {
- final long hash = skMem.getLong(8);
- gadget_.hashUpdate(hash);
- return;
- }
- return; //empty
- }
- ThetaUtil.checkSeedHashes(expectedSeedHash_, (short)extractSeedHash(skMem));
- final int curCountIn;
- final long thetaLongIn;
-
- if (preLongs == 2) { //exact mode
- curCountIn = extractCurCount(skMem);
- if (curCountIn == 0) { return; } //should be > 0, but if it is 0 return empty anyway.
- thetaLongIn = Long.MAX_VALUE;
- }
-
- else { //prelongs == 3
- //curCount may be 0 (e.g., from intersection); but sketch cannot be empty.
- curCountIn = extractCurCount(skMem);
- thetaLongIn = extractThetaLong(skMem);
- }
-
- unionThetaLong_ = min(min(unionThetaLong_, thetaLongIn), gadget_.getThetaLong()); //theta rule
- unionEmpty_ = false;
- final int flags = extractFlags(skMem);
- final boolean ordered = (flags & ORDERED_FLAG_MASK) != 0;
- if (ordered) { //must be compact
-
- for (int i = 0; i < curCountIn; i++ ) {
- final int offsetBytes = preLongs + i << 3;
- final long hashIn = skMem.getLong(offsetBytes);
- if (hashIn >= unionThetaLong_) { break; } // "early stop"
- gadget_.hashUpdate(hashIn); //backdoor update, hash function is bypassed
- }
- }
-
- else { //not-ordered, could be compact or hash-table form
- final boolean compact = (flags & COMPACT_FLAG_MASK) != 0;
- final int size = compact ? curCountIn : 1 << extractLgArrLongs(skMem);
-
- for (int i = 0; i < size; i++ ) {
- final int offsetBytes = preLongs + i << 3;
- final long hashIn = skMem.getLong(offsetBytes);
- if (hashIn <= 0L || hashIn >= unionThetaLong_) { continue; }
- gadget_.hashUpdate(hashIn); //backdoor update, hash function is bypassed
- }
- }
-
- unionThetaLong_ = min(unionThetaLong_, gadget_.getThetaLong()); //sync thetaLongs
-
- if (gadget_.hasMemory()) {
- final WritableMemory wmem = (WritableMemory)gadget_.getMemory();
- PreambleUtil.insertUnionThetaLong(wmem, unionThetaLong_);
- PreambleUtil.clearEmpty(wmem);
+ if (skMem != null) {
+ union(Sketch.wrap(skMem));
}
}
diff --git a/src/main/java/org/apache/datasketches/theta/UpdateSketch.java b/src/main/java/org/apache/datasketches/theta/UpdateSketch.java
index 882c5e2e9..cb6854b02 100644
--- a/src/main/java/org/apache/datasketches/theta/UpdateSketch.java
+++ b/src/main/java/org/apache/datasketches/theta/UpdateSketch.java
@@ -343,6 +343,7 @@ public UpdateReturnState update(final long[] data) {
/**
* All potential updates converge here.
+ *
* Don't ever call this unless you really know what you are doing!
*
* @param hash the given input hash value. A hash of zero or Long.MAX_VALUE is ignored.
diff --git a/src/main/java/org/apache/datasketches/theta/WrappedCompactCompressedSketch.java b/src/main/java/org/apache/datasketches/theta/WrappedCompactCompressedSketch.java
new file mode 100644
index 000000000..170c98cad
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/theta/WrappedCompactCompressedSketch.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.theta;
+
+import static org.apache.datasketches.theta.PreambleUtil.wholeBytesToHoldBits;
+import static org.apache.datasketches.theta.PreambleUtil.ENTRY_BITS_BYTE_V4;
+import static org.apache.datasketches.theta.PreambleUtil.NUM_ENTRIES_BYTES_BYTE_V4;
+import static org.apache.datasketches.theta.PreambleUtil.PREAMBLE_LONGS_BYTE;
+
+import org.apache.datasketches.common.ByteArrayUtil;
+import org.apache.datasketches.thetacommon.ThetaUtil;
+
+/**
+ * Wrapper around a serialized compact compressed read-only sketch. It is not empty, not a single item.
+ *
+ * This sketch can only be associated with a Serialization Version 4 format binary image.
+ */
+class WrappedCompactCompressedSketch extends WrappedCompactSketch {
+
+ /**
+ * Construct this sketch with the given bytes.
+ * @param bytes containing serialized compact compressed sketch.
+ */
+ WrappedCompactCompressedSketch(final byte[] bytes) {
+ super(bytes);
+ }
+
+ /**
+ * Wraps the given bytes, which must be a SerVer 4 compressed CompactSketch image.
+ * @param bytes representation of serialized compressed compact sketch.
+ * @param seedHash The update seedHash.
+ * See Seed Hash.
+ * @return this sketch
+ */
+ static WrappedCompactCompressedSketch wrapInstance(final byte[] bytes, final short seedHash) {
+ ThetaUtil.checkSeedHashes(ByteArrayUtil.getShortLE(bytes, PreambleUtil.SEED_HASH_SHORT), seedHash);
+ return new WrappedCompactCompressedSketch(bytes);
+ }
+
+ //Sketch Overrides
+
+ @Override
+ public int getCurrentBytes() {
+ final int preLongs = bytes_[PREAMBLE_LONGS_BYTE];
+ final int entryBits = bytes_[ENTRY_BITS_BYTE_V4];
+ final int numEntriesBytes = bytes_[NUM_ENTRIES_BYTES_BYTE_V4];
+ return preLongs * Long.BYTES + numEntriesBytes + wholeBytesToHoldBits(getRetainedEntries() * entryBits);
+ }
+
+ private static final int START_PACKED_DATA_EXACT_MODE = 8;
+ private static final int START_PACKED_DATA_ESTIMATION_MODE = 16;
+
+ @Override
+ public int getRetainedEntries(final boolean valid) { //compact is always valid
+ // number of entries is stored using variable length encoding
+ // most significant bytes with all zeros are not stored
+ // one byte in the preamble has the number of non-zero bytes used
+ final int preLongs = bytes_[PREAMBLE_LONGS_BYTE]; // if > 1 then the second long has theta
+ final int numEntriesBytes = bytes_[NUM_ENTRIES_BYTES_BYTE_V4];
+ int offsetBytes = preLongs > 1 ? START_PACKED_DATA_ESTIMATION_MODE : START_PACKED_DATA_EXACT_MODE;
+ int numEntries = 0;
+ for (int i = 0; i < numEntriesBytes; i++) {
+ numEntries |= Byte.toUnsignedInt(bytes_[offsetBytes++]) << (i << 3);
+ }
+ return numEntries;
+ }
+
+ @Override
+ public long getThetaLong() {
+ final int preLongs = bytes_[PREAMBLE_LONGS_BYTE];
+ return (preLongs > 1) ? ByteArrayUtil.getLongLE(bytes_, 8) : Long.MAX_VALUE;
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return false;
+ }
+
+ @Override
+ public boolean isOrdered() {
+ return true;
+ }
+
+ @Override
+ public HashIterator iterator() {
+ return new BytesCompactCompressedHashIterator(
+ bytes_,
+ (bytes_[PREAMBLE_LONGS_BYTE] > 1 ? START_PACKED_DATA_ESTIMATION_MODE : START_PACKED_DATA_EXACT_MODE)
+ + bytes_[NUM_ENTRIES_BYTES_BYTE_V4],
+ bytes_[ENTRY_BITS_BYTE_V4],
+ getRetainedEntries()
+ );
+ }
+}
diff --git a/src/main/java/org/apache/datasketches/theta/WrappedCompactSketch.java b/src/main/java/org/apache/datasketches/theta/WrappedCompactSketch.java
new file mode 100644
index 000000000..519857d21
--- /dev/null
+++ b/src/main/java/org/apache/datasketches/theta/WrappedCompactSketch.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.theta;
+
+import static org.apache.datasketches.common.ByteArrayUtil.getIntLE;
+import static org.apache.datasketches.common.ByteArrayUtil.getLongLE;
+import static org.apache.datasketches.common.ByteArrayUtil.getShortLE;
+import static org.apache.datasketches.theta.CompactOperations.memoryToCompact;
+import static org.apache.datasketches.theta.PreambleUtil.EMPTY_FLAG_MASK;
+import static org.apache.datasketches.theta.PreambleUtil.ORDERED_FLAG_MASK;
+import static org.apache.datasketches.theta.PreambleUtil.FLAGS_BYTE;
+import static org.apache.datasketches.theta.PreambleUtil.RETAINED_ENTRIES_INT;
+import static org.apache.datasketches.theta.PreambleUtil.PREAMBLE_LONGS_BYTE;
+import static org.apache.datasketches.theta.PreambleUtil.THETA_LONG;
+import static org.apache.datasketches.theta.PreambleUtil.SEED_HASH_SHORT;
+
+import java.util.Arrays;
+
+import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.memory.WritableMemory;
+import org.apache.datasketches.thetacommon.ThetaUtil;
+
+/**
+ * Wrapper around a serialized compact read-only sketch. It is not empty, not a single item.
+ *
+ * This sketch can only be associated with a Serialization Version 3 format binary image.
+ */
+class WrappedCompactSketch extends CompactSketch {
+ final byte[] bytes_;
+
+ /**
+ * Construct this sketch with the given bytes.
+ * @param bytes containing serialized compact sketch.
+ */
+ WrappedCompactSketch(final byte[] bytes) {
+ bytes_ = bytes;
+ }
+
+ /**
+ * Wraps the given Memory, which must be a SerVer 3 CompactSketch image.
+ * @param bytes representation of serialized compressed compact sketch.
+ * @param seedHash The update seedHash.
+ * See Seed Hash.
+ * @return this sketch
+ */
+ static WrappedCompactSketch wrapInstance(final byte[] bytes, final short seedHash) {
+ ThetaUtil.checkSeedHashes(getShortLE(bytes, SEED_HASH_SHORT), seedHash);
+ return new WrappedCompactSketch(bytes);
+ }
+
+ //Sketch Overrides
+
+ @Override
+ public CompactSketch compact(final boolean dstOrdered, final WritableMemory dstMem) {
+ return memoryToCompact(Memory.wrap(bytes_), dstOrdered, dstMem);
+ }
+
+ @Override
+ public int getCurrentBytes() {
+ final int preLongs = bytes_[PreambleUtil.PREAMBLE_LONGS_BYTE];
+ final int numEntries = (preLongs == 1) ? 0 : getIntLE(bytes_, RETAINED_ENTRIES_INT);
+ return (preLongs + numEntries) << 3;
+ }
+
+ @Override
+ public int getRetainedEntries(final boolean valid) { //compact is always valid
+ final int preLongs = bytes_[PREAMBLE_LONGS_BYTE];
+ return (preLongs == 1) ? 0 : getIntLE(bytes_, RETAINED_ENTRIES_INT);
+ }
+
+ @Override
+ public long getThetaLong() {
+ final int preLongs = bytes_[PREAMBLE_LONGS_BYTE];
+ return (preLongs > 2) ? getLongLE(bytes_, THETA_LONG) : Long.MAX_VALUE;
+ }
+
+ @Override
+ public boolean hasMemory() {
+ return false;
+ }
+
+ @Override
+ public boolean isDirect() {
+ return false;
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return (bytes_[FLAGS_BYTE] & EMPTY_FLAG_MASK) > 0;
+ }
+
+ @Override
+ public boolean isOrdered() {
+ return (bytes_[FLAGS_BYTE] & ORDERED_FLAG_MASK) > 0;
+ }
+
+ @Override
+ public boolean isSameResource(final Memory that) {
+ return false;
+ }
+
+ @Override
+ public HashIterator iterator() {
+ return new BytesCompactHashIterator(
+ bytes_,
+ bytes_[PREAMBLE_LONGS_BYTE] << 3,
+ getRetainedEntries()
+ );
+ }
+
+ @Override
+ public byte[] toByteArray() {
+ return Arrays.copyOf(bytes_, getCurrentBytes());
+ }
+
+ //restricted methods
+
+ @Override
+ long[] getCache() {
+ final long[] cache = new long[getRetainedEntries()];
+ int i = 0;
+ final HashIterator it = iterator();
+ while (it.next()) {
+ cache[i++] = it.get();
+ }
+ return cache;
+ }
+
+ @Override
+ int getCompactPreambleLongs() {
+ return bytes_[PREAMBLE_LONGS_BYTE];
+ }
+
+ @Override
+ int getCurrentPreambleLongs() {
+ return bytes_[PREAMBLE_LONGS_BYTE];
+ }
+
+ @Override
+ Memory getMemory() {
+ return null;
+ }
+
+ @Override
+ short getSeedHash() {
+ return getShortLE(bytes_, SEED_HASH_SHORT);
+ }
+}
diff --git a/src/main/java/org/apache/datasketches/thetacommon/HashOperations.java b/src/main/java/org/apache/datasketches/thetacommon/HashOperations.java
index 848bb47ed..f6b22cb50 100644
--- a/src/main/java/org/apache/datasketches/thetacommon/HashOperations.java
+++ b/src/main/java/org/apache/datasketches/thetacommon/HashOperations.java
@@ -287,6 +287,7 @@ public static int hashSearchOrInsertMemory(final WritableMemory wmem, final int
//Other related methods
/**
+ * Checks that the given theta is not negative nor zero.
* @param thetaLong must be greater than zero otherwise throws an exception.
* See Theta Long
*/
@@ -299,6 +300,7 @@ public static void checkThetaCorruption(final long thetaLong) {
}
/**
+ * Checks that the given hash value is not negative.
* @param hash must be greater than -1 otherwise throws an exception.
* Note a hash of zero is normally ignored, but a negative hash is never allowed.
*/
diff --git a/src/main/java/org/apache/datasketches/thetacommon/SetOperationCornerCases.java b/src/main/java/org/apache/datasketches/thetacommon/SetOperationCornerCases.java
index 20dd6ee7d..2457b37fe 100644
--- a/src/main/java/org/apache/datasketches/thetacommon/SetOperationCornerCases.java
+++ b/src/main/java/org/apache/datasketches/thetacommon/SetOperationCornerCases.java
@@ -28,14 +28,17 @@
* Simplifies and speeds up set operations by resolving specific corner cases.
* @author Lee Rhodes
*/
-@SuppressWarnings("javadoc")
+
public class SetOperationCornerCases {
private static final long MAX = Long.MAX_VALUE;
/** Intersection actions */
public enum IntersectAction {
+ /** Degenerate{MinTheta, 0, F} */
DEGEN_MIN_0_F("D", "Degenerate{MinTheta, 0, F}"),
+ /** Empty{1.0, 0, T */
EMPTY_1_0_T("E", "Empty{1.0, 0, T}"),
+ /** Full Intersect */
FULL_INTERSECT("I", "Full Intersect");
private String actionId;
@@ -46,10 +49,18 @@ private IntersectAction(final String actionId, final String actionDescription) {
this.actionDescription = actionDescription;
}
+ /**
+ * Gets the Action ID
+ * @return the actionId
+ */
public String getActionId() {
return actionId;
}
+ /**
+ * Gets the Action Description
+ * @return the actionDescription
+ */
public String getActionDescription() {
return actionDescription;
}
@@ -57,11 +68,17 @@ public String getActionDescription() {
/** A not B actions */
public enum AnotbAction {
+ /** Sketch A Exact */
SKETCH_A("A", "Sketch A Exactly"),
+ /** Trim Sketch A by MinTheta */
TRIM_A("TA", "Trim Sketch A by MinTheta"),
+ /** Degenerate{MinTheta, 0, F} */
DEGEN_MIN_0_F("D", "Degenerate{MinTheta, 0, F}"),
+ /** Degenerate{ThetaA, 0, F} */
DEGEN_THA_0_F("DA", "Degenerate{ThetaA, 0, F}"),
+ /** Empty{1.0, 0, T} */
EMPTY_1_0_T("E", "Empty{1.0, 0, T}"),
+ /** Full AnotB */
FULL_ANOTB("N", "Full AnotB");
private String actionId;
@@ -72,24 +89,42 @@ private AnotbAction(final String actionId, final String actionDescription) {
this.actionDescription = actionDescription;
}
+ /**
+ * Gets the Action ID
+ * @return the actionId
+ */
public String getActionId() {
return actionId;
}
+ /**
+ * Gets the action description
+ * @return the action description
+ */
public String getActionDescription() {
return actionDescription;
}
}
+ /** List of union actions */
public enum UnionAction {
+ /** Sketch A Exactly */
SKETCH_A("A", "Sketch A Exactly"),
+ /** Trim Sketch A by MinTheta */
TRIM_A("TA", "Trim Sketch A by MinTheta"),
+ /** Sketch B Exactly */
SKETCH_B("B", "Sketch B Exactly"),
+ /** Trim Sketch B by MinTheta */
TRIM_B("TB", "Trim Sketch B by MinTheta"),
+ /** Degenerate{MinTheta, 0, F} */
DEGEN_MIN_0_F("D", "Degenerate{MinTheta, 0, F}"),
+ /** Degenerate{ThetaA, 0, F} */
DEGEN_THA_0_F("DA", "Degenerate{ThetaA, 0, F}"),
+ /** Degenerate{ThetaB, 0, F} */
DEGEN_THB_0_F("DB", "Degenerate{ThetaB, 0, F}"),
+ /** Empty{1.0, 0, T} */
EMPTY_1_0_T("E", "Empty{1.0, 0, T}"),
+ /** Full Union */
FULL_UNION("N", "Full Union");
private String actionId;
@@ -100,49 +135,74 @@ private UnionAction(final String actionId, final String actionDescription) {
this.actionDescription = actionDescription;
}
+ /**
+ * Gets the action ID
+ * @return the actionId
+ */
public String getActionId() {
return actionId;
}
+ /**
+ * Gets the action description
+ * @return the actionDescription
+ */
public String getActionDescription() {
return actionDescription;
}
}
+ /** List of corner cases */
public enum CornerCase {
+ /** Empty Empty */
Empty_Empty(055, "A{ 1.0, 0, T} ; B{ 1.0, 0, T}",
IntersectAction.EMPTY_1_0_T, AnotbAction.EMPTY_1_0_T, UnionAction.EMPTY_1_0_T),
+ /** Empty Exact */
Empty_Exact(056, "A{ 1.0, 0, T} ; B{ 1.0,>0, F}",
IntersectAction.EMPTY_1_0_T, AnotbAction.EMPTY_1_0_T, UnionAction.SKETCH_B),
+ /** Empty Estimation */
Empty_Estimation(052, "A{ 1.0, 0, T} ; B{<1.0,>0, F",
IntersectAction.EMPTY_1_0_T, AnotbAction.EMPTY_1_0_T, UnionAction.SKETCH_B),
+ /** Empty Degen */
Empty_Degen(050, "A{ 1.0, 0, T} ; B{<1.0, 0, F}",
IntersectAction.EMPTY_1_0_T, AnotbAction.EMPTY_1_0_T, UnionAction.DEGEN_THB_0_F),
+ /** Exact Empty */
Exact_Empty(065, "A{ 1.0,>0, F} ; B{ 1.0, 0, T}",
IntersectAction.EMPTY_1_0_T, AnotbAction.SKETCH_A, UnionAction.SKETCH_A),
+ /** Exact Exact */
Exact_Exact(066, "A{ 1.0,>0, F} ; B{ 1.0,>0, F}",
IntersectAction.FULL_INTERSECT, AnotbAction.FULL_ANOTB, UnionAction.FULL_UNION),
+ /** Exact Estimation */
Exact_Estimation(062, "A{ 1.0,>0, F} ; B{<1.0,>0, F}",
IntersectAction.FULL_INTERSECT, AnotbAction.FULL_ANOTB, UnionAction.FULL_UNION),
+ /** Exact Degen */
Exact_Degen(060, "A{ 1.0,>0, F} ; B{<1.0, 0, F}",
IntersectAction.DEGEN_MIN_0_F, AnotbAction.TRIM_A, UnionAction.TRIM_A),
+ /** Estimation_Empty */
Estimation_Empty(025, "A{<1.0,>0, F} ; B{ 1.0, 0, T}",
IntersectAction.EMPTY_1_0_T, AnotbAction.SKETCH_A, UnionAction.SKETCH_A),
+ /** Estimation_Exact */
Estimation_Exact(026, "A{<1.0,>0, F} ; B{ 1.0,>0, F}",
IntersectAction.FULL_INTERSECT, AnotbAction.FULL_ANOTB, UnionAction.FULL_UNION),
+ /** Estimation_Estimation */
Estimation_Estimation(022, "A{<1.0,>0, F} ; B{<1.0,>0, F}",
IntersectAction.FULL_INTERSECT, AnotbAction.FULL_ANOTB, UnionAction.FULL_UNION),
+ /** Estimation_Degen */
Estimation_Degen(020, "A{<1.0,>0, F} ; B{<1.0, 0, F}",
IntersectAction.DEGEN_MIN_0_F, AnotbAction.TRIM_A, UnionAction.TRIM_A),
+ /** Degen_Empty */
Degen_Empty(005, "A{<1.0, 0, F} ; B{ 1.0, 0, T}",
IntersectAction.EMPTY_1_0_T, AnotbAction.DEGEN_THA_0_F, UnionAction.DEGEN_THA_0_F),
+ /** Degen_Exact */
Degen_Exact(006, "A{<1.0, 0, F} ; B{ 1.0,>0, F}",
IntersectAction.DEGEN_MIN_0_F, AnotbAction.DEGEN_THA_0_F, UnionAction.TRIM_B),
+ /** Degen_Estimation */
Degen_Estimation(002, "A{<1.0, 0, F} ; B{<1.0,>0, F}",
IntersectAction.DEGEN_MIN_0_F, AnotbAction.DEGEN_MIN_0_F, UnionAction.TRIM_B),
+ /** Degen_Degen */
Degen_Degen(000, "A{<1.0, 0, F} ; B{<1.0, 0, F}",
IntersectAction.DEGEN_MIN_0_F, AnotbAction.DEGEN_MIN_0_F, UnionAction.DEGEN_MIN_0_F);
@@ -168,27 +228,52 @@ private CornerCase(final int caseId, final String caseDescription,
this.unionAction = unionAction;
}
+ /**
+ * Gets the case ID
+ * @return the caseId
+ */
public int getId() {
return caseId;
}
+ /**
+ * Gets the case description
+ * @return the caseDescription
+ */
public String getCaseDescription() {
return caseDescription;
}
+ /**
+ * Gets the intersect action
+ * @return the intersectAction
+ */
public IntersectAction getIntersectAction() {
return intersectAction;
}
+ /**
+ * Gets the AnotB action
+ * @return the anotbAction
+ */
public AnotbAction getAnotbAction() {
return anotbAction;
}
+ /**
+ * Gets the union action
+ * @return the unionAction
+ */
public UnionAction getUnionAction() {
return unionAction;
}
//See checkById test in /tuple/MiscTest.
+ /**
+ * Converts caseId to CornerCaseId
+ * @param id the case ID
+ * @return the Corner Case ID
+ */
public static CornerCase caseIdToCornerCase(final int id) {
final CornerCase cc = caseIdToCornerCaseMap.get(id);
if (cc == null) {
@@ -198,12 +283,29 @@ public static CornerCase caseIdToCornerCase(final int id) {
}
} //end of enum CornerCase
+ /**
+ * Creates the CornerCase ID
+ * @param thetaLongA the theta of A as a long
+ * @param countA the count of A
+ * @param emptyA true if A is empty
+ * @param thetaLongB the theta of B as a long
+ * @param countB the count of B
+ * @param emptyB true if B is empty
+ * @return the Corner Case ID
+ */
public static int createCornerCaseId(
final long thetaLongA, final int countA, final boolean emptyA,
final long thetaLongB, final int countB, final boolean emptyB) {
return (sketchStateId(emptyA, countA, thetaLongA) << 3) | sketchStateId(emptyB, countB, thetaLongB);
}
+ /**
+ * Returns the sketch state ID
+ * @param isEmpty true if empty
+ * @param numRetained the number of items retained
+ * @param thetaLong the value of theta as a long
+ * @return the sketch state ID
+ */
public static int sketchStateId(final boolean isEmpty, final int numRetained, final long thetaLong) {
// assume thetaLong = MAX if empty
return (((thetaLong == MAX) || isEmpty) ? 4 : 0) | ((numRetained > 0) ? 2 : 0) | (isEmpty ? 1 : 0);
diff --git a/src/main/java/org/apache/datasketches/tuple/DeserializeResult.java b/src/main/java/org/apache/datasketches/tuple/DeserializeResult.java
index 21adb45e2..db8ad682b 100644
--- a/src/main/java/org/apache/datasketches/tuple/DeserializeResult.java
+++ b/src/main/java/org/apache/datasketches/tuple/DeserializeResult.java
@@ -38,6 +38,7 @@ public DeserializeResult(final T object, final int size) {
}
/**
+ * Returns Deserialized object
* @return Deserialized object
*/
public T getObject() {
@@ -45,7 +46,8 @@ public T getObject() {
}
/**
- * @return Size in bytes occupied by the object in the serialized form
+ * Returns size in bytes occupied by the object in the serialized form
+ * @return size in bytes occupied by the object in the serialized form
*/
public int getSize() {
return size;
diff --git a/src/main/java/org/apache/datasketches/tuple/Filter.java b/src/main/java/org/apache/datasketches/tuple/Filter.java
index 28b38295c..2ed156b84 100644
--- a/src/main/java/org/apache/datasketches/tuple/Filter.java
+++ b/src/main/java/org/apache/datasketches/tuple/Filter.java
@@ -19,10 +19,10 @@
package org.apache.datasketches.tuple;
+import java.lang.reflect.Array;
+import java.util.Arrays;
import java.util.function.Predicate;
-import org.apache.datasketches.common.ResizeFactor;
-
/**
* Class for filtering entries from a {@link Sketch} given a {@link Summary}
*
@@ -52,23 +52,25 @@ public CompactSketch filter(final Sketch sketchIn) {
if (sketchIn == null) {
return new CompactSketch<>(null, null, Long.MAX_VALUE, true);
}
-
- final QuickSelectSketch sketch =
- new QuickSelectSketch<>(sketchIn.getRetainedEntries(), ResizeFactor.X1.lg(), null);
+ final long[] hashes = new long[sketchIn.getRetainedEntries()];
+ T[] summaries = null; // lazy init to get class from the first entry
+ int i = 0;
final TupleSketchIterator it = sketchIn.iterator();
while (it.next()) {
final T summary = it.getSummary();
if (predicate.test(summary)) {
- sketch.insert(it.getHash(), (T)summary.copy());
+ hashes[i] = it.getHash();
+ if (summaries == null) {
+ summaries = (T[]) Array.newInstance(summary.getClass(), sketchIn.getRetainedEntries());
+ }
+ summaries[i++] = (T) summary.copy();
}
}
-
- sketch.setThetaLong(sketchIn.getThetaLong());
- if (!sketchIn.isEmpty()) {
- sketch.setEmpty(false);
+ final boolean isEmpty = i == 0 && !sketchIn.isEstimationMode();
+ if (i == 0) {
+ return new CompactSketch<>(null, null, sketchIn.getThetaLong(), isEmpty);
}
-
- return sketch.compact();
+ return new CompactSketch<>(Arrays.copyOf(hashes, i), Arrays.copyOf(summaries, i), sketchIn.getThetaLong(), isEmpty);
}
}
diff --git a/src/main/java/org/apache/datasketches/tuple/SerializerDeserializer.java b/src/main/java/org/apache/datasketches/tuple/SerializerDeserializer.java
index 44d1d9cc0..a30d47edf 100644
--- a/src/main/java/org/apache/datasketches/tuple/SerializerDeserializer.java
+++ b/src/main/java/org/apache/datasketches/tuple/SerializerDeserializer.java
@@ -31,9 +31,17 @@ public final class SerializerDeserializer {
/**
* Defines the sketch classes that this SerializerDeserializer can handle.
*/
- @SuppressWarnings("javadoc")
- public static enum SketchType { QuickSelectSketch, CompactSketch, ArrayOfDoublesQuickSelectSketch,
- ArrayOfDoublesCompactSketch, ArrayOfDoublesUnion }
+ public static enum SketchType {
+ /** QuickSelectSketch */
+ QuickSelectSketch,
+ /** CompactSketch */
+ CompactSketch,
+ /** ArrayOfDoublesQuickSelectSketch */
+ ArrayOfDoublesQuickSelectSketch,
+ /** ArrayOfDoublesCompactSketch */
+ ArrayOfDoublesCompactSketch,
+ /** ArrayOfDoublesUnion */
+ ArrayOfDoublesUnion }
static final int TYPE_BYTE_OFFSET = 3;
diff --git a/src/main/java/org/apache/datasketches/tuple/Sketch.java b/src/main/java/org/apache/datasketches/tuple/Sketch.java
index dcc7a2c16..f0003bf3c 100644
--- a/src/main/java/org/apache/datasketches/tuple/Sketch.java
+++ b/src/main/java/org/apache/datasketches/tuple/Sketch.java
@@ -145,6 +145,7 @@ public boolean isEstimationMode() {
}
/**
+ * Returns number of retained entries
* @return number of retained entries
*/
public abstract int getRetainedEntries();
diff --git a/src/main/java/org/apache/datasketches/tuple/Sketches.java b/src/main/java/org/apache/datasketches/tuple/Sketches.java
index b4c1eed14..fba8ab604 100644
--- a/src/main/java/org/apache/datasketches/tuple/Sketches.java
+++ b/src/main/java/org/apache/datasketches/tuple/Sketches.java
@@ -28,6 +28,7 @@
public final class Sketches {
/**
+ * Creates an empty sketch.
* @param Type of Summary
* @return an empty instance of Sketch
*/
diff --git a/src/main/java/org/apache/datasketches/tuple/SummaryFactory.java b/src/main/java/org/apache/datasketches/tuple/SummaryFactory.java
index 4b20b6772..5c5648f9d 100644
--- a/src/main/java/org/apache/datasketches/tuple/SummaryFactory.java
+++ b/src/main/java/org/apache/datasketches/tuple/SummaryFactory.java
@@ -26,6 +26,7 @@
public interface SummaryFactory {
/**
+ * Returns new instance of Summary
* @return new instance of Summary
*/
public S newSummary();
diff --git a/src/main/java/org/apache/datasketches/tuple/Union.java b/src/main/java/org/apache/datasketches/tuple/Union.java
index 653312fa0..acefa2ab5 100644
--- a/src/main/java/org/apache/datasketches/tuple/Union.java
+++ b/src/main/java/org/apache/datasketches/tuple/Union.java
@@ -100,8 +100,7 @@ public CompactSketch union(final Sketch tupleSketch,
/**
* Performs a stateful union of the internal set with the given tupleSketch.
* @param tupleSketch input tuple sketch to merge with the internal set.
- *
- * Nulls and empty sketches are ignored.
+ * Nulls and empty sketches are ignored.
*/
public void union(final Sketch tupleSketch) {
if (tupleSketch == null || tupleSketch.isEmpty()) { return; }
diff --git a/src/main/java/org/apache/datasketches/tuple/Util.java b/src/main/java/org/apache/datasketches/tuple/Util.java
index 88fef2602..92193ca56 100644
--- a/src/main/java/org/apache/datasketches/tuple/Util.java
+++ b/src/main/java/org/apache/datasketches/tuple/Util.java
@@ -119,16 +119,18 @@ public static String stringConcat(final String[] strArr) {
}
/**
+ * Returns the hash of the given string
* @param s the string to hash
- * @return the hash of the string
+ * @return the hash of the given string
*/
public static long stringHash(final String s) {
return hashString(s, 0, s.length(), PRIME);
}
/**
+ * Returns the hash of the concatenated strings
* @param strArray array of Strings
- * @return long hash of concatenated strings.
+ * @return the hash of concatenated strings.
*/
public static long stringArrHash(final String[] strArray) {
final String s = stringConcat(strArray);
diff --git a/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSketch.java b/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSketch.java
index d78f4e8b1..1bb9edeca 100644
--- a/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSketch.java
+++ b/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSketch.java
@@ -24,6 +24,7 @@
import org.apache.datasketches.tuple.UpdatableSketch;
/**
+ * Extends UpdatableSketch<Double, DoubleSummary>
* @author Lee Rhodes
*/
public class DoubleSketch extends UpdatableSketch {
diff --git a/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummary.java b/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummary.java
index e7abae0d4..f678fba06 100644
--- a/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummary.java
+++ b/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummary.java
@@ -41,24 +41,28 @@ public enum Mode {
/**
* The aggregation mode is the summation function.
+ *
* New retained value = previous retained value + incoming value
*/
Sum,
/**
* The aggregation mode is the minimum function.
+ *
* New retained value = min(previous retained value, incoming value)
*/
Min,
/**
* The aggregation mode is the maximum function.
+ *
* New retained value = max(previous retained value, incoming value)
*/
Max,
/**
* The aggregation mode is always one.
+ *
* New retained value = 1.0
*/
AlwaysOne
@@ -121,6 +125,7 @@ public DoubleSummary copy() {
}
/**
+ * Returns current value of the DoubleSummary
* @return current value of the DoubleSummary
*/
public double getValue() {
diff --git a/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummaryDeserializer.java b/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummaryDeserializer.java
index f8fae67a7..90e01b72b 100644
--- a/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummaryDeserializer.java
+++ b/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummaryDeserializer.java
@@ -24,6 +24,7 @@
import org.apache.datasketches.tuple.SummaryDeserializer;
/**
+ * Implements SummaryDeserializer<DoubleSummary>
* @author Lee Rhodes
*/
public class DoubleSummaryDeserializer implements SummaryDeserializer {
diff --git a/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSketch.java b/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSketch.java
index 438879824..0bad790cc 100644
--- a/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSketch.java
+++ b/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSketch.java
@@ -24,6 +24,7 @@
import org.apache.datasketches.tuple.UpdatableSketch;
/**
+ * Extends UpdatableSketch<Integer, IntegerSummary>
* @author Lee Rhodes
*/
public class IntegerSketch extends UpdatableSketch {
diff --git a/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummary.java b/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummary.java
index 4c04fa2c2..a8cfdab56 100644
--- a/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummary.java
+++ b/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummary.java
@@ -41,24 +41,28 @@ public enum Mode {
/**
* The aggregation mode is the summation function.
+ *
* New retained value = previous retained value + incoming value
*/
Sum,
/**
* The aggregation mode is the minimum function.
+ *
* New retained value = min(previous retained value, incoming value)
*/
Min,
/**
* The aggregation mode is the maximum function.
+ *
* New retained value = max(previous retained value, incoming value)
*/
Max,
/**
* The aggregation mode is always one.
+ *
* New retained value = 1
*/
AlwaysOne
@@ -121,6 +125,7 @@ public IntegerSummary copy() {
}
/**
+ * Returns the current value of the IntegerSummary
* @return current value of the IntegerSummary
*/
public int getValue() {
diff --git a/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummaryDeserializer.java b/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummaryDeserializer.java
index 0a4529e07..68d468785 100644
--- a/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummaryDeserializer.java
+++ b/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummaryDeserializer.java
@@ -24,6 +24,7 @@
import org.apache.datasketches.tuple.SummaryDeserializer;
/**
+ * Implements SummaryDeserializer<IntegerSummary>
* @author Lee Rhodes
*/
public class IntegerSummaryDeserializer implements SummaryDeserializer {
diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesSketch.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesSketch.java
index 9453c872d..111427b7c 100644
--- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesSketch.java
+++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesSketch.java
@@ -172,6 +172,7 @@ public boolean isEmpty() {
}
/**
+ * Returns number of double values associated with each key
* @return number of double values associated with each key
*/
public int getNumValues() {
@@ -196,11 +197,13 @@ public double getTheta() {
}
/**
+ * Returns number of retained entries
* @return number of retained entries
*/
public abstract int getRetainedEntries();
/**
+ * Returns the maximum number of bytes for this sketch when serialized.
* @return the maximum number of bytes for this sketch when serialized.
*/
public abstract int getMaxBytes();
@@ -212,11 +215,13 @@ public double getTheta() {
public abstract int getCurrentBytes();
/**
+ * Returns serialized representation of the sketch
* @return serialized representation of the sketch
*/
public abstract byte[] toByteArray();
/**
+ * Returns array of arrays of double values in the sketch
* @return array of arrays of double values in the sketch
*/
public abstract double[][] getValues();
@@ -226,6 +231,7 @@ public double getTheta() {
abstract long[] getKeys();
/**
+ * Returns the value of theta as a long
* @return the value of theta as a long
*/
long getThetaLong() {
@@ -235,11 +241,13 @@ long getThetaLong() {
abstract short getSeedHash();
/**
- * @return iterator over the sketch
+ * Returns an iterator over the sketch
+ * @return an iterator over the sketch
*/
public abstract ArrayOfDoublesSketchIterator iterator();
/**
+ * Returns this sketch in compact form, which is immutable.
* @return this sketch in compact form, which is immutable.
*/
public ArrayOfDoublesCompactSketch compact() {
@@ -247,7 +255,7 @@ public ArrayOfDoublesCompactSketch compact() {
}
/**
- *
+ * Returns this sketch in compact form, which is immutable.
* @param dstMem the destination WritableMemory
* @return this sketch in compact form, which is immutable.
*/
diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesUnion.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesUnion.java
index 599418f75..9e1db0ada 100644
--- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesUnion.java
+++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesUnion.java
@@ -189,6 +189,7 @@ public void reset() {
// || 15 | 14 | 13 | 12 | 11 | 10 | 9 | 8 |
// 1 ||---------------------------Union Theta Long-----------------------------------------|
/**
+ * Returns a byte array representation of this object
* @return a byte array representation of this object
*/
public byte[] toByteArray() {
@@ -206,6 +207,7 @@ public byte[] toByteArray() {
}
/**
+ * Returns maximum required storage bytes given nomEntries and numValues
* @param nomEntries Nominal number of entries. Forced to the nearest power of 2 greater than or equal to
* given value.
* @param numValues Number of double values to keep for each key
diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesCompactSketch.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesCompactSketch.java
index 91d4eade4..52f827149 100644
--- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesCompactSketch.java
+++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesCompactSketch.java
@@ -30,6 +30,7 @@
/**
* Direct Compact Sketch of type ArrayOfDoubles.
+ *
* This implementation uses data in a given Memory that is owned and managed by the caller.
* This Memory can be off-heap, which if managed properly will greatly reduce the need for
* the JVM to perform garbage collection.
diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesIntersection.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesIntersection.java
index 3dd019d74..7c1b1bf07 100644
--- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesIntersection.java
+++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesIntersection.java
@@ -23,6 +23,7 @@
/**
* Direct Intersection operation for tuple sketches of type ArrayOfDoubles.
+ *
* This implementation uses data in a given Memory that is owned and managed by the caller.
* This Memory can be off-heap, which if managed properly will greatly reduce the need for
* the JVM to perform garbage collection.
@@ -43,7 +44,7 @@ final class DirectArrayOfDoublesIntersection extends ArrayOfDoublesIntersection
}
@Override
- protected ArrayOfDoublesQuickSelectSketch createSketch(final int nomEntries, final int numValues,
+ protected ArrayOfDoublesQuickSelectSketch createSketch(final int nomEntries, final int numValues,
final long seed) {
return new DirectArrayOfDoublesQuickSelectSketch(nomEntries, 0, 1f, numValues, seed, mem_);
}
diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesQuickSelectSketch.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesQuickSelectSketch.java
index 1b4e86904..ae1aa3dc0 100644
--- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesQuickSelectSketch.java
+++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesQuickSelectSketch.java
@@ -33,6 +33,7 @@
/**
* Direct QuickSelect tuple sketch of type ArrayOfDoubles.
+ *
* This implementation uses data in a given Memory that is owned and managed by the caller.
* This Memory can be off-heap, which if managed properly will greatly reduce the need for
* the JVM to perform garbage collection.
diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesSketchIterator.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesSketchIterator.java
index 15503fc04..dcdab1313 100644
--- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesSketchIterator.java
+++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesSketchIterator.java
@@ -23,6 +23,7 @@
/**
* Iterator over the off-heap, Direct tuple sketch of type ArrayOfDoubles (compact or hash table).
+ *
* This implementation uses data in a given Memory that is owned and managed by the caller.
* This Memory can be off-heap, which if managed properly will greatly reduce the need for
* the JVM to perform garbage collection.
diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesUnion.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesUnion.java
index 00310f534..734019632 100644
--- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesUnion.java
+++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesUnion.java
@@ -27,6 +27,7 @@
/**
* Direct Union operation for tuple sketches of type ArrayOfDoubles.
+ *
* This implementation uses data in a given Memory that is owned and managed by the caller.
* This Memory can be off-heap, which if managed properly will greatly reduce the need for
* the JVM to perform garbage collection.
diff --git a/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSketch.java b/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSketch.java
index c66802f06..99ddeee16 100644
--- a/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSketch.java
+++ b/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSketch.java
@@ -26,6 +26,7 @@
import org.apache.datasketches.tuple.UpdatableSketch;
/**
+ * Extends UpdatableSketch<String[], ArrayOfStringsSummary>
* @author Lee Rhodes
*/
public class ArrayOfStringsSketch extends UpdatableSketch {
diff --git a/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummary.java b/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummary.java
index 69ee83e8e..48b2d6d6a 100644
--- a/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummary.java
+++ b/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummary.java
@@ -31,6 +31,7 @@
import org.apache.datasketches.tuple.UpdatableSummary;
/**
+ * Implements UpdatableSummary<String[]>
* @author Lee Rhodes
*/
public final class ArrayOfStringsSummary implements UpdatableSummary {
@@ -115,6 +116,7 @@ public boolean equals(final Object summary) {
}
/**
+ * Returns the nodes array for this summary.
* @return the nodes array for this summary.
*/
public String[] getValue() {
diff --git a/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummaryDeserializer.java b/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummaryDeserializer.java
index 1fb47db1a..16467f338 100644
--- a/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummaryDeserializer.java
+++ b/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummaryDeserializer.java
@@ -24,6 +24,7 @@
import org.apache.datasketches.tuple.SummaryDeserializer;
/**
+ * Implements SummaryDeserializer<ArrayOfStringsSummary>
* @author Lee Rhodes
*/
public class ArrayOfStringsSummaryDeserializer implements SummaryDeserializer {
diff --git a/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummaryFactory.java b/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummaryFactory.java
index 8294b2bae..98fb7fb58 100644
--- a/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummaryFactory.java
+++ b/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummaryFactory.java
@@ -22,6 +22,7 @@
import org.apache.datasketches.tuple.SummaryFactory;
/**
+ * Implements SummaryFactory<ArrayOfStringsSummary>
* @author Lee Rhodes
*/
public class ArrayOfStringsSummaryFactory implements SummaryFactory {
diff --git a/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummarySetOperations.java b/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummarySetOperations.java
index dd24eabaa..2650b84ae 100644
--- a/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummarySetOperations.java
+++ b/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummarySetOperations.java
@@ -22,6 +22,7 @@
import org.apache.datasketches.tuple.SummarySetOperations;
/**
+ * Implements SummarySetOperations<ArrayOfStringsSummary>
* @author Lee Rhodes
*/
public class ArrayOfStringsSummarySetOperations implements SummarySetOperations {
diff --git a/src/test/java/org/apache/datasketches/cpc/CpcCompressionTest.java b/src/test/java/org/apache/datasketches/cpc/CpcCompressionTest.java
index 9a33aa71f..c5ec12399 100644
--- a/src/test/java/org/apache/datasketches/cpc/CpcCompressionTest.java
+++ b/src/test/java/org/apache/datasketches/cpc/CpcCompressionTest.java
@@ -56,7 +56,7 @@ public void checkWriteReadUnary() {
for (int i = 0; i < 100; i++) {
- //TODO Inline WriteUnary
+ // Inline WriteUnary
ptrArr[NEXT_WORD_IDX] = nextWordIndex;
ptrArr[BIT_BUF] = bitBuf;
ptrArr[BUF_BITS] = bufBits;
@@ -92,7 +92,7 @@ public void checkWriteReadUnary() {
for (int i = 0; i < 100; i++) {
- //TODO Inline ReadUnary
+ // Inline ReadUnary
ptrArr[NEXT_WORD_IDX] = nextWordIndex;
ptrArr[BIT_BUF] = bitBuf;
ptrArr[BUF_BITS] = bufBits;
diff --git a/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterCrossLanguageTest.java b/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterCrossLanguageTest.java
new file mode 100644
index 000000000..12c6f20df
--- /dev/null
+++ b/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterCrossLanguageTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.filters.bloomfilter;
+
+import static org.apache.datasketches.common.TestUtil.CHECK_CPP_FILES;
+import static org.apache.datasketches.common.TestUtil.GENERATE_JAVA_FILES;
+import static org.apache.datasketches.common.TestUtil.cppPath;
+import static org.apache.datasketches.common.TestUtil.javaPath;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+import java.io.IOException;
+import java.nio.file.Files;
+
+import org.apache.datasketches.memory.Memory;
+import org.testng.annotations.Test;
+
+/**
+ * Serialize binary sketches to be tested by C++ code.
+ * Test deserialization of binary sketches serialized by C++ code.
+ */
+public class BloomFilterCrossLanguageTest {
+
+ @Test(groups = {GENERATE_JAVA_FILES})
+ public void generatBloomFilterBinariesForCompatibilityTesting() throws IOException {
+ final int[] nArr = {0, 10_000, 2_000_000, 300_000_00};
+ final short[] hArr = {3, 5};
+ for (int n : nArr) {
+ for (short numHashes : hArr) {
+ final long configBits = Math.max(n, 1000L); // so empty still has valid bit size
+ BloomFilter bf = BloomFilterBuilder.createBySize(configBits, numHashes);
+ for (int i = 0; i < n / 10; ++i) {
+ bf.update(i);
+ }
+ if (n > 0) { bf.update(Float.NaN); }
+ assertEquals(bf.isEmpty(), n == 0);
+ assertTrue(bf.isEmpty() || (bf.getBitsUsed() > (n / 10)));
+ Files.newOutputStream(javaPath.resolve("bf_n" + n + "_h" + numHashes + "_java.sk")).write(bf.toByteArray());
+ }
+ }
+ }
+
+ @Test(groups = {CHECK_CPP_FILES})
+ public void readBloomFilterBinariesForCompatibilityTesting() throws IOException {
+ final int[] nArr = {0, 10_000, 2_000_000, 300_000_00};
+ final short[] hArr = {3, 5};
+ for (int n : nArr) {
+ for (short numHashes : hArr) {
+ final byte[] bytes = Files.readAllBytes(cppPath.resolve("bf_n" + n + "_h" + numHashes + "_cpp.sk"));
+ final BloomFilter bf = BloomFilter.heapify(Memory.wrap(bytes));
+ assertEquals(bf.isEmpty(), n == 0);
+ assertTrue(bf.isEmpty() || (bf.getBitsUsed() > (n / 10)));
+
+ for (int i = 0; i < n / 10; ++i) {
+ assertTrue(bf.query(i));
+ }
+ if (n > 0) {
+ assert(bf.query(Double.NaN));
+ }
+ }
+ }
+ }
+}
diff --git a/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java b/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java
index dedaf9db3..2d71805ac 100644
--- a/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java
+++ b/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java
@@ -28,10 +28,11 @@
import org.apache.datasketches.common.SketchesArgumentException;
import org.apache.datasketches.common.SketchesReadOnlyException;
import org.apache.datasketches.memory.Memory;
-import org.apache.datasketches.memory.WritableHandle;
import org.apache.datasketches.memory.WritableMemory;
import org.testng.annotations.Test;
+import java.lang.foreign.Arena;
+
public class BloomFilterTest {
@Test
@@ -51,8 +52,9 @@ public void createNewFilterTest() throws Exception {
assertFalse(bf1.isDirect());
assertFalse(bf1.isReadOnly());
- try (WritableHandle wh = WritableMemory.allocateDirect(sizeBytes)) {
- final WritableMemory wmem = wh.getWritable();
+
+ try (Arena arena = Arena.ofConfined()) {
+ WritableMemory wmem = WritableMemory.allocateDirect(sizeBytes, arena);
final BloomFilter bf2 = new BloomFilter(numBits, numHashes, seed, wmem);
assertTrue(bf2.isEmpty());
assertTrue(bf2.hasMemory());
@@ -157,8 +159,9 @@ public void basicFilterOperationsTest() {
int numFound = 0;
for (long i = 0; i < 2 * n; ++i) {
- if (bf.query(i))
+ if (bf.query(i)) {
++numFound;
+ }
}
assertTrue(numFound >= n);
assertTrue(numFound < 1.1 * n);
@@ -338,8 +341,12 @@ public void nonEmptySerializationTest() {
int fromBytesCount = 0;
for (int i = 0; i < numBits; ++i) {
boolean val = fromBytes.query(0.5 + i);
- if (val) ++fromBytesCount;
- if (i < n) assertTrue(val);
+ if (val) {
+ ++fromBytesCount;
+ }
+ if (i < n) {
+ assertTrue(val);
+ }
}
assertEquals(fromBytesCount, n + count); // same numbers of items should match
@@ -353,8 +360,12 @@ public void nonEmptySerializationTest() {
int fromLongsCount = 0;
for (int i = 0; i < numBits; ++i) {
boolean val = fromLongs.query(0.5 + i);
- if (val) ++fromLongsCount;
- if (i < n) assertTrue(val);
+ if (val) {
+ ++fromLongsCount;
+ }
+ if (i < n) {
+ assertTrue(val);
+ }
}
assertEquals(fromLongsCount, n + count); // same numbers of items should match
diff --git a/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayRTest.java b/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayRTest.java
index 521019e62..bbedd2fb7 100644
--- a/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayRTest.java
+++ b/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayRTest.java
@@ -142,8 +142,8 @@ public void bitAddressOutOfBoundsNonEmptyTest() {
final Memory mem = bitArrayToMemory(hba);
DirectBitArrayR dba = DirectBitArrayR.wrap(mem, hba.isEmpty());
- assertThrows(AssertionError.class, () -> dba.getBit(-10));
- assertThrows(AssertionError.class, () -> dba.getBit(2048));
+ assertThrows(IndexOutOfBoundsException.class, () -> dba.getBit(-10));
+ assertThrows(IndexOutOfBoundsException.class, () -> dba.getBit(2048));
}
@Test
diff --git a/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayTest.java b/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayTest.java
index a45bcbb82..8327a0d5e 100644
--- a/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayTest.java
+++ b/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayTest.java
@@ -139,7 +139,7 @@ public void basicWritableWrapTest() {
@Test
public void countWritableWrappedBitsWhenDirty() {
// like basicOperationTest but with setBit which does
- // not neecssarily track numBitsSet_
+ // not necessarily track numBitsSet_
final HeapBitArray hba = new HeapBitArray(128);
assertFalse(hba.getAndSetBit(1));
assertFalse(hba.getAndSetBit(2));
@@ -172,12 +172,12 @@ public void bitAddresOutOfBoundsNonEmptyTest() {
dba.getAndSetBit(i);
}
- assertThrows(AssertionError.class, () -> dba.getBit(-10));
- assertThrows(AssertionError.class, () -> dba.getBit(2048));
- assertThrows(AssertionError.class, () -> dba.setBit(-20));
- assertThrows(AssertionError.class, () -> dba.setBit(4096));
- assertThrows(AssertionError.class, () -> dba.getAndSetBit(-30));
- assertThrows(AssertionError.class, () -> dba.getAndSetBit(8192));
+ assertThrows(IndexOutOfBoundsException.class, () -> dba.getBit(-10));
+ assertThrows(IndexOutOfBoundsException.class, () -> dba.getBit(2048));
+ assertThrows(IndexOutOfBoundsException.class, () -> dba.setBit(-20));
+ assertThrows(IndexOutOfBoundsException.class, () -> dba.setBit(4096));
+ assertThrows(IndexOutOfBoundsException.class, () -> dba.getAndSetBit(-30));
+ assertThrows(IndexOutOfBoundsException.class, () -> dba.getAndSetBit(8192));
}
@Test
diff --git a/src/test/java/org/apache/datasketches/hash/MurmurHash3AdaptorTest.java b/src/test/java/org/apache/datasketches/hash/MurmurHash3AdaptorTest.java
deleted file mode 100644
index 16a8e6652..000000000
--- a/src/test/java/org/apache/datasketches/hash/MurmurHash3AdaptorTest.java
+++ /dev/null
@@ -1,377 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.datasketches.hash;
-
-import static org.apache.datasketches.hash.MurmurHash3Adaptor.asDouble;
-import static org.apache.datasketches.hash.MurmurHash3Adaptor.asInt;
-import static org.apache.datasketches.hash.MurmurHash3Adaptor.hashToBytes;
-import static org.apache.datasketches.hash.MurmurHash3Adaptor.hashToLongs;
-import static org.apache.datasketches.hash.MurmurHash3Adaptor.modulo;
-
-import org.apache.datasketches.common.SketchesArgumentException;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-/**
- * @author Lee Rhodes
- */
-public class MurmurHash3AdaptorTest {
-
- @Test
- public void checkToBytesLong() {
- byte[] result = hashToBytes(2L, 0L);
- for (int i = 8; i-- > 0;) {
- Assert.assertNotEquals(result[i], 0);
- }
- }
-
- @Test
- public void checkToBytesLongArr() {
- long[] arr = { 1L, 2L };
- byte[] result = hashToBytes(arr, 0L);
- for (int i = 8; i-- > 0;) {
- Assert.assertNotEquals(result[i], 0);
- }
- arr = null;
- result = hashToBytes(arr, 0L);
- Assert.assertEquals(result, null);
-
- arr = new long[0];
- result = hashToBytes(arr, 0L);
- Assert.assertEquals(result, null);
- }
-
- @Test
- public void checkToBytesIntArr() {
- int[] arr = { 1, 2 };
- byte[] result = hashToBytes(arr, 0L);
- for (int i = 8; i-- > 0;) {
- Assert.assertNotEquals(result[i], 0);
- }
- arr = null;
- result = hashToBytes(arr, 0L);
- Assert.assertEquals(result, null);
-
- arr = new int[0];
- result = hashToBytes(arr, 0L);
- Assert.assertEquals(result, null);
- }
-
- @Test
- public void checkToBytesCharArr() {
- char[] arr = { 1, 2 };
- byte[] result = hashToBytes(arr, 0L);
- for (int i = 8; i-- > 0;) {
- Assert.assertNotEquals(result[i], 0);
- }
- arr = null;
- result = hashToBytes(arr, 0L);
- Assert.assertEquals(result, null);
-
- arr = new char[0];
- result = hashToBytes(arr, 0L);
- Assert.assertEquals(result, null);
- }
-
- @Test
- public void checkToBytesByteArr() {
- byte[] arr = { 1, 2 };
- byte[] result = hashToBytes(arr, 0L);
- for (int i = 8; i-- > 0;) {
- Assert.assertNotEquals(result[i], 0);
- }
- arr = null;
- result = hashToBytes(arr, 0L);
- Assert.assertEquals(result, null);
-
- arr = new byte[0];
- result = hashToBytes(arr, 0L);
- Assert.assertEquals(result, null);
-
- }
-
- @Test
- public void checkToBytesDouble() {
- byte[] result = hashToBytes(1.0, 0L);
- for (int i = 8; i-- > 0;) {
- Assert.assertNotEquals(result[i], 0);
- }
- result = hashToBytes(0.0, 0L);
- for (int i = 8; i-- > 0;) {
- Assert.assertNotEquals(result[i], 0);
- }
- result = hashToBytes( -0.0, 0L);
- for (int i = 8; i-- > 0;) {
- Assert.assertNotEquals(result[i], 0);
- }
- }
-
- @Test
- public void checkToBytesString() {
- byte[] result = hashToBytes("1", 0L);
- for (int i = 8; i-- > 0;) {
- Assert.assertNotEquals(result[i], 0);
- }
- result = hashToBytes("", 0L);
- Assert.assertEquals(result, null);
-
- String s = null;
- result = hashToBytes(s, 0L);
- Assert.assertEquals(result, null);
- }
-
- /************/
-
- @Test
- public void checkToLongsLong() {
- long[] result = hashToLongs(2L, 0L);
- for (int i = 2; i-- > 0;) {
- Assert.assertNotEquals(result[i], 0);
- }
- }
-
- @Test
- public void checkToLongsLongArr() {
- long[] arr = { 1L, 2L };
- long[] result = hashToLongs(arr, 0L);
- for (int i = 2; i-- > 0;) {
- Assert.assertNotEquals(result[i], 0);
- }
- arr = null;
- result = hashToLongs(arr, 0L);
- Assert.assertEquals(result, null);
-
- arr = new long[0];
- result = hashToLongs(arr, 0L);
- Assert.assertEquals(result, null);
- }
-
- @Test
- public void checkToLongsIntArr() {
- int[] arr = { 1, 2 };
- long[] result = hashToLongs(arr, 0L);
- for (int i = 2; i-- > 0;) {
- Assert.assertNotEquals(result[i], 0);
- }
- arr = null;
- result = hashToLongs(arr, 0L);
- Assert.assertEquals(result, null);
-
- arr = new int[0];
- result = hashToLongs(arr, 0L);
- Assert.assertEquals(result, null);
- }
-
- @Test
- public void checkToLongsCharArr() {
- char[] arr = { 1, 2 };
- long[] result = hashToLongs(arr, 0L);
- for (int i = 2; i-- > 0;) {
- Assert.assertNotEquals(result[i], 0);
- }
- arr = null;
- result = hashToLongs(arr, 0L);
- Assert.assertEquals(result, null);
-
- arr = new char[0];
- result = hashToLongs(arr, 0L);
- Assert.assertEquals(result, null);
- }
-
- @Test
- public void checkToLongsByteArr() {
- byte[] arr = { 1, 2 };
- long[] result = hashToLongs(arr, 0L);
- for (int i = 2; i-- > 0;) {
- Assert.assertNotEquals(result[i], 0);
- }
- arr = null;
- result = hashToLongs(arr, 0L);
- Assert.assertEquals(result, null);
-
- arr = new byte[0];
- result = hashToLongs(arr, 0L);
- Assert.assertEquals(result, null);
-
- }
-
- @Test
- public void checkToLongsDouble() {
- long[] result = hashToLongs(1.0, 0L);
- for (int i = 2; i-- > 0;) {
- Assert.assertNotEquals(result[i], 0);
- }
- result = hashToLongs(0.0, 0L);
- for (int i = 2; i-- > 0;) {
- Assert.assertNotEquals(result[i], 0);
- }
- result = hashToLongs( -0.0, 0L);
- for (int i = 2; i-- > 0;) {
- Assert.assertNotEquals(result[i], 0);
- }
- }
-
- @Test
- public void checkToLongsString() {
- long[] result = hashToLongs("1", 0L);
- for (int i = 2; i-- > 0;) {
- Assert.assertNotEquals(result[i], 0);
- }
- result = hashToLongs("", 0L);
- Assert.assertEquals(result, null);
- String s = null;
- result = hashToLongs(s, 0L);
- Assert.assertEquals(result, null);
- }
-
- /*************/
-
- @Test
- public void checkModulo() {
- int div = 7;
- for (int i = 20; i-- > 0;) {
- long[] out = hashToLongs(i, 9001);
- int mod = modulo(out[0], out[1], div);
- Assert.assertTrue((mod < div) && (mod >= 0));
- mod = modulo(out, div);
- Assert.assertTrue((mod < div) && (mod >= 0));
- }
- }
-
- @Test
- public void checkAsDouble() {
- for (int i = 0; i < 10000; i++ ) {
- double result = asDouble(hashToLongs(i, 0));
- Assert.assertTrue((result >= 0) && (result < 1.0));
- }
- }
-
- //Check asInt() functions
-
- @Test
- public void checkAsInt() {
- int lo = (3 << 28);
- int hi = (1 << 30) + 1;
- for (byte i = 0; i < 126; i++ ) {
- long[] longArr = {i, i+1}; //long[]
- int result = asInt(longArr, lo);
- Assert.assertTrue((result >= 0) && (result < lo));
- result = asInt(longArr, hi);
- Assert.assertTrue((result >= 0) && (result < hi));
-
- int[] intArr = {i, i+1}; //int[]
- result = asInt(intArr, lo);
- Assert.assertTrue((result >= 0) && (result < lo));
- result = asInt(intArr, hi);
- Assert.assertTrue((result >= 0) && (result < hi));
-
- byte[] byteArr = {i, (byte)(i+1)}; //byte[]
- result = asInt(byteArr, lo);
- Assert.assertTrue((result >= 0) && (result < lo));
- result = asInt(byteArr, hi);
- Assert.assertTrue((result >= 0) && (result < hi));
-
- long longV = i; //long
- result = asInt(longV, lo);
- Assert.assertTrue((result >= 0) && (result < lo));
- result = asInt(longV, hi);
- Assert.assertTrue((result >= 0) && (result < hi));
-
- double v = i; //double
- result = asInt(v, lo);
- Assert.assertTrue((result >= 0) && (result < lo));
- result = asInt(v, hi);
- Assert.assertTrue((result >= 0) && (result < hi));
-
- String s = Integer.toString(i); //String
- result = asInt(s, lo);
- Assert.assertTrue((result >= 0) && (result < lo));
- result = asInt(s, hi);
- Assert.assertTrue((result >= 0) && (result < hi));
- }
- }
-
- @Test (expectedExceptions = SketchesArgumentException.class)
- public void checkAsIntCornerCaseLongNull() {
- long[] arr = null;
- asInt(arr, 1000);
- }
-
- @Test (expectedExceptions = SketchesArgumentException.class)
- public void checkAsIntCornerCaseLongEmpty() {
- long[] arr = new long[0];
- asInt(arr, 1000);
- }
-
- @Test (expectedExceptions = SketchesArgumentException.class)
- public void checkAsIntCornerCaseIntNull() {
- int[] arr = null;
- asInt(arr, 1000);
- }
-
- @Test (expectedExceptions = SketchesArgumentException.class)
- public void checkAsIntCornerCaseIntEmpty() {
- int[] arr = new int[0];
- asInt(arr, 1000);
- }
-
- @Test (expectedExceptions = SketchesArgumentException.class)
- public void checkAsIntCornerCaseByteNull() {
- byte[] arr = null;
- asInt(arr, 1000);
- }
-
- @Test (expectedExceptions = SketchesArgumentException.class)
- public void checkAsIntCornerCaseByteEmpty() {
- byte[] arr = new byte[0];
- asInt(arr, 1000);
- }
-
- @Test (expectedExceptions = SketchesArgumentException.class)
- public void checkAsIntCornerCaseStringNull() {
- String s = null;
- asInt(s, 1000);
- }
-
- @Test (expectedExceptions = SketchesArgumentException.class)
- public void checkAsIntCornerCaseStringEmpty() {
- String s = "";
- asInt(s, 1000);
- }
-
- @Test (expectedExceptions = SketchesArgumentException.class)
- public void checkAsIntCornerCaseNTooSmall() {
- String s = "abc";
- asInt(s, 1);
- }
-
- @Test
- public void printlnTest() {
- println("PRINTING: "+this.getClass().getName());
- }
-
- /**
- * @param s value to print
- */
- static void println(String s) {
- //System.out.println(s); //disable here
- }
-
-}
diff --git a/src/test/java/org/apache/datasketches/hash/MurmurHash3v2Test.java b/src/test/java/org/apache/datasketches/hash/MurmurHash3FFM21Test.java
similarity index 82%
rename from src/test/java/org/apache/datasketches/hash/MurmurHash3v2Test.java
rename to src/test/java/org/apache/datasketches/hash/MurmurHash3FFM21Test.java
index 23f369e63..d9771d304 100644
--- a/src/test/java/org/apache/datasketches/hash/MurmurHash3v2Test.java
+++ b/src/test/java/org/apache/datasketches/hash/MurmurHash3FFM21Test.java
@@ -28,13 +28,12 @@
import org.testng.annotations.Test;
import org.apache.datasketches.memory.Memory;
-import org.apache.datasketches.memory.MurmurHash3v2;
import org.apache.datasketches.memory.WritableMemory;
/**
* @author Lee Rhodes
*/
-public class MurmurHash3v2Test {
+public class MurmurHash3FFM21Test {
private Random rand = new Random();
private static final int trials = 1 << 20;
@@ -154,37 +153,35 @@ private static final long[] hashV1(byte[] key, long seed) {
}
private static final long[] hashV2(long[] key, long seed) {
- return MurmurHash3v2.hash(key, seed);
+ return MurmurHash3FFM21.hash(key, seed);
}
private static final long[] hashV2(int[] key2, long seed) {
- return MurmurHash3v2.hash(key2, seed);
+ return MurmurHash3FFM21.hash(key2, seed);
}
private static final long[] hashV2(char[] key, long seed) {
- return MurmurHash3v2.hash(key, seed);
+ return MurmurHash3FFM21.hash(key, seed);
}
private static final long[] hashV2(byte[] key, long seed) {
- return MurmurHash3v2.hash(key, seed);
+ return MurmurHash3FFM21.hash(key, seed);
}
//V2 single primitives
private static final long[] hashV2(long key, long seed, long[] out) {
- return MurmurHash3v2.hash(key, seed, out);
+ return MurmurHash3FFM21.hash(key, seed, out);
}
// private static final long[] hashV2(double key, long seed, long[] out) {
-// return MurmurHash3v2.hash(key, seed, out);
+// return MurmurHash3v4.hash(key, seed, out);
// }
// private static final long[] hashV2(String key, long seed, long[] out) {
-// return MurmurHash3v2.hash(key, seed, out);
+// return MurmurHash3v4.hash(key, seed, out);
// }
-
-
@Test
public void offsetChecks() {
long seed = 12345;
@@ -199,7 +196,7 @@ public void offsetChecks() {
for (int offset = 0; offset < 16; offset++) {
int arrLen = cap - offset;
- hash1 = MurmurHash3v2.hash(wmem, offset, arrLen, seed, hash1);
+ hash1 = MurmurHash3FFM21.hash(wmem, offset, arrLen, seed, hash1);
byte[] byteArr2 = new byte[arrLen];
wmem.getByteArray(offset, byteArr2, 0, arrLen);
hash2 = MurmurHash3.hash(byteArr2, seed);
@@ -222,8 +219,8 @@ public void byteArrChecks() {
for (int i = 0; i < j; i++) { wmem.putByte(i, (byte) (-128 + i)); }
long[] hash1 = MurmurHash3.hash(in, 0);
- hash2 = MurmurHash3v2.hash(wmem, offset, bytes, seed, hash2);
- long[] hash3 = MurmurHash3v2.hash(in, seed);
+ hash2 = MurmurHash3FFM21.hash(wmem, offset, bytes, seed, hash2);
+ long[] hash3 = MurmurHash3FFM21.hash(in, seed);
assertEquals(hash1, hash2);
assertEquals(hash1, hash3);
@@ -246,8 +243,8 @@ public void charArrChecks() {
for (int i = 0; i < j; i++) { wmem.putInt(i, i); }
long[] hash1 = MurmurHash3.hash(in, 0);
- hash2 = MurmurHash3v2.hash(wmem, offset, bytes, seed, hash2);
- long[] hash3 = MurmurHash3v2.hash(in, seed);
+ hash2 = MurmurHash3FFM21.hash(wmem, offset, bytes, seed, hash2);
+ long[] hash3 = MurmurHash3FFM21.hash(in, seed);
assertEquals(hash1, hash2);
assertEquals(hash1, hash3);
@@ -270,8 +267,8 @@ public void intArrChecks() {
for (int i = 0; i < j; i++) { wmem.putInt(i, i); }
long[] hash1 = MurmurHash3.hash(in, 0);
- hash2 = MurmurHash3v2.hash(wmem, offset, bytes, seed, hash2);
- long[] hash3 = MurmurHash3v2.hash(in, seed);
+ hash2 = MurmurHash3FFM21.hash(wmem, offset, bytes, seed, hash2);
+ long[] hash3 = MurmurHash3FFM21.hash(in, seed);
assertEquals(hash1, hash2);
assertEquals(hash1, hash3);
@@ -294,8 +291,8 @@ public void longArrChecks() {
for (int i = 0; i < j; i++) { wmem.putLong(i, i); }
long[] hash1 = MurmurHash3.hash(in, 0);
- hash2 = MurmurHash3v2.hash(wmem, offset, bytes, seed, hash2);
- long[] hash3 = MurmurHash3v2.hash(in, seed);
+ hash2 = MurmurHash3FFM21.hash(wmem, offset, bytes, seed, hash2);
+ long[] hash3 = MurmurHash3FFM21.hash(in, seed);
assertEquals(hash1, hash2);
assertEquals(hash1, hash3);
@@ -313,8 +310,8 @@ public void longCheck() {
WritableMemory wmem = WritableMemory.writableWrap(in);
long[] hash1 = MurmurHash3.hash(in, 0);
- hash2 = MurmurHash3v2.hash(wmem, offset, bytes, seed, hash2);
- long[] hash3 = MurmurHash3v2.hash(in, seed);
+ hash2 = MurmurHash3FFM21.hash(wmem, offset, bytes, seed, hash2);
+ long[] hash3 = MurmurHash3FFM21.hash(in, seed);
assertEquals(hash1, hash2);
assertEquals(hash1, hash3);
@@ -325,62 +322,57 @@ public void checkEmptiesNulls() {
long seed = 123;
long[] hashOut = new long[2];
try {
- MurmurHash3v2.hash(Memory.wrap(new long[0]), 0, 0, seed, hashOut); //mem empty
- fail();
- } catch (final IllegalArgumentException e) { } //OK
- try {
- Memory mem = null;
- MurmurHash3v2.hash(mem, 0, 0, seed, hashOut); //mem null
+ MurmurHash3FFM21.hash(Memory.wrap(new long[0]), 0, 0, seed, hashOut); //mem empty
fail();
} catch (final IllegalArgumentException e) { } //OK
try {
String s = "";
- MurmurHash3v2.hash(s, seed, hashOut); //string empty
+ MurmurHash3FFM21.hash(s, seed, hashOut); //string empty
fail();
} catch (final IllegalArgumentException e) { } //OK
try {
String s = null;
- MurmurHash3v2.hash(s, seed, hashOut); //string null
+ MurmurHash3FFM21.hash(s, seed, hashOut); //string null
fail();
} catch (final IllegalArgumentException e) { } //OK
try {
byte[] barr = new byte[0];
- MurmurHash3v2.hash(barr, seed); //byte[] empty
+ MurmurHash3FFM21.hash(barr, seed); //byte[] empty
fail();
} catch (final IllegalArgumentException e) { } //OK
try {
byte[] barr = null;
- MurmurHash3v2.hash(barr, seed); //byte[] null
+ MurmurHash3FFM21.hash(barr, seed); //byte[] null
fail();
} catch (final IllegalArgumentException e) { } //OK
try {
char[] carr = new char[0];
- MurmurHash3v2.hash(carr, seed); //char[] empty
+ MurmurHash3FFM21.hash(carr, seed); //char[] empty
fail();
} catch (final IllegalArgumentException e) { } //OK
try {
char[] carr = null;
- MurmurHash3v2.hash(carr, seed); //char[] null
+ MurmurHash3FFM21.hash(carr, seed); //char[] null
fail();
} catch (final IllegalArgumentException e) { } //OK
try {
int[] iarr = new int[0];
- MurmurHash3v2.hash(iarr, seed); //int[] empty
+ MurmurHash3FFM21.hash(iarr, seed); //int[] empty
fail();
} catch (final IllegalArgumentException e) { } //OK
try {
int[] iarr = null;
- MurmurHash3v2.hash(iarr, seed); //int[] null
+ MurmurHash3FFM21.hash(iarr, seed); //int[] null
fail();
} catch (final IllegalArgumentException e) { } //OK
try {
long[] larr = new long[0];
- MurmurHash3v2.hash(larr, seed); //long[] empty
+ MurmurHash3FFM21.hash(larr, seed); //long[] empty
fail();
} catch (final IllegalArgumentException e) { } //OK
try {
long[] larr = null;
- MurmurHash3v2.hash(larr, seed); //long[] null
+ MurmurHash3FFM21.hash(larr, seed); //long[] null
fail();
} catch (final IllegalArgumentException e) { } //OK
}
@@ -390,9 +382,9 @@ public void checkStringLong() {
long seed = 123;
long[] hashOut = new long[2];
String s = "123";
- assertTrue(MurmurHash3v2.hash(s, seed, hashOut)[0] != 0);
+ assertTrue(MurmurHash3FFM21.hash(s, seed, hashOut)[0] != 0);
long v = 123;
- assertTrue(MurmurHash3v2.hash(v, seed, hashOut)[0] != 0);
+ assertTrue(MurmurHash3FFM21.hash(v, seed, hashOut)[0] != 0);
}
@Test
@@ -420,8 +412,8 @@ private static long[] checkDouble(double dbl) {
WritableMemory wmem = WritableMemory.writableWrap(dataArr);
long[] hash1 = MurmurHash3.hash(dataArr, 0);
- hash2 = MurmurHash3v2.hash(wmem, offset, bytes, seed, hash2);
- long[] hash3 = MurmurHash3v2.hash(dbl, seed, hash2);
+ hash2 = MurmurHash3FFM21.hash(wmem, offset, bytes, seed, hash2);
+ long[] hash3 = MurmurHash3FFM21.hash(dbl, seed, hash2);
assertEquals(hash1, hash2);
assertEquals(hash1, hash3);
diff --git a/src/test/java/org/apache/datasketches/hash/MurmurHash3FFM21bTest.java b/src/test/java/org/apache/datasketches/hash/MurmurHash3FFM21bTest.java
new file mode 100644
index 000000000..1e23772b5
--- /dev/null
+++ b/src/test/java/org/apache/datasketches/hash/MurmurHash3FFM21bTest.java
@@ -0,0 +1,419 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hash;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.datasketches.hash.MurmurHash3FFM21.hash;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+import java.lang.foreign.Arena;
+import java.lang.foreign.MemorySegment;
+import java.nio.ByteOrder;
+
+import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.memory.MemoryRequestServer;
+import org.apache.datasketches.memory.Resource;
+import org.apache.datasketches.memory.WritableMemory;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+/**
+ * Tests the MurmurHash3 against specific, known hash results given known
+ * inputs obtained from the public domain C++ version 150.
+ *
+ * @author Lee Rhodes
+ */
+public class MurmurHash3FFM21bTest {
+ private static final MemoryRequestServer memReqSvr = Resource.defaultMemReqSvr;
+
+ @Test
+ public void checkByteArrRemainderGT8() { //byte[], remainder > 8
+ String keyStr = "The quick brown fox jumps over the lazy dog";
+ byte[] key = keyStr.getBytes(UTF_8);
+ long[] result = hash(key, 0);
+ //Should be:
+ long h1 = 0xe34bbc7bbc071b6cL;
+ long h2 = 0x7a433ca9c49a9347L;
+ Assert.assertEquals(result[0], h1);
+ Assert.assertEquals(result[1], h2);
+ }
+
+ @Test
+ public void checkByteArrRemainderGT8withSegment() { //byte[], remainder > 8
+ String keyStr = "The quick brown fox jumps over the lazy dog";
+ byte[] key = keyStr.getBytes(UTF_8);
+ long[] out = new long[2];
+ MemorySegment seg = MemorySegment.ofArray(key);
+ long[] result = hash(seg, 0, seg.byteSize(), 0, out);
+ //Should be:
+ long h1 = 0xe34bbc7bbc071b6cL;
+ long h2 = 0x7a433ca9c49a9347L;
+ Assert.assertEquals(result[0], h1);
+ Assert.assertEquals(result[1], h2);
+ }
+
+ @Test
+ public void checkByteArrChange1bit() { //byte[], change one bit
+ String keyStr = "The quick brown fox jumps over the lazy eog";
+ byte[] key = keyStr.getBytes(UTF_8);
+ long[] result = hash(key, 0);
+ //Should be:
+ long h1 = 0x362108102c62d1c9L;
+ long h2 = 0x3285cd100292b305L;
+ Assert.assertEquals(result[0], h1);
+ Assert.assertEquals(result[1], h2);
+ }
+
+ @Test
+ public void checkByteArrRemainderLt8() { //byte[], test a remainder < 8
+ String keyStr = "The quick brown fox jumps over the lazy dogdogdog";
+ byte[] key = keyStr.getBytes(UTF_8);
+ long[] result = hash(key, 0);
+ //Should be;
+ long h1 = 0x9c8205300e612fc4L;
+ long h2 = 0xcbc0af6136aa3df9L;
+ Assert.assertEquals(result[0], h1);
+ Assert.assertEquals(result[1], h2);
+ }
+
+ @Test
+ public void checkByteArrReaminderEQ8() { //byte[], test a remainder = 8
+ String keyStr = "The quick brown fox jumps over the lazy1";
+ byte[] key = keyStr.getBytes(UTF_8);
+ long[] result = hash(key, 0);
+ //Should be:
+ long h1 = 0xe3301a827e5cdfe3L;
+ long h2 = 0xbdbf05f8da0f0392L;
+ Assert.assertEquals(result[0], h1);
+ Assert.assertEquals(result[1], h2);
+ }
+
+ /**
+ * This test should have the exact same output as Test4
+ */
+ @Test
+ public void checkLongArrRemainderEQ8() { //long[], test a remainder = 8
+ String keyStr = "The quick brown fox jumps over the lazy1";
+ long[] key = stringToLongs(keyStr);
+ long[] result = hash(key, 0);
+ //Should be:
+ long h1 = 0xe3301a827e5cdfe3L;
+ long h2 = 0xbdbf05f8da0f0392L;
+ Assert.assertEquals(result[0], h1);
+ Assert.assertEquals(result[1], h2);
+ }
+
+ /**
+ * This test should have the exact same output as Test4
+ */
+ @Test
+ public void checkIntArrRemainderEQ8() { //int[], test a remainder = 8
+ String keyStr = "The quick brown fox jumps over the lazy1"; //40B
+ int[] key = stringToInts(keyStr);
+ long[] result = hash(key, 0);
+ //Should be:
+ long h1 = 0xe3301a827e5cdfe3L;
+ long h2 = 0xbdbf05f8da0f0392L;
+ Assert.assertEquals(result[0], h1);
+ Assert.assertEquals(result[1], h2);
+ }
+
+ @Test
+ public void checkIntArrRemainderEQ0() { //int[], test a remainder = 0
+ String keyStr = "The quick brown fox jumps over t"; //32B
+ int[] key = stringToInts(keyStr);
+ long[] result = hash(key, 0);
+ //Should be:
+ long h1 = 0xdf6af91bb29bdacfL;
+ long h2 = 0x91a341c58df1f3a6L;
+ Assert.assertEquals(result[0], h1);
+ Assert.assertEquals(result[1], h2);
+ }
+
+ /**
+ * Tests an odd remainder of int[].
+ */
+ @Test
+ public void checkIntArrOddRemainder() { //int[], odd remainder
+ String keyStr = "The quick brown fox jumps over the lazy dog"; //43B
+ int[] key = stringToInts(keyStr);
+ long[] result = hash(key, 0);
+ //Should be:
+ long h1 = 0x1eb232b0087543f5L;
+ long h2 = 0xfc4c1383c3ace40fL;
+ Assert.assertEquals(result[0], h1);
+ Assert.assertEquals(result[1], h2);
+ }
+
+ /**
+ * Tests an odd remainder of int[].
+ */
+ @Test
+ public void checkCharArrOddRemainder() { //char[], odd remainder
+ String keyStr = "The quick brown fox jumps over the lazy dog.."; //45B
+ char[] key = keyStr.toCharArray();
+ long[] result = hash(key, 0);
+ //Should be:
+ long h1 = 0xca77b498ea9ed953L;
+ long h2 = 0x8b8f8ec3a8f4657eL;
+ Assert.assertEquals(result[0], h1);
+ Assert.assertEquals(result[1], h2);
+ }
+
+ /**
+ * Tests an odd remainder of int[].
+ */
+ @Test
+ public void checkCharArrRemainderEQ0() { //char[], remainder of 0
+ String keyStr = "The quick brown fox jumps over the lazy "; //40B
+ char[] key = keyStr.toCharArray();
+ long[] result = hash(key, 0);
+ //Should be:
+ long h1 = 0x51b15e9d0887f9f1L;
+ long h2 = 0x8106d226786511ebL;
+ Assert.assertEquals(result[0], h1);
+ Assert.assertEquals(result[1], h2);
+ }
+
+ @Test
+ public void checkByteArrAllOnesZeros() { //byte[], test a ones byte and a zeros byte
+ byte[] key = {
+ 0x54, 0x68, 0x65, 0x20, 0x71, 0x75, 0x69, 0x63, 0x6b, 0x20, 0x62, 0x72, 0x6f, 0x77, 0x6e,
+ 0x20, 0x66, 0x6f, 0x78, 0x20, 0x6a, 0x75, 0x6d, 0x70, 0x73, 0x20, 0x6f, 0x76, 0x65,
+ 0x72, 0x20, 0x74, 0x68, 0x65, 0x20, 0x6c, 0x61, 0x7a, 0x79, 0x20, 0x64, 0x6f, 0x67,
+ (byte) 0xff, 0x64, 0x6f, 0x67, 0x00
+ };
+ long[] result = MurmurHash3FFM21.hash(key, 0);
+ //Should be:
+ long h1 = 0xe88abda785929c9eL;
+ long h2 = 0x96b98587cacc83d6L;
+ Assert.assertEquals(result[0], h1);
+ Assert.assertEquals(result[1], h2);
+ }
+
+ /**
+ * This test demonstrates that the hash of byte[], char[], int[], or long[] will produce the
+ * same hash result if, and only if, all the arrays have the same exact length in bytes, and if
+ * the contents of the values in the arrays have the same byte endianness and overall order.
+ */
+ @Test
+ public void checkCrossTypeHashConsistency() {
+ long[] out;
+ println("Bytes");
+ byte[] bArr = {1,2,3,4,5,6,7,8, 9,10,11,12,13,14,15,16, 17,18,19,20,21,22,23,24};
+ long[] out1 = hash(bArr, 0L);
+ println(longToHexBytes(out1[0]));
+ println(longToHexBytes(out1[1]));
+
+ println("Chars");
+ char[] cArr = {0X0201, 0X0403, 0X0605, 0X0807, 0X0a09, 0X0c0b, 0X0e0d, 0X100f,
+ 0X1211, 0X1413, 0X1615, 0X1817};
+ out = hash(cArr, 0L);
+ Assert.assertEquals(out, out1);
+ println(longToHexBytes(out[0]));
+ println(longToHexBytes(out[1]));
+
+ println("Ints");
+ int[] iArr = {0X04030201, 0X08070605, 0X0c0b0a09, 0X100f0e0d, 0X14131211, 0X18171615};
+ out = hash(iArr, 0L);
+ Assert.assertEquals(out, out1);
+ println(longToHexBytes(out[0]));
+ println(longToHexBytes(out[1]));
+
+ println("Longs");
+ long[] lArr = {0X0807060504030201L, 0X100f0e0d0c0b0a09L, 0X1817161514131211L};
+ out = hash(lArr, 0L);
+ Assert.assertEquals(out, out1);
+ println(longToHexBytes(out[0]));
+ println(longToHexBytes(out[1]));
+ }
+
+ @Test
+ public void checkEmptyOrNullExceptions() {
+ try {
+ long[] arr = null;
+ hash(arr, 1L);
+ fail();
+ }
+ catch (final IllegalArgumentException e) { }
+ try {
+ int[] arr = null; hash(arr, 1L); fail();
+ } catch (final IllegalArgumentException e) { }
+ try {
+ char[] arr = null; hash(arr, 1L); fail();
+ } catch (final IllegalArgumentException e) { }
+ try {
+ byte[] arr = null; hash(arr, 1L); fail();
+ } catch (final IllegalArgumentException e) { }
+
+ long[] out = new long[2];
+ try {
+ String in = null; hash(in, 1L, out); fail();
+ } catch (final IllegalArgumentException e) { }
+ try {
+ Memory mem = Memory.wrap(new byte[0]);
+ hash(mem, 0L, 4L, 1L, out);
+ } catch (final IllegalArgumentException e) { }
+ try (Arena arena = Arena.ofConfined()) {
+ Memory mem = WritableMemory.allocateDirect(8, 1, ByteOrder.nativeOrder(), memReqSvr, arena);
+ out = hash(mem, 0L, 4L, 1L, out);
+ }
+ assertTrue((out[0] != 0) && (out[1] != 0));
+ }
+
+ @Test
+ public void checkHashTails() {
+ long[] out = new long[2];
+ WritableMemory mem = WritableMemory.allocate(32);
+ mem.fill((byte)85);
+
+ for (int i = 16; i <= 32; i++) {
+ out = hash(mem, 0, i, 1L, out);
+ }
+ }
+
+ @Test
+ public void checkSinglePrimitives() {
+ long[] out = new long[2];
+ out = hash(1L, 1L, out);
+ out = hash(0.0, 1L, out);
+ out = hash("123", 1L, out);
+ }
+
+ //Helper methods
+
+ private static long[] stringToLongs(String in) {
+ byte[] bArr = in.getBytes(UTF_8);
+ int inLen = bArr.length;
+ int outLen = (inLen / 8) + (((inLen % 8) != 0) ? 1 : 0);
+ long[] out = new long[outLen];
+
+ for (int i = 0; i < (outLen - 1); i++ ) {
+ for (int j = 0; j < 8; j++ ) {
+ out[i] |= ((bArr[(i * 8) + j] & 0xFFL) << (j * 8));
+ }
+ }
+ int inTail = 8 * (outLen - 1);
+ int rem = inLen - inTail;
+ for (int j = 0; j < rem; j++ ) {
+ out[outLen - 1] |= ((bArr[inTail + j] & 0xFFL) << (j * 8));
+ }
+ return out;
+ }
+
+ private static int[] stringToInts(String in) {
+ byte[] bArr = in.getBytes(UTF_8);
+ int inLen = bArr.length;
+ int outLen = (inLen / 4) + (((inLen % 4) != 0) ? 1 : 0);
+ int[] out = new int[outLen];
+
+ for (int i = 0; i < (outLen - 1); i++ ) {
+ for (int j = 0; j < 4; j++ ) {
+ out[i] |= ((bArr[(i * 4) + j] & 0xFFL) << (j * 8));
+ }
+ }
+ int inTail = 4 * (outLen - 1);
+ int rem = inLen - inTail;
+ for (int j = 0; j < rem; j++ ) {
+ out[outLen - 1] |= ((bArr[inTail + j] & 0xFFL) << (j * 8));
+ }
+ return out;
+ }
+
+ /**
+ * Returns a string of spaced hex bytes in Big-Endian order.
+ * @param v the given long
+ * @return string of spaced hex bytes in Big-Endian order.
+ */
+ private static String longToHexBytes(final long v) {
+ final long mask = 0XFFL;
+ final StringBuilder sb = new StringBuilder();
+ for (int i = 8; i-- > 0; ) {
+ final String s = Long.toHexString((v >>> (i * 8)) & mask);
+ sb.append(zeroPad(s, 2)).append(" ");
+ }
+ return sb.toString();
+ }
+
+ /**
+ * Prepend the given string with zeros. If the given string is equal or greater than the given
+ * field length, it will be returned without modification.
+ * @param s the given string
+ * @param fieldLength desired total field length including the given string
+ * @return the given string prepended with zeros.
+ */
+ private static final String zeroPad(final String s, final int fieldLength) {
+ return characterPad(s, fieldLength, '0', false);
+ }
+
+ /**
+ * Prepend or postpend the given string with the given character to fill the given field length.
+ * If the given string is equal or greater than the given field length, it will be returned
+ * without modification.
+ * @param s the given string
+ * @param fieldLength the desired field length
+ * @param padChar the desired pad character
+ * @param postpend if true append the padCharacters to the end of the string.
+ * @return prepended or postpended given string with the given character to fill the given field
+ * length.
+ */
+ private static final String characterPad(final String s, final int fieldLength, final char padChar,
+ final boolean postpend) {
+ final char[] chArr = s.toCharArray();
+ final int sLen = chArr.length;
+ if (sLen < fieldLength) {
+ final char[] out = new char[fieldLength];
+ final int blanks = fieldLength - sLen;
+
+ if (postpend) {
+ for (int i = 0; i < sLen; i++) {
+ out[i] = chArr[i];
+ }
+ for (int i = sLen; i < fieldLength; i++) {
+ out[i] = padChar;
+ }
+ } else { //prepend
+ for (int i = 0; i < blanks; i++) {
+ out[i] = padChar;
+ }
+ for (int i = blanks; i < fieldLength; i++) {
+ out[i] = chArr[i - blanks];
+ }
+ }
+
+ return String.valueOf(out);
+ }
+ return s;
+ }
+
+ @Test
+ public void printlnTest() {
+ println("PRINTING: "+this.getClass().getName());
+ }
+
+ /**
+ * @param s value to print
+ */
+ static void println(String s) {
+ //System.out.println(s); //disable here
+ }
+
+}
diff --git a/src/test/java/org/apache/datasketches/hll/CouponListTest.java b/src/test/java/org/apache/datasketches/hll/CouponListTest.java
index dbd5bfb0b..774e2c304 100644
--- a/src/test/java/org/apache/datasketches/hll/CouponListTest.java
+++ b/src/test/java/org/apache/datasketches/hll/CouponListTest.java
@@ -42,7 +42,7 @@ private static void checkIterator(boolean direct, int lgK, int n) {
TgtHllType tgtHllType = TgtHllType.HLL_4;
int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgK, tgtHllType);
WritableMemory wmem = WritableMemory.allocate(bytes);
- HllSketch sk = (direct) ? new HllSketch(lgK, tgtHllType, wmem) : new HllSketch(lgK);
+ HllSketch sk = (direct) ? new HllSketch(lgK, tgtHllType, wmem) : new HllSketch(lgK, tgtHllType);
for (int i = 0; i < n; i++) { sk.update(i); }
String store = direct ? "Memory" : "Heap";
println("CurMode: " + sk.getCurMode().toString() + "; Store: " + store);
diff --git a/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java b/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java
index 78b18c1cc..ecebe4539 100644
--- a/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java
+++ b/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java
@@ -26,17 +26,13 @@
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
-import java.nio.ByteOrder;
+import java.lang.foreign.Arena;
import java.util.HashMap;
import org.apache.datasketches.common.SketchesStateException;
-import org.apache.datasketches.memory.DefaultMemoryRequestServer;
-import org.testng.annotations.Test;
-
import org.apache.datasketches.memory.Memory;
-import org.apache.datasketches.memory.WritableHandle;
import org.apache.datasketches.memory.WritableMemory;
-
+import org.testng.annotations.Test;
/**
* @author Lee Rhodes
@@ -48,53 +44,50 @@ public void checkGrow() {
int lgConfigK = 4;
TgtHllType tgtHllType = TgtHllType.HLL_4;
int n = 8; //put lgConfigK == 4 into HLL mode
- int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType);
+ long bytes = HllSketch.getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType);
HllSketch hllSketch;
- try (WritableHandle handle = WritableMemory.allocateDirect(bytes,
- ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())) {
- WritableMemory wmem = handle.getWritable();
- hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem);
- for (int i = 0; i < n; i++) {
- hllSketch.update(i);
- }
- hllSketch.couponUpdate(HllUtil.pair(7, 15)); //mock extreme values
- hllSketch.couponUpdate(HllUtil.pair(8, 15));
- hllSketch.couponUpdate(HllUtil.pair(9, 15));
- //println(hllSketch.toString(true, true, true, true));
- DirectHllArray dha = (DirectHllArray) hllSketch.hllSketchImpl;
- assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 2);
- assertTrue(hllSketch.isMemory());
- assertTrue(hllSketch.isOffHeap());
- assertTrue(hllSketch.isSameResource(wmem));
-
- //Check heapify
- byte[] byteArray = hllSketch.toCompactByteArray();
- HllSketch hllSketch2 = HllSketch.heapify(byteArray);
- HllArray ha = (HllArray) hllSketch2.hllSketchImpl;
- assertEquals(ha.getAuxHashMap().getLgAuxArrInts(), 2);
- assertEquals(ha.getAuxHashMap().getAuxCount(), 3);
-
- //Check wrap
- byteArray = hllSketch.toUpdatableByteArray();
- WritableMemory wmem2 = WritableMemory.writableWrap(byteArray);
- hllSketch2 = HllSketch.writableWrap(wmem2);
- //println(hllSketch2.toString(true, true, true, true));
- DirectHllArray dha2 = (DirectHllArray) hllSketch2.hllSketchImpl;
- assertEquals(dha2.getAuxHashMap().getLgAuxArrInts(), 2);
- assertEquals(dha2.getAuxHashMap().getAuxCount(), 3);
-
- //Check grow to on-heap
- hllSketch.couponUpdate(HllUtil.pair(10, 15)); //puts it over the edge, must grow
- //println(hllSketch.toString(true, true, true, true));
- dha = (DirectHllArray) hllSketch.hllSketchImpl;
- assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 3);
- assertEquals(dha.getAuxHashMap().getAuxCount(), 4);
- assertTrue(hllSketch.isMemory());
- assertFalse(hllSketch.isOffHeap());
- assertFalse(hllSketch.isSameResource(wmem));
- } catch (final Exception e) {
- throw new RuntimeException(e);
+ WritableMemory wmem = WritableMemory.allocateDirect(bytes, Arena.ofConfined());
+
+ hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem);
+ for (int i = 0; i < n; i++) {
+ hllSketch.update(i);
}
+ hllSketch.couponUpdate(HllUtil.pair(7, 15)); //mock extreme values
+ hllSketch.couponUpdate(HllUtil.pair(8, 15));
+ hllSketch.couponUpdate(HllUtil.pair(9, 15));
+ println(hllSketch.toString(true, true, true, true));
+ DirectHllArray dha = (DirectHllArray) hllSketch.hllSketchImpl;
+ assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 2);
+ assertTrue(hllSketch.isMemory());
+ assertTrue(hllSketch.isOffHeap());
+ assertTrue(hllSketch.isSameResource(wmem));
+
+ //Check heapify
+ byte[] byteArray = hllSketch.toCompactByteArray();
+ HllSketch hllSketch2 = HllSketch.heapify(byteArray);
+ HllArray ha = (HllArray) hllSketch2.hllSketchImpl;
+ assertEquals(ha.getAuxHashMap().getLgAuxArrInts(), 2);
+ assertEquals(ha.getAuxHashMap().getAuxCount(), 3);
+
+ //Check wrap
+ byteArray = hllSketch.toUpdatableByteArray();
+ WritableMemory wmem2 = WritableMemory.writableWrap(byteArray);
+ hllSketch2 = HllSketch.writableWrap(wmem2);
+ println(hllSketch2.toString(true, true, true, true));
+ DirectHllArray dha2 = (DirectHllArray) hllSketch2.hllSketchImpl;
+ assertEquals(dha2.getAuxHashMap().getLgAuxArrInts(), 2);
+ assertEquals(dha2.getAuxHashMap().getAuxCount(), 3);
+
+ //Check grow to on-heap
+ hllSketch.couponUpdate(HllUtil.pair(10, 15)); //puts it over the edge, must grow
+ println(hllSketch.toString(true, true, true, true));
+ dha = (DirectHllArray) hllSketch.hllSketchImpl;
+ assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 3);
+ assertEquals(dha.getAuxHashMap().getAuxCount(), 4);
+ assertTrue(hllSketch.isMemory());
+ assertFalse(hllSketch.isOffHeap());
+ assertFalse(hllSketch.isSameResource(wmem));
+ assertFalse(wmem.isAlive());
}
@Test
diff --git a/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java b/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java
index 38cbc4977..6da2f05fe 100644
--- a/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java
+++ b/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java
@@ -25,11 +25,11 @@
import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
-import org.testng.annotations.Test;
+import java.lang.foreign.Arena;
import org.apache.datasketches.memory.Memory;
-import org.apache.datasketches.memory.WritableHandle;
import org.apache.datasketches.memory.WritableMemory;
+import org.testng.annotations.Test;
/**
* @author Lee Rhodes
@@ -69,11 +69,9 @@ private static void promotions(int lgConfigK, int n, TgtHllType tgtHllType, bool
//println("DIRECT");
byte[] barr1;
- WritableMemory wmem = null;
- try (WritableHandle hand = WritableMemory.allocateDirect(bytes)) {
- wmem = hand.getWritable();
- //byte[] byteArr = new byte[bytes];
- //WritableMemory wmem = WritableMemory.wrap(byteArr);
+ WritableMemory wmem;
+ try (Arena arena = Arena.ofConfined()) {
+ wmem = WritableMemory.allocateDirect(bytes, arena);
hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem);
assertTrue(hllSketch.isEmpty());
diff --git a/src/test/java/org/apache/datasketches/hll/PreambleUtilTest.java b/src/test/java/org/apache/datasketches/hll/PreambleUtilTest.java
index cd1e0cbcf..17f3d0d0f 100644
--- a/src/test/java/org/apache/datasketches/hll/PreambleUtilTest.java
+++ b/src/test/java/org/apache/datasketches/hll/PreambleUtilTest.java
@@ -109,7 +109,6 @@ public void checkCorruptMemoryInput() {
HllSketch sk = new HllSketch(12);
byte[] memObj = sk.toCompactByteArray();
WritableMemory wmem = WritableMemory.writableWrap(memObj);
- long memAdd = wmem.getCumulativeOffset(0);
HllSketch bad;
//checkFamily
@@ -148,7 +147,6 @@ public void checkCorruptMemoryInput() {
for (int i = 1; i <= 15; i++) { sk.update(i); }
memObj = sk.toCompactByteArray();
wmem = WritableMemory.writableWrap(memObj);
- memAdd = wmem.getCumulativeOffset(0);
//check wrong PreInts and SET
try {
@@ -162,7 +160,6 @@ public void checkCorruptMemoryInput() {
for (int i = 15; i <= 1000; i++) { sk.update(i); }
memObj = sk.toCompactByteArray();
wmem = WritableMemory.writableWrap(memObj);
- memAdd = wmem.getCumulativeOffset(0);
//check wrong PreInts and HLL
try {
@@ -179,7 +176,6 @@ public void checkExtractFlags() {
int bytes = HllSketch.getMaxUpdatableSerializationBytes(4, TgtHllType.HLL_4);
WritableMemory wmem = WritableMemory.allocate(bytes);
Object memObj = wmem.getArray();
- long memAdd = wmem.getCumulativeOffset(0L);
HllSketch sk = new HllSketch(4, TgtHllType.HLL_4, wmem);
int flags = extractFlags(wmem);
assertEquals(flags, EMPTY_FLAG_MASK);
diff --git a/src/test/java/org/apache/datasketches/hll/SizeAndModeTransitions.java b/src/test/java/org/apache/datasketches/hll/SizeAndModeTransitions.java
new file mode 100644
index 000000000..c9ffa1e28
--- /dev/null
+++ b/src/test/java/org/apache/datasketches/hll/SizeAndModeTransitions.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.hll;
+
+import static org.testng.Assert.assertEquals;
+
+import org.apache.datasketches.memory.WritableMemory;
+import org.testng.annotations.Test;
+
+public class SizeAndModeTransitions {
+ static String hfmt = "%6s %7s %4s %10s %5s %10s %10s %10s %10s %10s %14s\n";
+ static String dfmt = "%6s %7s %4d %,10d %5s %,10d %,10d %,10d %,10d %,10d %,14.3f\n";
+ static String[] hdr = {"Type", "Store", "LgK", "N", "Mode", "ActCBytes", "CmpBytes", "ActUBytes", "UpdBytes", "MaxBytes", "Estimate"};
+
+ @Test
+ public void checkHLL8Heap() {
+ TgtHllType tgtHllType = TgtHllType.HLL_8;
+ boolean direct = false;
+ int lgK = 10;
+ int N = 97;
+ printf(hfmt, (Object[]) hdr);
+ int maxBytes = HllSketch.getMaxUpdatableSerializationBytes(lgK, tgtHllType);
+ WritableMemory wmem = null;;
+ HllSketch sk;
+ if (direct) {
+ wmem = WritableMemory.allocate(maxBytes);
+ sk = new HllSketch(lgK, tgtHllType, wmem);
+ } else {
+ sk = new HllSketch(lgK, tgtHllType);
+ }
+ String store = direct ? "Memory" : "Heap";
+ for (int i = 1; i <= N; i++) {
+ sk.update(i);
+ if (i == 7) { checkAtN(sk, tgtHllType, store, lgK, i, "LIST", 36, 40, 1064); }
+ if (i == 8) { checkAtN(sk, tgtHllType, store, lgK, i, "SET", 44, 140, 1064); }
+ if (i == 24) { checkAtN(sk, tgtHllType, store, lgK, i, "SET", 108, 140, 1064); }
+ if (i == 25) { checkAtN(sk, tgtHllType, store, lgK, i, "SET", 112, 268, 1064); }
+ if (i == 48) { checkAtN(sk, tgtHllType, store, lgK, i, "SET", 204, 268, 1064); }
+ if (i == 49) { checkAtN(sk, tgtHllType, store, lgK, i, "SET", 208, 524, 1064); }
+ if (i == 96) { checkAtN(sk, tgtHllType, store, lgK, i, "SET", 396, 524, 1064); }
+ if (i == 97) { checkAtN(sk, tgtHllType, store, lgK, i, "HLL", 1064, 1064, 1064); }
+ }
+ println("");
+ }
+
+ private static void checkAtN(HllSketch sk, TgtHllType tgtHllType, String store, int lgK, int n, String trueMode,
+ int cmpTrueBytes, int updTrueBytes, int maxTrueBytes) {
+ int maxBytes = HllSketch.getMaxUpdatableSerializationBytes(lgK, tgtHllType);
+ String type = tgtHllType.toString();
+ int cmpBytes = sk.getCompactSerializationBytes();
+ int updBytes = sk.getUpdatableSerializationBytes();
+ byte[] actCBytes = sk.toCompactByteArray();
+ byte[] actUBytes = sk.toUpdatableByteArray();
+ String mode = sk.getCurMode().toString();
+ double est = sk.getEstimate();
+ printf(dfmt, type, store, lgK, n, mode, actCBytes.length, cmpBytes, actUBytes.length, updBytes, maxBytes, est);
+ assertEquals(mode, trueMode);
+ assertEquals(cmpBytes, actCBytes.length);
+ assertEquals(cmpBytes, cmpTrueBytes);
+ assertEquals(updBytes, actUBytes.length);
+ assertEquals(updBytes, updTrueBytes);
+ assertEquals(maxBytes, maxTrueBytes);
+ }
+
+ @Test
+ public void printlnTest() {
+ println("PRINTING: "+this.getClass().getName());
+ }
+
+ /**
+ * @param s value to print
+ */
+ static void println(String s) {
+ //System.out.println(s); //disable here
+ }
+
+ /**
+ * @param fmt format
+ * @param args arguments
+ */
+ static void printf(String fmt, Object...args) {
+ //System.out.printf(fmt, args); //disable here
+ }
+
+}
diff --git a/src/test/java/org/apache/datasketches/kll/KllCrossLanguageTest.java b/src/test/java/org/apache/datasketches/kll/KllCrossLanguageTest.java
index 078f3503b..53b422b7c 100644
--- a/src/test/java/org/apache/datasketches/kll/KllCrossLanguageTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllCrossLanguageTest.java
@@ -67,6 +67,16 @@ public void generateKllFloatsSketchBinaries() throws IOException {
}
}
+ @Test(groups = {GENERATE_JAVA_FILES})
+ public void generateKllLongsSketchBinaries() throws IOException {
+ final int[] nArr = {0, 1, 10, 100, 1_000, 10_000, 100_000, 1_000_000};
+ for (int n: nArr) {
+ final KllLongsSketch sk = KllLongsSketch.newHeapInstance();
+ for (int i = 1; i <= n; i++) { sk.update(i); }
+ Files.newOutputStream(javaPath.resolve("kll_long_n" + n + "_java.sk")).write(sk.toByteArray());
+ }
+ }
+
@Test(groups = {GENERATE_JAVA_FILES})
public void generateKllItemsSketchBinaries() throws IOException {
final int[] nArr = {0, 1, 10, 100, 1_000, 10_000, 100_000, 1_000_000};
diff --git a/src/test/java/org/apache/datasketches/kll/KllDirectCompactDoublesSketchTest.java b/src/test/java/org/apache/datasketches/kll/KllDirectCompactDoublesSketchTest.java
index 9831c2f57..7a4d061ad 100644
--- a/src/test/java/org/apache/datasketches/kll/KllDirectCompactDoublesSketchTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllDirectCompactDoublesSketchTest.java
@@ -110,13 +110,13 @@ public void checkDirectCompactGetDoubleItemsArray() {
KllDoublesSketch sk2 = KllDoublesSketch.wrap(Memory.wrap(sk.toByteArray()));
double[] itemsArr = sk2.getDoubleItemsArray();
- for (int i = 0; i < 20; i++) { assertEquals(itemsArr[i], 0F); }
+ for (int i = 0; i < 20; i++) { assertEquals(itemsArr[i], 0.0); }
sk.update(1);
sk2 = KllDoublesSketch.wrap(Memory.wrap(sk.toByteArray()));
itemsArr = sk2.getDoubleItemsArray();
- for (int i = 0; i < 19; i++) { assertEquals(itemsArr[i], 0F); }
- assertEquals(itemsArr[19], 1F);
+ for (int i = 0; i < 19; i++) { assertEquals(itemsArr[i], 0.0); }
+ assertEquals(itemsArr[19], 1.0);
for (int i = 2; i <= 21; i++) { sk.update(i); }
sk2 = KllDoublesSketch.wrap(Memory.wrap(sk.toByteArray()));
@@ -169,12 +169,12 @@ public void checkMinAndMax() {
try { sk2.getMaxItem(); fail(); } catch (SketchesArgumentException e) {}
sk.update(1);
sk2 = KllDoublesSketch.wrap(Memory.wrap(sk.toByteArray()));
- assertEquals(sk2.getMaxItem(),1.0F);
- assertEquals(sk2.getMinItem(),1.0F);
+ assertEquals(sk2.getMaxItem(),1.0);
+ assertEquals(sk2.getMinItem(),1.0);
for (int i = 2; i <= 21; i++) { sk.update(i); }
sk2 = KllDoublesSketch.wrap(Memory.wrap(sk.toByteArray()));
- assertEquals(sk2.getMaxItem(),21.0F);
- assertEquals(sk2.getMinItem(),1.0F);
+ assertEquals(sk2.getMaxItem(),21.0);
+ assertEquals(sk2.getMinItem(),1.0);
}
@Test
diff --git a/src/test/java/org/apache/datasketches/kll/KllDirectCompactLongsSketchTest.java b/src/test/java/org/apache/datasketches/kll/KllDirectCompactLongsSketchTest.java
new file mode 100644
index 000000000..6b57fccac
--- /dev/null
+++ b/src/test/java/org/apache/datasketches/kll/KllDirectCompactLongsSketchTest.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.kll;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+import org.apache.datasketches.common.SketchesArgumentException;
+import org.apache.datasketches.kll.KllDirectLongsSketch.KllDirectCompactLongsSketch;
+import org.apache.datasketches.memory.DefaultMemoryRequestServer;
+import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.memory.WritableMemory;
+import org.testng.annotations.Test;
+
+public class KllDirectCompactLongsSketchTest {
+ private static final DefaultMemoryRequestServer memReqSvr = new DefaultMemoryRequestServer();
+
+ @Test
+ public void checkRODirectUpdatable_ROandWritable() {
+ int k = 20;
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance(k);
+ for (int i = 1; i <= k + 1; i++) { sk.update(i); }
+ byte[] byteArr = KllHelper.toByteArray(sk, true); //request updatable
+ Memory srcMem = Memory.wrap(byteArr); //cast to Memory -> read only
+ KllLongsSketch sk2 = KllLongsSketch.wrap(srcMem);
+ assertTrue(sk2 instanceof KllDirectLongsSketch);
+
+ assertTrue(sk2.isMemoryUpdatableFormat());
+ assertTrue(sk2.isReadOnly());
+ assertEquals(sk2.getMinItem(), 1L);
+ assertEquals(sk2.getMaxItem(), 21L);
+
+ WritableMemory srcWmem = WritableMemory.writableWrap(byteArr);
+ KllLongsSketch sk3 = KllLongsSketch.writableWrap(srcWmem, memReqSvr);
+ assertTrue(sk3 instanceof KllDirectLongsSketch);
+ println(sk3.toString(true, false));
+ assertFalse(sk3.isReadOnly());
+ sk3.update(22);
+ assertEquals(sk2.getMinItem(), 1L);
+ assertEquals(sk2.getMaxItem(), 22L);
+ }
+
+ @Test
+ public void checkRODirectCompact() {
+ int k = 20;
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance(k);
+ for (int i = 1; i <= k + 1; i++) { sk.update(i); }
+ Memory srcMem = Memory.wrap(sk.toByteArray()); //compact RO fmt
+ KllLongsSketch sk2 = KllLongsSketch.wrap(srcMem);
+ assertTrue(sk2 instanceof KllDirectCompactLongsSketch);
+ //println(sk2.toString(true, false));
+ assertFalse(sk2.isMemoryUpdatableFormat());
+ assertTrue(sk2.isReadOnly());
+ assertEquals(sk2.getMinItem(), 1L);
+ assertEquals(sk2.getMaxItem(), 21L);
+ Memory srcMem2 = Memory.wrap(sk2.toByteArray());
+ KllLongsSketch sk3 = KllLongsSketch.writableWrap((WritableMemory)srcMem2, memReqSvr);
+ assertTrue(sk3 instanceof KllDirectCompactLongsSketch);
+ assertFalse(sk2.isMemoryUpdatableFormat());
+ //println(sk3.toString(true, false));
+ assertTrue(sk3.isReadOnly());
+ assertEquals(sk3.getMinItem(), 1L);
+ assertEquals(sk3.getMaxItem(), 21L);
+ }
+
+ @Test
+ public void checkDirectCompactSingleItem() {
+ int k = 20;
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance(k);
+
+ sk.update(1);
+ KllLongsSketch sk2 = KllLongsSketch.wrap(Memory.wrap(sk.toByteArray()));
+ assertTrue(sk2 instanceof KllDirectCompactLongsSketch);
+ //println(sk2.toString(true, false));
+ assertTrue(sk2.isReadOnly());
+ assertEquals(sk2.getLongSingleItem(), 1L);
+
+ sk.update(2);
+ sk2 = KllLongsSketch.wrap(Memory.wrap(sk.toByteArray()));
+ assertEquals(sk2.getN(), 2);
+ try {
+ sk2.getLongSingleItem();
+ fail();
+ } catch (SketchesArgumentException e) { }
+ }
+
+ @Test
+ public void checkDirectCompactGetLongItemsArray() {
+ int k = 20;
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance(k);
+
+ KllLongsSketch sk2 = KllLongsSketch.wrap(Memory.wrap(sk.toByteArray()));
+ long[] itemsArr = sk2.getLongItemsArray();
+ for (int i = 0; i < 20; i++) { assertEquals(itemsArr[i], 0); }
+
+ sk.update(1);
+ sk2 = KllLongsSketch.wrap(Memory.wrap(sk.toByteArray()));
+ itemsArr = sk2.getLongItemsArray();
+ for (int i = 0; i < 19; i++) { assertEquals(itemsArr[i], 0); }
+ assertEquals(itemsArr[19], 1L);
+
+ for (int i = 2; i <= 21; i++) { sk.update(i); }
+ sk2 = KllLongsSketch.wrap(Memory.wrap(sk.toByteArray()));
+ itemsArr = sk2.getLongItemsArray();
+ assertEquals(itemsArr.length, 33);
+ assertEquals(itemsArr[22], 21);
+ }
+
+ @Test
+ public void checkHeapAndDirectCompactGetRetainedItemsArray() {
+ int k = 20;
+
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance(k);
+ long[] retArr = sk.getLongRetainedItemsArray();
+ assertEquals(retArr.length, 0);
+
+ KllLongsSketch sk2 = KllLongsSketch.wrap(Memory.wrap(sk.toByteArray()));
+ retArr = sk2.getLongRetainedItemsArray();
+ assertEquals(retArr.length, sk.getNumRetained());
+ assertEquals(retArr.length, 0);
+
+ sk.update(1);
+ retArr = sk.getLongRetainedItemsArray();
+ assertEquals(retArr.length, sk.getNumRetained());
+ assertEquals(retArr.length, 1);
+ assertEquals(retArr[0], 1L);
+
+ sk2 = KllLongsSketch.wrap(Memory.wrap(sk.toByteArray()));
+ retArr = sk2.getLongRetainedItemsArray();
+ assertEquals(retArr.length, sk.getNumRetained());
+ assertEquals(retArr.length, 1);
+ assertEquals(retArr[0], 1L);
+
+ for (int i = 2; i <= 21; i++) { sk.update(i); }
+ retArr = sk.getLongRetainedItemsArray();
+ assertEquals(retArr.length, sk.getNumRetained());
+ assertEquals(retArr.length, 11);
+
+ sk2 = KllLongsSketch.wrap(Memory.wrap(sk.toByteArray()));
+ assertEquals(retArr.length, sk2.getNumRetained());
+ assertEquals(retArr.length, 11);
+ }
+
+ @Test
+ public void checkMinAndMax() {
+ int k = 20;
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance(k);
+ KllLongsSketch sk2 = KllLongsSketch.wrap(Memory.wrap(sk.toByteArray()));
+ try { sk2.getMinItem(); fail(); } catch (SketchesArgumentException e) {}
+ try { sk2.getMaxItem(); fail(); } catch (SketchesArgumentException e) {}
+ sk.update(1);
+ sk2 = KllLongsSketch.wrap(Memory.wrap(sk.toByteArray()));
+ assertEquals(sk2.getMaxItem(),1L);
+ assertEquals(sk2.getMinItem(),1L);
+ for (int i = 2; i <= 21; i++) { sk.update(i); }
+ sk2 = KllLongsSketch.wrap(Memory.wrap(sk.toByteArray()));
+ assertEquals(sk2.getMaxItem(),21L);
+ assertEquals(sk2.getMinItem(),1L);
+ }
+
+ @Test
+ public void checkQuantile() {
+ KllLongsSketch sk1 = KllLongsSketch.newHeapInstance();
+ for (int i = 1; i <= 1000; i++) { sk1.update(i); }
+ KllLongsSketch sk2 = KllLongsSketch.wrap(Memory.wrap(sk1.toByteArray()));
+ long med2 = sk2.getQuantile(0.5);
+ long med1 = sk1.getQuantile(0.5);
+ assertEquals(med1, med2);
+ println("Med1: " + med1);
+ println("Med2: " + med2);
+ }
+
+ @Test
+ public void checkCompactSingleItemMerge() {
+ int k = 20;
+ KllLongsSketch skH1 = KllLongsSketch.newHeapInstance(k); //Heap with 1 (single)
+ skH1.update(21);
+ KllLongsSketch skDC1 = KllLongsSketch.wrap(Memory.wrap(skH1.toByteArray())); //Direct Compact with 1 (single)
+ KllLongsSketch skH20 = KllLongsSketch.newHeapInstance(k); //Heap with 20
+ for (int i = 1; i <= 20; i++) { skH20.update(i); }
+ skH20.merge(skDC1);
+ assertEquals(skH20.getN(), 21);
+
+ WritableMemory wmem = WritableMemory.allocate(1000);
+ KllLongsSketch skDU20 = KllLongsSketch.newDirectInstance(k, wmem, memReqSvr);//Direct Updatable with 21
+ for (int i = 1; i <= 20; i++) { skDU20.update(i); }
+ skDU20.merge(skDC1);
+ assertEquals(skDU20.getN(), 21);
+ }
+
+ @Test
+ public void printlnTest() {
+ println("PRINTING: " + this.getClass().getName());
+ }
+
+ private final static boolean enablePrinting = false;
+
+ /**
+ * @param o the Object to println
+ */
+ private static final void println(final Object o) {
+ if (enablePrinting) { System.out.println(o.toString()); }
+ }
+
+}
diff --git a/src/test/java/org/apache/datasketches/kll/KllDirectDoublesSketchIteratorTest.java b/src/test/java/org/apache/datasketches/kll/KllDirectDoublesSketchIteratorTest.java
index 78a3b9cd5..4bfdfa4fc 100644
--- a/src/test/java/org/apache/datasketches/kll/KllDirectDoublesSketchIteratorTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllDirectDoublesSketchIteratorTest.java
@@ -41,7 +41,7 @@ public void oneItemSketch() {
sketch.update(0);
QuantilesDoublesSketchIterator it = sketch.iterator();
Assert.assertTrue(it.next());
- Assert.assertEquals(it.getQuantile(), 0f);
+ Assert.assertEquals(it.getQuantile(), 0);
Assert.assertEquals(it.getWeight(), 1);
Assert.assertFalse(it.next());
}
diff --git a/src/test/java/org/apache/datasketches/kll/KllDirectDoublesSketchTest.java b/src/test/java/org/apache/datasketches/kll/KllDirectDoublesSketchTest.java
index 33219a806..6342ac33d 100644
--- a/src/test/java/org/apache/datasketches/kll/KllDirectDoublesSketchTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllDirectDoublesSketchTest.java
@@ -189,11 +189,11 @@ public void mergeLowerK() {
sketch2.update(2 * n - i - 1);
}
- assertEquals(sketch1.getMinItem(), 0.0f);
- assertEquals(sketch1.getMaxItem(), n - 1f);
+ assertEquals(sketch1.getMinItem(), 0.0);
+ assertEquals(sketch1.getMaxItem(), n - 1.0);
assertEquals(sketch2.getMinItem(), n);
- assertEquals(sketch2.getMaxItem(), 2f * n - 1f);
+ assertEquals(sketch2.getMaxItem(), 2.0 * n - 1.0);
assertTrue(sketch1.getNormalizedRankError(false) < sketch2.getNormalizedRankError(false));
assertTrue(sketch1.getNormalizedRankError(true) < sketch2.getNormalizedRankError(true));
@@ -613,7 +613,7 @@ public void checkWritableWrapOfCompactForm() {
public void checkReadOnlyExceptions() {
int k = 20;
double[] dblArr = new double[0];
- double dblV = 1.0f;
+ double dblV = 1.0;
int idx = 1;
boolean bool = true;
KllDoublesSketch sk = KllDoublesSketch.newHeapInstance(k);
diff --git a/src/test/java/org/apache/datasketches/kll/KllDirectLongsSketchIteratorTest.java b/src/test/java/org/apache/datasketches/kll/KllDirectLongsSketchIteratorTest.java
new file mode 100644
index 000000000..8be509f10
--- /dev/null
+++ b/src/test/java/org/apache/datasketches/kll/KllDirectLongsSketchIteratorTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.kll;
+
+import org.apache.datasketches.memory.DefaultMemoryRequestServer;
+import org.apache.datasketches.memory.WritableMemory;
+import org.apache.datasketches.quantilescommon.QuantilesLongsSketchIterator;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class KllDirectLongsSketchIteratorTest {
+ private static final DefaultMemoryRequestServer memReqSvr = new DefaultMemoryRequestServer();
+
+ @Test
+ public void emptySketch() {
+ final KllLongsSketch sketch = getDLSketch(200, 0);
+ QuantilesLongsSketchIterator it = sketch.iterator();
+ Assert.assertFalse(it.next());
+ }
+
+ @Test
+ public void oneItemSketch() {
+ final KllLongsSketch sketch = getDLSketch(200, 0);
+ sketch.update(0);
+ QuantilesLongsSketchIterator it = sketch.iterator();
+ Assert.assertTrue(it.next());
+ Assert.assertEquals(it.getQuantile(), 0);
+ Assert.assertEquals(it.getWeight(), 1);
+ Assert.assertFalse(it.next());
+ }
+
+ @Test
+ public void bigSketches() {
+ for (int n = 1000; n < 100000; n += 2000) {
+ final KllLongsSketch sketch = getDLSketch(200, 0);
+ for (int i = 0; i < n; i++) {
+ sketch.update(i);
+ }
+ QuantilesLongsSketchIterator it = sketch.iterator();
+ int count = 0;
+ int weight = 0;
+ while (it.next()) {
+ count++;
+ weight += (int)it.getWeight();
+ }
+ Assert.assertEquals(count, sketch.getNumRetained());
+ Assert.assertEquals(weight, n);
+ }
+ }
+
+ private static KllLongsSketch getDLSketch(final int k, final int n) {
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance(k);
+ for (int i = 1; i <= n; i++) { sk.update(i); }
+ byte[] byteArr = KllHelper.toByteArray(sk, true);
+ WritableMemory wmem = WritableMemory.writableWrap(byteArr);
+
+ KllLongsSketch dlsk = KllLongsSketch.writableWrap(wmem, memReqSvr);
+ return dlsk;
+ }
+
+}
diff --git a/src/test/java/org/apache/datasketches/kll/KllDirectLongsSketchTest.java b/src/test/java/org/apache/datasketches/kll/KllDirectLongsSketchTest.java
new file mode 100644
index 000000000..f1784b7ce
--- /dev/null
+++ b/src/test/java/org/apache/datasketches/kll/KllDirectLongsSketchTest.java
@@ -0,0 +1,686 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.kll;
+
+import org.apache.datasketches.common.SketchesArgumentException;
+import org.apache.datasketches.kll.KllSketch.SketchStructure;
+import org.apache.datasketches.memory.DefaultMemoryRequestServer;
+import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.memory.WritableMemory;
+import org.testng.annotations.Test;
+
+import static org.apache.datasketches.kll.KllSketch.SketchType.LONGS_SKETCH;
+import static org.apache.datasketches.quantilescommon.QuantileSearchCriteria.EXCLUSIVE;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+public class KllDirectLongsSketchTest {
+
+ private static final double PMF_EPS_FOR_K_8 = 0.35; // PMF rank error (epsilon) for k=8
+ private static final double PMF_EPS_FOR_K_128 = 0.025; // PMF rank error (epsilon) for k=128
+ private static final double PMF_EPS_FOR_K_256 = 0.013; // PMF rank error (epsilon) for k=256
+ private static final double NUMERIC_NOISE_TOLERANCE = 1E-6;
+ private static final DefaultMemoryRequestServer memReqSvr = new DefaultMemoryRequestServer();
+
+ @Test
+ public void empty() {
+ final KllLongsSketch sketch = getUpdatableDirectLongSketch(200, 0);
+ assertTrue(sketch.isEmpty());
+ assertEquals(sketch.getN(), 0);
+ assertEquals(sketch.getNumRetained(), 0);
+ try { sketch.getRank(0); fail(); } catch (SketchesArgumentException e) {}
+ try { sketch.getMinItem(); fail(); } catch (SketchesArgumentException e) {}
+ try { sketch.getMaxItem(); fail(); } catch (SketchesArgumentException e) {}
+ try { sketch.getQuantile(0.5); fail(); } catch (SketchesArgumentException e) {}
+ try { sketch.getQuantiles(new double[] {0.0, 1.0}); fail(); } catch (SketchesArgumentException e) {}
+ try { sketch.getPMF(new long[] {0}); fail(); } catch (SketchesArgumentException e) {}
+ try { sketch.getCDF(new long[0]); fail(); } catch (SketchesArgumentException e) {}
+ assertNotNull(sketch.toString(true, true));
+ assertNotNull(sketch.toString());
+ }
+
+ @Test(expectedExceptions = SketchesArgumentException.class)
+ public void getQuantileInvalidArg() {
+ final KllLongsSketch sketch = getUpdatableDirectLongSketch(200, 0);
+ sketch.update(1);
+ sketch.getQuantile(-1.0);
+ }
+
+ @Test(expectedExceptions = SketchesArgumentException.class)
+ public void getQuantilesInvalidArg() {
+ final KllLongsSketch sketch = getUpdatableDirectLongSketch(200, 0);
+ sketch.update(1);
+ sketch.getQuantiles(new double[] {2.0});
+ }
+
+ @Test
+ public void oneValue() {
+ final KllLongsSketch sketch = getUpdatableDirectLongSketch(200, 0);
+ sketch.update(1);
+ assertFalse(sketch.isEmpty());
+ assertEquals(sketch.getN(), 1);
+ assertEquals(sketch.getNumRetained(), 1);
+ assertEquals(sketch.getRank(1, EXCLUSIVE), 0.0);
+ assertEquals(sketch.getRank(2, EXCLUSIVE), 1.0);
+ assertEquals(sketch.getMinItem(), 1L);
+ assertEquals(sketch.getMaxItem(), 1L);
+ assertEquals(sketch.getQuantile(0.5, EXCLUSIVE), 1L);
+ }
+
+ @Test
+ public void manyValuesEstimationMode() {
+ final KllLongsSketch sketch = getUpdatableDirectLongSketch(200, 0);
+ final int n = 1_000_000;
+
+ for (int i = 0; i < n; i++) {
+ sketch.update(i);
+ }
+ assertEquals(sketch.getN(), n);
+
+ // test getRank
+ for (int i = 0; i < n; i++) {
+ final double trueRank = (double) i / n;
+ assertEquals(sketch.getRank(i), trueRank, PMF_EPS_FOR_K_256, "for value " + i);
+ }
+
+ // test getPMF
+ final double[] pmf = sketch.getPMF(new long[] {n / 2}); // split at median
+ assertEquals(pmf.length, 2);
+ assertEquals(pmf[0], 0.5, PMF_EPS_FOR_K_256);
+ assertEquals(pmf[1], 0.5, PMF_EPS_FOR_K_256);
+ assertEquals(sketch.getMinItem(), 0); // min value is exact
+ assertEquals(sketch.getMaxItem(), n - 1L); // max value is exact
+
+ // check at every 0.1 percentage point
+ final double[] ranks = new double[1001];
+ final double[] reverseRanks = new double[1001]; // check that ordering doesn't matter
+ for (int i = 0; i <= 1000; i++) {
+ ranks[i] = (double) i / 1000;
+ reverseRanks[1000 - i] = ranks[i];
+ }
+ final long[] quantiles = sketch.getQuantiles(ranks);
+ final long[] reverseQuantiles = sketch.getQuantiles(reverseRanks);
+ long previousQuantile = 0;
+ for (int i = 0; i <= 1000; i++) {
+ final long quantile = sketch.getQuantile(ranks[i]);
+ assertEquals(quantile, quantiles[i]);
+ assertEquals(quantile, reverseQuantiles[1000 - i]);
+ assertTrue(previousQuantile <= quantile);
+ previousQuantile = quantile;
+ }
+ }
+
+ @Test
+ public void getRankGetCdfGetPmfConsistency() {
+ final KllLongsSketch sketch = getUpdatableDirectLongSketch(200, 0);
+ final int n = 1000;
+ final long[] values = new long[n];
+ for (int i = 0; i < n; i++) {
+ sketch.update(i);
+ values[i] = i;
+ }
+ final double[] ranks = sketch.getCDF(values);
+ final double[] pmf = sketch.getPMF(values);
+ double sumPmf = 0;
+ for (int i = 0; i < n; i++) {
+ assertEquals(ranks[i], sketch.getRank(values[i]), NUMERIC_NOISE_TOLERANCE,
+ "rank vs CDF for value " + i);
+ sumPmf += pmf[i];
+ assertEquals(ranks[i], sumPmf, NUMERIC_NOISE_TOLERANCE, "CDF vs PMF for value " + i);
+ }
+ sumPmf += pmf[n];
+ assertEquals(sumPmf, 1.0, NUMERIC_NOISE_TOLERANCE);
+ assertEquals(ranks[n], 1.0, NUMERIC_NOISE_TOLERANCE);
+ }
+
+ @Test
+ public void merge() {
+ final KllLongsSketch sketch1 = getUpdatableDirectLongSketch(200, 0);
+ final KllLongsSketch sketch2 = getUpdatableDirectLongSketch(200, 0);
+ final int n = 10_000;
+ for (int i = 0; i < n; i++) {
+ sketch1.update(i);
+ sketch2.update(2 * n - i - 1);
+ }
+
+ assertEquals(sketch1.getMinItem(), 0);
+ assertEquals(sketch1.getMaxItem(), n - 1);
+
+ assertEquals(sketch2.getMinItem(), n);
+ assertEquals(sketch2.getMaxItem(), 2 * n - 1);
+
+ sketch1.merge(sketch2);
+
+ assertFalse(sketch1.isEmpty());
+ assertEquals(sketch1.getN(), 2L * n);
+ assertEquals(sketch1.getMinItem(), 0);
+ assertEquals(sketch1.getMaxItem(), 2 * n - 1L);
+ assertEquals(sketch1.getQuantile(0.5), n, n * PMF_EPS_FOR_K_256);
+ }
+
+ @Test
+ public void mergeLowerK() {
+ final KllLongsSketch sketch1 = getUpdatableDirectLongSketch(256, 0);
+ final KllLongsSketch sketch2 = getUpdatableDirectLongSketch(128, 0);
+ final int n = 10_000;
+ for (int i = 0; i < n; i++) {
+ sketch1.update(i);
+ sketch2.update(2 * n - i - 1);
+ }
+
+ assertEquals(sketch1.getMinItem(), 0);
+ assertEquals(sketch1.getMaxItem(), n - 1);
+
+ assertEquals(sketch2.getMinItem(), n);
+ assertEquals(sketch2.getMaxItem(), 2 * n - 1);
+
+ assertTrue(sketch1.getNormalizedRankError(false) < sketch2.getNormalizedRankError(false));
+ assertTrue(sketch1.getNormalizedRankError(true) < sketch2.getNormalizedRankError(true));
+ sketch1.merge(sketch2);
+
+ // sketch1 must get "contaminated" by the lower K in sketch2
+ assertEquals(sketch1.getNormalizedRankError(false), sketch2.getNormalizedRankError(false));
+ assertEquals(sketch1.getNormalizedRankError(true), sketch2.getNormalizedRankError(true));
+
+ assertFalse(sketch1.isEmpty());
+ assertEquals(sketch1.getN(), 2 * n);
+ assertEquals(sketch1.getMinItem(), 0);
+ assertEquals(sketch1.getMaxItem(), 2 * n - 1);
+ assertEquals(sketch1.getQuantile(0.5), n, n * PMF_EPS_FOR_K_128);
+ }
+
+ @Test
+ public void mergeEmptyLowerK() {
+ final KllLongsSketch sketch1 = getUpdatableDirectLongSketch(256, 0);
+ final KllLongsSketch sketch2 = getUpdatableDirectLongSketch(128, 0);
+ final int n = 10_000;
+ for (int i = 0; i < n; i++) {
+ sketch1.update(i);
+ }
+
+ // rank error should not be affected by a merge with an empty sketch with lower K
+ final double rankErrorBeforeMerge = sketch1.getNormalizedRankError(true);
+ sketch1.merge(sketch2);
+ assertEquals(sketch1.getNormalizedRankError(true), rankErrorBeforeMerge);
+
+ assertFalse(sketch1.isEmpty());
+ assertEquals(sketch1.getN(), n);
+ assertEquals(sketch1.getMinItem(), 0);
+ assertEquals(sketch1.getMaxItem(), n - 1);
+ assertEquals(sketch1.getQuantile(0.5), n / 2, n / 2 * PMF_EPS_FOR_K_256);
+
+ //merge the other way
+ sketch2.merge(sketch1);
+ assertFalse(sketch1.isEmpty());
+ assertEquals(sketch1.getN(), n);
+ assertEquals(sketch1.getMinItem(), 0);
+ assertEquals(sketch1.getMaxItem(), n - 1);
+ assertEquals(sketch1.getQuantile(0.5), n / 2, n / 2 * PMF_EPS_FOR_K_256);
+ }
+
+ @Test
+ public void mergeExactModeLowerK() {
+ final KllLongsSketch sketch1 = getUpdatableDirectLongSketch(256, 0);
+ final KllLongsSketch sketch2 = getUpdatableDirectLongSketch(128, 0);
+ final int n = 10000;
+ for (int i = 0; i < n; i++) {
+ sketch1.update(i);
+ }
+ sketch2.update(1);
+
+ // rank error should not be affected by a merge with a sketch in exact mode with lower K
+ final double rankErrorBeforeMerge = sketch1.getNormalizedRankError(true);
+ sketch1.merge(sketch2);
+ assertEquals(sketch1.getNormalizedRankError(true), rankErrorBeforeMerge);
+ }
+
+ @Test
+ public void mergeMinMinValueFromOther() {
+ final KllLongsSketch sketch1 = getUpdatableDirectLongSketch(200, 0);
+ final KllLongsSketch sketch2 = getUpdatableDirectLongSketch(200, 0);
+ sketch1.update(1);
+ sketch2.update(2);
+ sketch2.merge(sketch1);
+ assertEquals(sketch2.getMinItem(), 1L);
+ }
+
+ @Test
+ public void mergeMinAndMaxFromOther() {
+ final KllLongsSketch sketch1 = getUpdatableDirectLongSketch(200, 0);
+ final KllLongsSketch sketch2 = getUpdatableDirectLongSketch(200, 0);
+ int n = 1_000_000;
+ for (int i = 1; i <= n; i++) {
+ sketch1.update(i);
+ }
+ sketch2.merge(sketch1);
+ assertEquals(sketch2.getMinItem(), 1L);
+ assertEquals(sketch2.getMaxItem(), 1_000_000L);
+ }
+
+ @Test(expectedExceptions = SketchesArgumentException.class)
+ public void kTooSmall() {
+ getUpdatableDirectLongSketch(KllSketch.DEFAULT_M - 1, 0);
+ }
+
+ @Test(expectedExceptions = SketchesArgumentException.class)
+ public void kTooLarge() {
+ getUpdatableDirectLongSketch(KllSketch.MAX_K + 1, 0);
+ }
+
+ @Test
+ public void minK() {
+ final KllLongsSketch sketch = getUpdatableDirectLongSketch(KllSketch.DEFAULT_M, 0);
+ for (int i = 0; i < 1000; i++) {
+ sketch.update(i);
+ }
+ assertEquals(sketch.getK(), KllSketch.DEFAULT_M);
+ assertEquals(sketch.getQuantile(0.5), 500, 500 * PMF_EPS_FOR_K_8);
+ }
+
+ @Test
+ public void maxK() {
+ final KllLongsSketch sketch = getUpdatableDirectLongSketch(KllSketch.MAX_K, 0);
+ for (int i = 0; i < 1000; i++) {
+ sketch.update(i);
+ }
+ assertEquals(sketch.getK(), KllSketch.MAX_K);
+ assertEquals(sketch.getQuantile(0.5), 500, 500 * PMF_EPS_FOR_K_256);
+ }
+
+ @Test
+ public void serializeDeserializeEmptyViaCompactHeapify() {
+ final KllLongsSketch sketch1 = getUpdatableDirectLongSketch(200, 0);
+ final byte[] bytes = sketch1.toByteArray(); //compact
+ final KllLongsSketch sketch2 = KllLongsSketch.heapify(Memory.wrap(bytes));
+ assertEquals(bytes.length, sketch1.currentSerializedSizeBytes(false));
+ assertTrue(sketch2.isEmpty());
+ assertEquals(sketch2.getNumRetained(), sketch1.getNumRetained());
+ assertEquals(sketch2.getN(), sketch1.getN());
+ assertEquals(sketch2.getNormalizedRankError(false), sketch1.getNormalizedRankError(false));
+ try { sketch2.getMinItem(); fail(); } catch (SketchesArgumentException e) {}
+ try { sketch2.getMaxItem(); fail(); } catch (SketchesArgumentException e) {}
+ assertEquals(sketch2.currentSerializedSizeBytes(false),
+ sketch1.currentSerializedSizeBytes(false));
+ }
+
+ @Test
+ public void serializeDeserializeEmptyViaUpdatableWritableWrap() {
+ final KllLongsSketch sketch1 = getUpdatableDirectLongSketch(200, 0);
+ final byte[] bytes = KllHelper.toByteArray(sketch1, true);
+ final KllLongsSketch sketch2 =
+ KllLongsSketch.writableWrap(WritableMemory.writableWrap(bytes),memReqSvr);
+ assertEquals(bytes.length, sketch1.currentSerializedSizeBytes(true));
+ assertTrue(sketch2.isEmpty());
+ assertEquals(sketch2.getNumRetained(), sketch1.getNumRetained());
+ assertEquals(sketch2.getN(), sketch1.getN());
+ assertEquals(sketch2.getNormalizedRankError(false), sketch1.getNormalizedRankError(false));
+ try { sketch2.getMinItem(); fail(); } catch (SketchesArgumentException e) {}
+ try { sketch2.getMaxItem(); fail(); } catch (SketchesArgumentException e) {}
+ assertEquals(sketch2.currentSerializedSizeBytes(true),
+ sketch1.currentSerializedSizeBytes(true));
+ }
+
+ @Test
+ public void serializeDeserializeOneValueViaCompactHeapify() {
+ final KllLongsSketch sketch1 = getUpdatableDirectLongSketch(200, 0);
+ sketch1.update(1);
+ final byte[] bytes = sketch1.toByteArray();
+ final KllLongsSketch sketch2 = KllLongsSketch.heapify(Memory.wrap(bytes));
+ assertEquals(bytes.length, sketch1.currentSerializedSizeBytes(false));
+ assertFalse(sketch2.isEmpty());
+ assertEquals(sketch2.getNumRetained(), 1);
+ assertEquals(sketch2.getN(), 1);
+ assertEquals(sketch2.getNormalizedRankError(false), sketch1.getNormalizedRankError(false));
+ assertTrue(sketch2.getMinItem() < Long.MAX_VALUE);
+ assertTrue(sketch2.getMaxItem() > Long.MIN_VALUE);
+ assertEquals(sketch2.currentSerializedSizeBytes(false), 8 + Long.BYTES);
+ }
+
+ @Test
+ public void serializeDeserializeOneValueViaUpdatableWritableWrap() {
+ final KllLongsSketch sketch1 = getUpdatableDirectLongSketch(200, 0);
+ sketch1.update(1);
+ final byte[] bytes = KllHelper.toByteArray(sketch1, true);
+ final KllLongsSketch sketch2 =
+ KllLongsSketch.writableWrap(WritableMemory.writableWrap(bytes),memReqSvr);
+ assertEquals(bytes.length, sketch1.currentSerializedSizeBytes(true));
+ assertFalse(sketch2.isEmpty());
+ assertEquals(sketch2.getNumRetained(), 1);
+ assertEquals(sketch2.getN(), 1);
+ assertEquals(sketch2.getNormalizedRankError(false), sketch1.getNormalizedRankError(false));
+ assertEquals(sketch2.getMinItem(), 1L);
+ assertEquals(sketch2.getMaxItem(), 1L);
+ assertEquals(sketch2.currentSerializedSizeBytes(false), 8 + Long.BYTES);
+ assertEquals(sketch2.currentSerializedSizeBytes(true), bytes.length);
+ }
+
+ @Test
+ public void serializeDeserializeFullViaCompactHeapify() {
+ final KllLongsSketch sketch1 = getUpdatableDirectLongSketch(200, 1000);
+ final byte[] byteArr1 = sketch1.toByteArray(); //compact
+ final KllLongsSketch sketch2 = KllLongsSketch.heapify(Memory.wrap(byteArr1));
+ assertEquals(byteArr1.length, sketch1.currentSerializedSizeBytes(false));
+ assertFalse(sketch2.isEmpty());
+ assertEquals(sketch2.getNumRetained(), sketch1.getNumRetained());
+ assertEquals(sketch2.getN(), sketch1.getN());
+ assertEquals(sketch2.getNormalizedRankError(false), sketch1.getNormalizedRankError(false));
+ assertEquals(sketch2.getMinItem(), sketch1.getMinItem());
+ assertEquals(sketch2.getMaxItem(), sketch1.getMaxItem());
+ assertEquals(sketch2.currentSerializedSizeBytes(false), sketch1.currentSerializedSizeBytes(false));
+ }
+
+ @Test
+ public void serializeDeserializeFullViaUpdatableWritableWrap() {
+ final KllLongsSketch sketch1 = getUpdatableDirectLongSketch(200, 0);
+ final int n = 1000;
+ for (int i = 1; i <= n; i++) {
+ sketch1.update(i);
+ }
+ final byte[] bytes = KllHelper.toByteArray(sketch1, true); //updatable
+ final KllLongsSketch sketch2 =
+ KllLongsSketch.writableWrap(WritableMemory.writableWrap(bytes), memReqSvr);
+ assertEquals(bytes.length, sketch1.currentSerializedSizeBytes(true));
+ assertFalse(sketch2.isEmpty());
+ assertEquals(sketch2.getNumRetained(), sketch1.getNumRetained());
+ assertEquals(sketch2.getN(), sketch1.getN());
+ assertEquals(sketch2.getNormalizedRankError(false), sketch1.getNormalizedRankError(false));
+ assertEquals(sketch2.getMinItem(), sketch1.getMinItem());
+ assertEquals(sketch2.getMaxItem(), sketch1.getMaxItem());
+ assertEquals(sketch2.currentSerializedSizeBytes(true), sketch1.currentSerializedSizeBytes(true));
+ }
+
+ @Test(expectedExceptions = SketchesArgumentException.class)
+ public void outOfOrderSplitPoints() {
+ final KllLongsSketch sketch = getUpdatableDirectLongSketch(200, 0);
+ sketch.update(0);
+ sketch.getCDF(new long[] {1, 0});
+ }
+
+ @Test
+ public void checkSimpleMergeDirect() { //used for troubleshooting
+ int k = 20;
+ int n1 = 21;
+ int n2 = 43;
+ KllLongsSketch sk1 = KllLongsSketch.newHeapInstance(k);
+ KllLongsSketch sk2 = KllLongsSketch.newHeapInstance(k);
+ for (int i = 1; i <= n1; i++) {
+ sk1.update(i);
+ }
+ for (int i = 1; i <= n2; i++) {
+ sk2.update(i + 100);
+ }
+ println("SK1:");
+ println(sk1.toString(true, true));
+ println("SK2:");
+ println(sk2.toString(true, true));
+ WritableMemory wmem1 = WritableMemory.writableWrap(KllHelper.toByteArray(sk1, true));
+ WritableMemory wmem2 = WritableMemory.writableWrap(KllHelper.toByteArray(sk2, true));
+ KllLongsSketch dsk1 = KllLongsSketch.writableWrap(wmem1, memReqSvr);
+ KllLongsSketch dsk2 = KllLongsSketch.writableWrap(wmem2, memReqSvr);
+ println("BEFORE MERGE");
+ println(dsk1.toString(true, true));
+ dsk1.merge(dsk2);
+ println("AFTER MERGE");
+ println(dsk1.toString(true, true));
+ }
+
+ @Test
+ public void checkSketchInitializeDirectLongUpdatableMem() {
+ int k = 20; //don't change this
+ KllLongsSketch sk;
+ KllLongsSketch sk2;
+ byte[] compBytes;
+ WritableMemory wmem;
+
+ println("#### CASE: LONG FULL DIRECT FROM UPDATABLE");
+ sk2 = KllLongsSketch.newHeapInstance(k);
+ for (int i = 1; i <= k + 1; i++) { sk2.update(i); }
+ //println(sk2.toString(true, true));
+ compBytes = KllHelper.toByteArray(sk2, true);
+ wmem = WritableMemory.writableWrap(compBytes);
+ println(KllPreambleUtil.toString(compBytes, LONGS_SKETCH, true));
+ sk = KllLongsSketch.writableWrap(wmem, memReqSvr);
+ assertEquals(sk.getK(), k);
+ assertEquals(sk.getN(), k + 1);
+ assertEquals(sk.getNumRetained(), 11);
+ assertFalse(sk.isEmpty());
+ assertTrue(sk.isEstimationMode());
+ assertEquals(sk.getMinK(), k);
+ assertEquals(sk.getLongItemsArray().length, 33);
+ assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 3);
+ assertEquals(sk.getMaxItem(), 21);
+ assertEquals(sk.getMinItem(), 1);
+ assertEquals(sk.getNumLevels(), 2);
+ assertFalse(sk.isLevelZeroSorted());
+
+ println("#### CASE: LONG EMPTY HEAPIFIED FROM UPDATABLE");
+ sk2 = KllLongsSketch.newHeapInstance(k);
+ //println(sk.toString(true, true));
+ compBytes = KllHelper.toByteArray(sk2, true);
+ wmem = WritableMemory.writableWrap(compBytes);
+ println(KllPreambleUtil.toString(compBytes, LONGS_SKETCH, true));
+ sk = KllLongsSketch.writableWrap(wmem, memReqSvr);
+ assertEquals(sk.getK(), k);
+ assertEquals(sk.getN(), 0);
+ assertEquals(sk.getNumRetained(), 0);
+ assertTrue(sk.isEmpty());
+ assertFalse(sk.isEstimationMode());
+ assertEquals(sk.getMinK(), k);
+ assertEquals(sk.getLongItemsArray().length, 20);
+ assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 2);
+ try { sk.getMaxItem(); fail(); } catch (SketchesArgumentException e) {}
+ try { sk.getMinItem(); fail(); } catch (SketchesArgumentException e) {}
+ assertEquals(sk.getNumLevels(), 1);
+ assertFalse(sk.isLevelZeroSorted());
+
+ println("#### CASE: LONG SINGLE HEAPIFIED FROM UPDATABLE");
+ sk2 = KllLongsSketch.newHeapInstance(k);
+ sk2.update(1);
+ //println(sk.toString(true, true));
+ compBytes = KllHelper.toByteArray(sk2, true);
+ wmem = WritableMemory.writableWrap(compBytes);
+ println(KllPreambleUtil.toString(compBytes, LONGS_SKETCH, true));
+ sk = KllLongsSketch.writableWrap(wmem, memReqSvr);
+ assertEquals(sk.getK(), k);
+ assertEquals(sk.getN(), 1);
+ assertEquals(sk.getNumRetained(), 1);
+ assertFalse(sk.isEmpty());
+ assertFalse(sk.isEstimationMode());
+ assertEquals(sk.getMinK(), k);
+ assertEquals(sk.getLongItemsArray().length, 20);
+ assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 2);
+ assertEquals(sk.getMaxItem(), 1L);
+ assertEquals(sk.getMinItem(), 1L);
+ assertEquals(sk.getNumLevels(), 1);
+ assertFalse(sk.isLevelZeroSorted());
+ }
+
+ @Test
+ public void checkGetWritableMemory() {
+ final KllLongsSketch sketch = getUpdatableDirectLongSketch(200, 200);
+ assertEquals(sketch.getK(), 200);
+ assertEquals(sketch.getN(), 200);
+ assertFalse(sketch.isEmpty());
+ assertTrue(sketch.isMemoryUpdatableFormat());
+ assertFalse(sketch.isEstimationMode());
+ assertTrue(sketch.isLongsSketch());
+ assertFalse(sketch.isLevelZeroSorted());
+ assertFalse(sketch.isDoublesSketch());
+
+ final WritableMemory wmem = sketch.getWritableMemory();
+ final KllLongsSketch sk = KllHeapLongsSketch.heapifyImpl(wmem);
+ assertEquals(sk.getK(), 200);
+ assertEquals(sk.getN(), 200);
+ assertFalse(sk.isEmpty());
+ assertFalse(sk.isMemoryUpdatableFormat());
+ assertFalse(sk.isEstimationMode());
+ assertTrue(sk.isLongsSketch());
+ assertFalse(sk.isLevelZeroSorted());
+ assertFalse(sk.isDoublesSketch());
+ }
+
+ @Test
+ public void checkReset() {
+ WritableMemory dstMem = WritableMemory.allocate(3000);
+ KllLongsSketch sk = KllLongsSketch.newDirectInstance(20, dstMem, memReqSvr);
+ for (int i = 1; i <= 100; i++) { sk.update(i); }
+ long n1 = sk.getN();
+ long min1 = sk.getMinItem();
+ long max1 = sk.getMaxItem();
+ sk.reset();
+ for (int i = 1; i <= 100; i++) { sk.update(i); }
+ long n2 = sk.getN();
+ long min2 = sk.getMinItem();
+ long max2 = sk.getMaxItem();
+ assertEquals(n2, n1);
+ assertEquals(min2, min1);
+ assertEquals(max2, max1);
+ }
+
+ @Test
+ public void checkHeapify() {
+ WritableMemory dstMem = WritableMemory.allocate(6000);
+ KllLongsSketch sk = KllLongsSketch.newDirectInstance(20, dstMem, memReqSvr);
+ for (int i = 1; i <= 100; i++) { sk.update(i); }
+ KllLongsSketch sk2 = KllHeapLongsSketch.heapifyImpl(dstMem);
+ assertEquals(sk2.getMinItem(), 1L);
+ assertEquals(sk2.getMaxItem(), 100L);
+ }
+
+ @Test
+ public void checkMergeKllLongsSketch() {
+ WritableMemory dstMem = WritableMemory.allocate(6000);
+ KllLongsSketch sk = KllLongsSketch.newDirectInstance(20, dstMem, memReqSvr);
+ for (int i = 1; i <= 21; i++) { sk.update(i); }
+ KllLongsSketch sk2 = KllLongsSketch.newHeapInstance(20);
+ for (int i = 1; i <= 21; i++ ) { sk2.update(i + 100); }
+ sk.merge(sk2);
+ assertEquals(sk.getMinItem(), 1L);
+ assertEquals(sk.getMaxItem(), 121L);
+ }
+
+ @Test
+ public void checkReverseMergeKllLongsSketch() {
+ WritableMemory dstMem = WritableMemory.allocate(6000);
+ KllLongsSketch sk = KllLongsSketch.newDirectInstance(20, dstMem, memReqSvr);
+ for (int i = 1; i <= 21; i++) { sk.update(i); }
+ KllLongsSketch sk2 = KllLongsSketch.newHeapInstance(20);
+ for (int i = 1; i <= 21; i++ ) { sk2.update(i + 100); }
+ sk2.merge(sk);
+ assertEquals(sk2.getMinItem(), 1L);
+ assertEquals(sk2.getMaxItem(), 121L);
+ }
+
+ @Test
+ public void checkWritableWrapOfCompactForm() {
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance(20);
+ for (int i = 1; i <= 21; i++ ) { sk.update(i); }
+ WritableMemory srcMem = WritableMemory.writableWrap(sk.toByteArray());
+ KllLongsSketch sk2 = KllLongsSketch.writableWrap(srcMem, memReqSvr);
+ assertEquals(sk2.getMinItem(), 1L);
+ assertEquals(sk2.getMaxItem(), 21L);
+ }
+
+ @Test
+ public void checkReadOnlyExceptions() {
+ int k = 20;
+ long[] fltArr = new long[0];
+ long fltV = 1;
+ int idx = 1;
+ boolean bool = true;
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance(k);
+ KllLongsSketch sk2 = KllLongsSketch.wrap(Memory.wrap(sk.toByteArray()));
+ try { sk2.incN(1); fail(); } catch (SketchesArgumentException e) { }
+ try { sk2.incNumLevels(); fail(); } catch (SketchesArgumentException e) { }
+ try { sk2.setLongItemsArray(fltArr); fail(); } catch (SketchesArgumentException e) { }
+ try { sk2.setLongItemsArrayAt(idx, fltV); fail(); } catch (SketchesArgumentException e) { }
+ try { sk2.setLevelZeroSorted(bool); fail(); } catch (SketchesArgumentException e) { }
+ try { sk2.setMaxItem(fltV); fail(); } catch (SketchesArgumentException e) { }
+ try { sk2.setMinItem(fltV); fail(); } catch (SketchesArgumentException e) { }
+ try { sk2.setMinK(idx); fail(); } catch (SketchesArgumentException e) { }
+ try { sk2.setN(idx); fail(); } catch (SketchesArgumentException e) { }
+ try { sk2.setNumLevels(idx); fail(); } catch (SketchesArgumentException e) { }
+ }
+
+ @Test(expectedExceptions = SketchesArgumentException.class)
+ public void checkMergeExceptions() {
+ KllLongsSketch sk1 = KllLongsSketch.newHeapInstance(20);
+ WritableMemory srcMem1 = WritableMemory.writableWrap(sk1.toByteArray());
+ KllLongsSketch sk2 = KllLongsSketch.writableWrap(srcMem1, memReqSvr);
+ sk2.merge(sk1);
+ }
+
+ @Test
+ public void checkVectorUpdate() {
+ WritableMemory dstMem = WritableMemory.allocate(6000);
+ KllLongsSketch sk = KllLongsSketch.newDirectInstance(20, dstMem, memReqSvr);
+ long[] v = new long[21];
+ for (int i = 0; i < 21; i++) { v[i] = i + 1; }
+ sk.update(v, 0, 21);
+ println(sk.toString(true, true));
+ int[] levelsArr = sk.getLevelsArray(SketchStructure.UPDATABLE);
+ assertEquals(levelsArr[0], 22);
+ long[] longsArr = sk.getLongItemsArray();
+ assertEquals(longsArr[22], 21);
+ }
+
+ @Test
+ public void checkWeightedUpdate() {
+ WritableMemory dstMem = WritableMemory.allocate(6000);
+ KllLongsSketch sk = KllLongsSketch.newDirectInstance(8, dstMem, memReqSvr);
+ for (int i = 0; i < 16; i++) {
+ sk.update(i + 1, 16);
+ }
+ println(sk.toString(true, true));
+ assertEquals(sk.getN(), 256);
+ assertEquals(sk.getMaxItem(), 16L);
+ assertEquals(sk.getMinItem(), 1L);
+ }
+
+ private static KllLongsSketch getUpdatableDirectLongSketch(int k, int n) {
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance(k);
+ for (int i = 1; i <= n; i++) { sk.update(i); }
+ byte[] byteArr = KllHelper.toByteArray(sk, true);
+ WritableMemory wmem = WritableMemory.writableWrap(byteArr);
+ KllLongsSketch dfsk = KllLongsSketch.writableWrap(wmem, memReqSvr);
+ return dfsk;
+ }
+
+ @Test
+ public void checkMergeExceptionsWrongType() {
+ KllLongsSketch sk1 = KllLongsSketch.newHeapInstance(20);
+ KllDoublesSketch sk2 = KllDoublesSketch.newHeapInstance(20);
+ try { sk1.merge(sk2); fail(); } catch (ClassCastException e) { }
+ try { sk2.merge(sk1); fail(); } catch (ClassCastException e) { }
+ }
+
+ private final static boolean enablePrinting = false;
+
+ /**
+ * @param o the Object to println
+ */
+ private static final void println(final Object o) {
+ if (enablePrinting) { System.out.println(o.toString()); }
+ }
+
+}
diff --git a/src/test/java/org/apache/datasketches/kll/KllDoublesSketchSerDeTest.java b/src/test/java/org/apache/datasketches/kll/KllDoublesSketchSerDeTest.java
index e07a395da..007cc8370 100644
--- a/src/test/java/org/apache/datasketches/kll/KllDoublesSketchSerDeTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllDoublesSketchSerDeTest.java
@@ -64,7 +64,7 @@ public void serializeDeserializeEmpty() {
@Test
public void serializeDeserializeOneValue() {
final KllDoublesSketch sk1 = KllDoublesSketch.newHeapInstance();
- sk1.update(1);
+ sk1.update(1.0);
//from heap -> byte[] -> heap
final byte[] bytes = sk1.toByteArray();
diff --git a/src/test/java/org/apache/datasketches/kll/KllDoublesSketchTest.java b/src/test/java/org/apache/datasketches/kll/KllDoublesSketchTest.java
index 0b3818f1f..e143577f4 100644
--- a/src/test/java/org/apache/datasketches/kll/KllDoublesSketchTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllDoublesSketchTest.java
@@ -165,8 +165,8 @@ public void manyValuesEstimationMode() {
assertEquals(pmf[0], 0.5, PMF_EPS_FOR_K_256);
assertEquals(pmf[1], 0.5, PMF_EPS_FOR_K_256);
- assertEquals(sketch.getMinItem(), 0f); // min value is exact
- assertEquals(sketch.getMaxItem(), n - 1f); // max value is exact
+ assertEquals(sketch.getMinItem(), 0.0); // min value is exact
+ assertEquals(sketch.getMaxItem(), n - 1.0); // max value is exact
// check at every 0.1 percentage point
final double[] fractions = new double[1001];
@@ -261,11 +261,11 @@ public void mergeLowerK() {
sketch2.update(2 * n - i - 1);
}
- assertEquals(sketch1.getMinItem(), 0.0f);
- assertEquals(sketch1.getMaxItem(), n - 1f);
+ assertEquals(sketch1.getMinItem(), 0.0);
+ assertEquals(sketch1.getMaxItem(), n - 1);
assertEquals(sketch2.getMinItem(), n);
- assertEquals(sketch2.getMaxItem(), 2f * n - 1.0);
+ assertEquals(sketch2.getMaxItem(), 2.0 * n - 1.0);
assertTrue(sketch1.getNormalizedRankError(false) < sketch2.getNormalizedRankError(false));
assertTrue(sketch1.getNormalizedRankError(true) < sketch2.getNormalizedRankError(true));
@@ -306,7 +306,7 @@ public void mergeEmptyLowerK() {
sketch2.merge(sketch1);
assertFalse(sketch1.isEmpty());
assertEquals(sketch1.getN(), n);
- assertEquals(sketch1.getMinItem(), 0f);
+ assertEquals(sketch1.getMinItem(), 0.0);
assertEquals(sketch1.getMaxItem(), n - 1.0);
assertEquals(sketch1.getQuantile(0.5), n / 2.0, n * PMF_EPS_FOR_K_256);
}
@@ -424,7 +424,7 @@ public void checkNewDirectInstanceAndSize() {
KllDoublesSketch.newDirectInstance(wmem, memReqSvr);
try { KllDoublesSketch.newDirectInstance(null, memReqSvr); fail(); }
catch (NullPointerException e) { }
- try { KllFloatsSketch.newDirectInstance(wmem, null); fail(); }
+ try { KllDoublesSketch.newDirectInstance(wmem, null); fail(); }
catch (NullPointerException e) { }
int updateSize = KllSketch.getMaxSerializedSizeBytes(200, 0, DOUBLES_SKETCH, true);
int compactSize = KllSketch.getMaxSerializedSizeBytes(200, 0, DOUBLES_SKETCH, false);
diff --git a/src/test/java/org/apache/datasketches/kll/KllItemsSketchTest.java b/src/test/java/org/apache/datasketches/kll/KllItemsSketchTest.java
index 00028e341..2e6cc1ea1 100644
--- a/src/test/java/org/apache/datasketches/kll/KllItemsSketchTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllItemsSketchTest.java
@@ -570,7 +570,7 @@ public void checkCDF_PDF() {
}
@Test
- public void checkWrapCase1Floats() {
+ public void checkWrapCase1Items() {
KllItemsSketch sk = KllItemsSketch.newHeapInstance(20, Comparator.naturalOrder(), serDe);
final int n = 21;
final int digits = Util.numDigits(n);
@@ -599,15 +599,18 @@ public void checkReadOnlyExceptions() {
@Test
public void checkIsSameResource() {
int cap = 128;
- WritableMemory wmem = WritableMemory.allocate(cap);
+ WritableMemory wmem = WritableMemory.allocate(cap); //heap
WritableMemory reg1 = wmem.writableRegion(0, 64);
WritableMemory reg2 = wmem.writableRegion(64, 64);
assertFalse(reg1 == reg2);
- assertFalse(reg1.isSameResource(reg2));
+ assertFalse(reg1.isSameResource(reg2)); //same original resource, but different offsets
WritableMemory reg3 = wmem.writableRegion(0, 64);
assertFalse(reg1 == reg3);
- assertTrue(reg1.isSameResource(reg3));
+ assertTrue(reg1.isSameResource(reg3)); //same original resource, same offsets, different views.
+ reg1.putInt(0, -1);
+ assertEquals(-1, reg3.getInt(0)); //proof
+ reg1.putInt(0, 0);
byte[] byteArr1 = KllItemsSketch.newHeapInstance(20, Comparator.naturalOrder(), serDe).toByteArray();
reg1.putByteArray(0, byteArr1, 0, byteArr1.length);
@@ -615,11 +618,11 @@ public void checkIsSameResource() {
byte[] byteArr2 = KllItemsSketch.newHeapInstance(20, Comparator.naturalOrder(), serDe).toByteArray();
reg2.putByteArray(0, byteArr2, 0, byteArr2.length);
- assertFalse(sk1.isSameResource(reg2));
+ assertFalse(sk1.isSameResource(reg2)); //same original resource, but different offsets
byte[] byteArr3 = KllItemsSketch.newHeapInstance(20, Comparator.naturalOrder(), serDe).toByteArray();
reg3.putByteArray(0, byteArr3, 0, byteArr3.length);
- assertTrue(sk1.isSameResource(reg3));
+ assertTrue(sk1.isSameResource(reg3)); //same original resource, same offsets, different views.
}
// New added tests specially for KllItemsSketch
diff --git a/src/test/java/org/apache/datasketches/kll/KllLongsSketchIteratorTest.java b/src/test/java/org/apache/datasketches/kll/KllLongsSketchIteratorTest.java
new file mode 100644
index 000000000..a98c32c9e
--- /dev/null
+++ b/src/test/java/org/apache/datasketches/kll/KllLongsSketchIteratorTest.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.kll;
+
+import org.apache.datasketches.quantilescommon.LongsSortedViewIterator;
+import org.apache.datasketches.quantilescommon.QuantilesLongsSketchIterator;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import static org.apache.datasketches.quantilescommon.QuantileSearchCriteria.EXCLUSIVE;
+import static org.apache.datasketches.quantilescommon.QuantileSearchCriteria.INCLUSIVE;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+public class KllLongsSketchIteratorTest {
+
+ @Test
+ public void emptySketch() {
+ KllLongsSketch sketch = KllLongsSketch.newHeapInstance();
+ QuantilesLongsSketchIterator it = sketch.iterator();
+ Assert.assertFalse(it.next());
+ }
+
+ @Test
+ public void oneItemSketch() {
+ KllLongsSketch sketch = KllLongsSketch.newHeapInstance();
+ sketch.update(1);
+ QuantilesLongsSketchIterator it = sketch.iterator();
+ Assert.assertTrue(it.next());
+ Assert.assertEquals(it.getQuantile(), 1L);
+ Assert.assertEquals(it.getWeight(), 1);
+ Assert.assertFalse(it.next());
+ }
+
+ @Test
+ public void twoItemSketchForIterator() {
+ KllLongsSketch sketch = KllLongsSketch.newHeapInstance();
+ sketch.update(1);
+ sketch.update(2);
+ QuantilesLongsSketchIterator itr = sketch.iterator();
+ assertTrue(itr.next());
+
+ assertEquals(itr.getQuantile(), 2L);
+ assertEquals(itr.getWeight(), 1);
+
+ assertTrue(itr.next());
+
+ assertEquals(itr.getQuantile(), 1L);
+ assertEquals(itr.getWeight(), 1);
+ }
+
+ @Test
+ public void twoItemSketchForSortedViewIterator() {
+ KllLongsSketch sketch = KllLongsSketch.newHeapInstance();
+ sketch.update(1);
+ sketch.update(2);
+ LongsSortedViewIterator itr = sketch.getSortedView().iterator();
+
+ assertTrue(itr.next());
+
+ assertEquals(itr.getQuantile(), 1L);
+ assertEquals(itr.getWeight(), 1);
+ assertEquals(itr.getNaturalRank(EXCLUSIVE), 0);
+ assertEquals(itr.getNaturalRank(INCLUSIVE), 1);
+ assertEquals(itr.getNormalizedRank(EXCLUSIVE), 0);
+ assertEquals(itr.getNormalizedRank(INCLUSIVE), 0.5);
+
+ assertTrue(itr.next());
+
+ assertEquals(itr.getQuantile(), 2L);
+ assertEquals(itr.getWeight(), 1);
+ assertEquals(itr.getNaturalRank(EXCLUSIVE), 1);
+ assertEquals(itr.getNaturalRank(INCLUSIVE), 2);
+ assertEquals(itr.getNormalizedRank(EXCLUSIVE), 0.5);
+ assertEquals(itr.getNormalizedRank(INCLUSIVE), 1.0);
+ }
+
+ @Test
+ public void bigSketches() {
+ for (int n = 1000; n < 100000; n += 2000) {
+ KllLongsSketch sketch = KllLongsSketch.newHeapInstance();
+ for (int i = 0; i < n; i++) {
+ sketch.update(i);
+ }
+ QuantilesLongsSketchIterator it = sketch.iterator();
+ int count = 0;
+ int weight = 0;
+ while (it.next()) {
+ count++;
+ weight += (int)it.getWeight();
+ }
+ Assert.assertEquals(count, sketch.getNumRetained());
+ Assert.assertEquals(weight, n);
+ }
+ }
+
+}
diff --git a/src/test/java/org/apache/datasketches/kll/KllLongsSketchSerDeTest.java b/src/test/java/org/apache/datasketches/kll/KllLongsSketchSerDeTest.java
new file mode 100644
index 000000000..b9b0f800d
--- /dev/null
+++ b/src/test/java/org/apache/datasketches/kll/KllLongsSketchSerDeTest.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.kll;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+import org.apache.datasketches.common.SketchesArgumentException;
+import org.apache.datasketches.memory.Memory;
+import org.testng.annotations.Test;
+
+public class KllLongsSketchSerDeTest {
+
+ @Test
+ public void serializeDeserializeEmpty() {
+ final int N = 20;
+
+ final KllLongsSketch sk1 = KllLongsSketch.newHeapInstance(N);
+ //Empty: from heap -> byte[] -> heap
+ final byte[] bytes = sk1.toByteArray();
+ final KllLongsSketch sk2 = KllLongsSketch.heapify(Memory.wrap(bytes));
+ assertEquals(bytes.length, sk1.getSerializedSizeBytes());
+ assertTrue(sk2.isEmpty());
+ assertEquals(sk2.getNumRetained(), sk1.getNumRetained());
+ assertEquals(sk2.getN(), sk1.getN());
+ assertEquals(sk2.getNormalizedRankError(false), sk1.getNormalizedRankError(false));
+ try { sk2.getMinItem(); fail(); } catch (SketchesArgumentException e) {}
+ try { sk2.getMaxItem(); fail(); } catch (SketchesArgumentException e) {}
+ assertEquals(sk2.getSerializedSizeBytes(), sk1.getSerializedSizeBytes());
+
+ //Empty: from heap -> byte[] -> off heap
+ final KllLongsSketch sk3 = KllLongsSketch.wrap(Memory.wrap(bytes));
+ assertTrue(sk3.isEmpty());
+ assertEquals(sk3.getNumRetained(), sk1.getNumRetained());
+ assertEquals(sk3.getN(), sk1.getN());
+ assertEquals(sk3.getNormalizedRankError(false), sk1.getNormalizedRankError(false));
+ try { sk3.getMinItem(); fail(); } catch (SketchesArgumentException e) {}
+ try { sk3.getMaxItem(); fail(); } catch (SketchesArgumentException e) {}
+ assertEquals(sk3.getSerializedSizeBytes(), sk1.getSerializedSizeBytes());
+ //from heap -> byte[] -> off heap -> byte[] -> compare byte[]
+ final byte[] bytes2 = sk3.toByteArray();
+ assertEquals(bytes, bytes2);
+ }
+
+ @Test
+ public void serializeDeserializeOneValue() {
+ final KllLongsSketch sk1 = KllLongsSketch.newHeapInstance();
+ sk1.update(1);
+
+ //from heap -> byte[] -> heap
+ final byte[] bytes = sk1.toByteArray();
+ final KllLongsSketch sk2 = KllLongsSketch.heapify(Memory.wrap(bytes));
+ assertEquals(bytes.length, sk1.getSerializedSizeBytes());
+ assertFalse(sk2.isEmpty());
+ assertEquals(sk2.getNumRetained(), 1);
+ assertEquals(sk2.getN(), 1);
+ assertEquals(sk2.getNormalizedRankError(false), sk1.getNormalizedRankError(false));
+ assertEquals(sk2.getMinItem(), 1L);
+ assertEquals(sk2.getMaxItem(), 1L);
+ assertEquals(sk2.getSerializedSizeBytes(), Long.BYTES + Long.BYTES);
+
+ //from heap -> byte[] -> off heap
+ final KllLongsSketch sk3 = KllLongsSketch.wrap(Memory.wrap(bytes));
+ assertFalse(sk3.isEmpty());
+ assertEquals(sk3.getNumRetained(), 1);
+ assertEquals(sk3.getN(), 1);
+ assertEquals(sk3.getNormalizedRankError(false), sk1.getNormalizedRankError(false));
+ assertEquals(sk3.getMinItem(), 1L);
+ assertEquals(sk3.getMaxItem(), 1L);
+ assertEquals(sk3.getSerializedSizeBytes(), sk1.getSerializedSizeBytes());
+ //from heap -> byte[] -> off heap -> byte[] -> compare byte[]
+ final byte[] bytes2 = sk3.toByteArray();
+ assertEquals(bytes, bytes2);
+ }
+
+ @Test
+ public void serializeDeserializeMultipleValues() {
+ final KllLongsSketch sk1 = KllLongsSketch.newHeapInstance();
+ final int n = 1000;
+ for (int i = 0; i < n; i++) {
+ sk1.update(i);
+ }
+ assertEquals(sk1.getMinItem(), 0);
+ assertEquals(sk1.getMaxItem(), 999L);
+
+ //from heap -> byte[] -> heap
+ final byte[] bytes = sk1.toByteArray();
+ final KllLongsSketch sk2 = KllLongsSketch.heapify(Memory.wrap(bytes));
+ assertEquals(bytes.length, sk1.getSerializedSizeBytes());
+ assertFalse(sk2.isEmpty());
+ assertEquals(sk2.getNumRetained(), sk1.getNumRetained());
+ assertEquals(sk2.getN(), sk1.getN());
+ assertEquals(sk2.getNormalizedRankError(false), sk1.getNormalizedRankError(false));
+ assertEquals(sk2.getMinItem(), sk1.getMinItem());
+ assertEquals(sk2.getMaxItem(), sk1.getMaxItem());
+ assertEquals(sk2.getSerializedSizeBytes(), sk1.getSerializedSizeBytes());
+
+ //from heap -> byte[] -> off heap
+ final KllLongsSketch sk3 = KllLongsSketch.wrap(Memory.wrap(bytes));
+ assertFalse(sk3.isEmpty());
+ assertEquals(sk3.getNumRetained(), sk1.getNumRetained());
+ assertEquals(sk3.getN(), sk1.getN());
+ assertEquals(sk3.getNormalizedRankError(false), sk1.getNormalizedRankError(false));
+ assertEquals(sk3.getMinItem(), sk1.getMinItem());
+ assertEquals(sk3.getMaxItem(), sk1.getMaxItem());
+ assertEquals(sk3.getSerializedSizeBytes(), sk1.getSerializedSizeBytes());
+ //from heap -> byte[] -> off heap -> byte[] -> compare byte[]
+ final byte[] bytes2 = sk3.toByteArray();
+ assertEquals(bytes, bytes2);
+ }
+
+}
diff --git a/src/test/java/org/apache/datasketches/kll/KllLongsSketchTest.java b/src/test/java/org/apache/datasketches/kll/KllLongsSketchTest.java
new file mode 100644
index 000000000..1e4c1004a
--- /dev/null
+++ b/src/test/java/org/apache/datasketches/kll/KllLongsSketchTest.java
@@ -0,0 +1,719 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.kll;
+
+import static java.lang.Math.min;
+import static org.apache.datasketches.kll.KllSketch.SketchType.LONGS_SKETCH;
+import static org.apache.datasketches.quantilescommon.QuantileSearchCriteria.EXCLUSIVE;
+import static org.apache.datasketches.quantilescommon.QuantileSearchCriteria.INCLUSIVE;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+import org.apache.datasketches.common.SketchesArgumentException;
+import org.apache.datasketches.memory.DefaultMemoryRequestServer;
+import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.memory.WritableMemory;
+import org.apache.datasketches.quantilescommon.LongsSortedView;
+import org.apache.datasketches.quantilescommon.LongsSortedViewIterator;
+import org.testng.annotations.Test;
+
+public class KllLongsSketchTest {
+ private static final String LS = System.getProperty("line.separator");
+ private static final double PMF_EPS_FOR_K_8 = KllSketch.getNormalizedRankError(8, true);
+ private static final double PMF_EPS_FOR_K_128 = KllSketch.getNormalizedRankError(128, true);
+ private static final double PMF_EPS_FOR_K_256 = KllSketch.getNormalizedRankError(256, true);
+ private static final double NUMERIC_NOISE_TOLERANCE = 1E-6;
+ private static final DefaultMemoryRequestServer memReqSvr = new DefaultMemoryRequestServer();
+
+ @Test
+ public void empty() {
+ final KllLongsSketch sketch = KllLongsSketch.newHeapInstance();
+ assertTrue(sketch.isEmpty());
+ assertEquals(sketch.getN(), 0);
+ assertEquals(sketch.getNumRetained(), 0);
+ try { sketch.getRank(0); fail(); } catch (SketchesArgumentException e) {}
+ try { sketch.getMinItem(); fail(); } catch (SketchesArgumentException e) {}
+ try { sketch.getMaxItem(); fail(); } catch (SketchesArgumentException e) {}
+ try { sketch.getQuantile(0.5); fail(); } catch (SketchesArgumentException e) {}
+ try { sketch.getQuantiles(new double[] {0}); fail(); } catch (SketchesArgumentException e) {}
+ try { sketch.getPMF(new long[] {0}); fail(); } catch (SketchesArgumentException e) {}
+ try { sketch.getCDF(new long[] {0}); fail(); } catch (SketchesArgumentException e) {}
+ assertNotNull(sketch.toString(true, true));
+ assertNotNull(sketch.toString());
+ }
+
+ @Test(expectedExceptions = SketchesArgumentException.class)
+ public void getQuantileInvalidArg() {
+ final KllLongsSketch sketch = KllLongsSketch.newHeapInstance();
+ sketch.update(1);
+ sketch.getQuantile(-1.0);
+ }
+
+ @Test(expectedExceptions = SketchesArgumentException.class)
+ public void getQuantilesInvalidArg() {
+ final KllLongsSketch sketch = KllLongsSketch.newHeapInstance();
+ sketch.update(1);
+ sketch.getQuantiles(new double[] {2.0});
+ }
+
+ @Test
+ public void oneValue() {
+ final KllLongsSketch sketch = KllLongsSketch.newHeapInstance();
+ sketch.update(1);
+ assertFalse(sketch.isEmpty());
+ assertEquals(sketch.getN(), 1);
+ assertEquals(sketch.getNumRetained(), 1);
+ assertEquals(sketch.getRank(0L, EXCLUSIVE), 0.0);
+ assertEquals(sketch.getRank(1L, EXCLUSIVE), 0.0);
+ assertEquals(sketch.getRank(2L, EXCLUSIVE), 1.0);
+ assertEquals(sketch.getRank(0L, INCLUSIVE), 0.0);
+ assertEquals(sketch.getRank(1L, INCLUSIVE), 1.0);
+ assertEquals(sketch.getRank(2L, INCLUSIVE), 1.0);
+ assertEquals(sketch.getMinItem(), 1L);
+ assertEquals(sketch.getMaxItem(), 1L);
+ assertEquals(sketch.getQuantile(0.5, EXCLUSIVE), 1L);
+ assertEquals(sketch.getQuantile(0.5, INCLUSIVE), 1L);
+ }
+
+ @Test
+ public void tenValues() {
+ final KllLongsSketch sketch = KllLongsSketch.newHeapInstance(20);
+ for (int i = 1; i <= 10; i++) { sketch.update(i); }
+ assertFalse(sketch.isEmpty());
+ assertEquals(sketch.getN(), 10);
+ assertEquals(sketch.getNumRetained(), 10);
+ for (int i = 1; i <= 10; i++) {
+ assertEquals(sketch.getRank(i, EXCLUSIVE), (i - 1) / 10.0);
+ assertEquals(sketch.getRank(i, INCLUSIVE), i / 10.0);
+ }
+ final long[] qArr = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10};
+ double[] rOut = sketch.getRanks(qArr); //inclusive
+ for (int i = 0; i < qArr.length; i++) {
+ assertEquals(rOut[i], (i + 1) / 10.0);
+ }
+ rOut = sketch.getRanks(qArr, EXCLUSIVE); //exclusive
+ for (int i = 0; i < qArr.length; i++) {
+ assertEquals(rOut[i], i / 10.0);
+ }
+
+ for (int i = 0; i >= 10; i++) {
+ double rank = i/10.0;
+ double q = rank == 1.0 ? i : i + 1;
+ assertEquals(sketch.getQuantile(rank, EXCLUSIVE), q);
+ q = rank == 0 ? i + 1 : i;
+ assertEquals(sketch.getQuantile(rank, INCLUSIVE), q);
+ }
+
+ {
+ // getQuantile() and getQuantiles() equivalence EXCLUSIVE
+ final long[] quantiles =
+ sketch.getQuantiles(new double[] {0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0}, EXCLUSIVE);
+ for (int i = 0; i <= 10; i++) {
+ assertEquals(sketch.getQuantile(i / 10.0, EXCLUSIVE), quantiles[i]);
+ }
+ }
+ {
+ // getQuantile() and getQuantiles() equivalence INCLUSIVE
+ final long[] quantiles =
+ sketch.getQuantiles(new double[] {0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0}, INCLUSIVE);
+ for (int i = 0; i <= 10; i++) {
+ assertEquals(sketch.getQuantile(i / 10.0, INCLUSIVE), quantiles[i]);
+ }
+ }
+ }
+
+ @Test
+ public void manyValuesEstimationMode() {
+ final KllLongsSketch sketch = KllLongsSketch.newHeapInstance();
+ final int n = 1_000_000;
+
+ for (int i = 0; i < n; i++) {
+ sketch.update(i);
+ }
+ assertEquals(sketch.getN(), n);
+
+ // test getRank
+ for (int i = 0; i < n; i++) {
+ final double trueRank = (double) i / n;
+ assertEquals(sketch.getRank(i), trueRank, PMF_EPS_FOR_K_256, "for value " + i);
+ }
+
+ // test getPMF
+ final double[] pmf = sketch.getPMF(new long[] {n / 2}); // split at median
+ assertEquals(pmf.length, 2);
+ assertEquals(pmf[0], 0.5, PMF_EPS_FOR_K_256);
+ assertEquals(pmf[1], 0.5, PMF_EPS_FOR_K_256);
+
+ assertEquals(sketch.getMinItem(), 0); // min value is exact
+ assertEquals(sketch.getMaxItem(), n - 1); // max value is exact
+
+ // check at every 0.1 percentage point
+ final double[] fractions = new double[1001];
+ final double[] reverseFractions = new double[1001]; // check that ordering doesn't matter
+ for (int i = 0; i <= 1000; i++) {
+ fractions[i] = (double) i / 1000;
+ reverseFractions[1000 - i] = fractions[i];
+ }
+ final long[] quantiles = sketch.getQuantiles(fractions);
+ final long[] reverseQuantiles = sketch.getQuantiles(reverseFractions);
+ double previousQuantile = 0;
+ for (int i = 0; i <= 1000; i++) {
+ final double quantile = sketch.getQuantile(fractions[i]);
+ assertEquals(quantile, quantiles[i]);
+ assertEquals(quantile, reverseQuantiles[1000 - i]);
+ assertTrue(previousQuantile <= quantile);
+ previousQuantile = quantile;
+ }
+ }
+
+ @Test
+ public void getRankGetCdfGetPmfConsistency() {
+ final KllLongsSketch sketch = KllLongsSketch.newHeapInstance();
+ final int n = 1000;
+ final long[] values = new long[n];
+ for (int i = 0; i < n; i++) {
+ sketch.update(i);
+ values[i] = i;
+ }
+ { // inclusive = false (default)
+ final double[] ranks = sketch.getCDF(values);
+ final double[] pmf = sketch.getPMF(values);
+ double sumPmf = 0;
+ for (int i = 0; i < n; i++) {
+ assertEquals(ranks[i], sketch.getRank(values[i]), NUMERIC_NOISE_TOLERANCE,
+ "rank vs CDF for value " + i);
+ sumPmf += pmf[i];
+ assertEquals(ranks[i], sumPmf, NUMERIC_NOISE_TOLERANCE, "CDF vs PMF for value " + i);
+ }
+ sumPmf += pmf[n];
+ assertEquals(sumPmf, 1.0, NUMERIC_NOISE_TOLERANCE);
+ assertEquals(ranks[n], 1.0, NUMERIC_NOISE_TOLERANCE);
+ }
+ { // inclusive = true
+ final double[] ranks = sketch.getCDF(values, INCLUSIVE);
+ final double[] pmf = sketch.getPMF(values, INCLUSIVE);
+ double sumPmf = 0;
+ for (int i = 0; i < n; i++) {
+ assertEquals(ranks[i], sketch.getRank(values[i], INCLUSIVE), NUMERIC_NOISE_TOLERANCE,
+ "rank vs CDF for value " + i);
+ sumPmf += pmf[i];
+ assertEquals(ranks[i], sumPmf, NUMERIC_NOISE_TOLERANCE, "CDF vs PMF for value " + i);
+ }
+ sumPmf += pmf[n];
+ assertEquals(sumPmf, 1.0, NUMERIC_NOISE_TOLERANCE);
+ assertEquals(ranks[n], 1.0, NUMERIC_NOISE_TOLERANCE);
+ }
+ }
+
+ @Test
+ public void merge() {
+ final KllLongsSketch sketch1 = KllLongsSketch.newHeapInstance();
+ final KllLongsSketch sketch2 = KllLongsSketch.newHeapInstance();
+ final int n = 10000;
+ for (int i = 0; i < n; i++) {
+ sketch1.update(i);
+ sketch2.update(2 * n - i - 1);
+ }
+
+ assertEquals(sketch1.getMinItem(), 0);
+ assertEquals(sketch1.getMaxItem(), (n - 1));
+
+ assertEquals(sketch2.getMinItem(), n);
+ assertEquals(sketch2.getMaxItem(), (2 * n - 1));
+
+ sketch1.merge(sketch2);
+
+ assertFalse(sketch1.isEmpty());
+ assertEquals(sketch1.getN(), 2L * n);
+ assertEquals(sketch1.getMinItem(), 0);
+ assertEquals(sketch1.getMaxItem(), (2 * n - 1));
+ assertEquals(sketch1.getQuantile(0.5), n, 2 * n * PMF_EPS_FOR_K_256);
+ }
+
+ @Test
+ public void mergeLowerK() {
+ final KllLongsSketch sketch1 = KllLongsSketch.newHeapInstance(256);
+ final KllLongsSketch sketch2 = KllLongsSketch.newHeapInstance(128);
+ final int n = 10000;
+ for (int i = 0; i < n; i++) {
+ sketch1.update(i);
+ sketch2.update(2 * n - i - 1);
+ }
+
+ assertEquals(sketch1.getMinItem(), 0);
+ assertEquals(sketch1.getMaxItem(), n - 1L);
+
+ assertEquals(sketch2.getMinItem(), n);
+ assertEquals(sketch2.getMaxItem(), 2L * n - 1L);
+
+ assertTrue(sketch1.getNormalizedRankError(false) < sketch2.getNormalizedRankError(false));
+ assertTrue(sketch1.getNormalizedRankError(true) < sketch2.getNormalizedRankError(true));
+ sketch1.merge(sketch2);
+
+ // sketch1 must get "contaminated" by the lower K in sketch2
+ assertEquals(sketch1.getNormalizedRankError(false), sketch2.getNormalizedRankError(false));
+ assertEquals(sketch1.getNormalizedRankError(true), sketch2.getNormalizedRankError(true));
+
+ assertFalse(sketch1.isEmpty());
+ assertEquals(sketch1.getN(), 2 * n);
+ assertEquals(sketch1.getMinItem(), 0);
+ assertEquals(sketch1.getMaxItem(), 2L * n - 1L);
+ assertEquals(sketch1.getQuantile(0.5), n, 2L * n * PMF_EPS_FOR_K_128);
+ }
+
+ @Test
+ public void mergeEmptyLowerK() {
+ final KllLongsSketch sketch1 = KllLongsSketch.newHeapInstance(256);
+ final KllLongsSketch sketch2 = KllLongsSketch.newHeapInstance(128);
+ final int n = 10_000;
+ for (int i = 0; i < n; i++) {
+ sketch1.update(i);
+ }
+
+ // rank error should not be affected by a merge with an empty sketch with lower K
+ final double rankErrorBeforeMerge = sketch1.getNormalizedRankError(true);
+ sketch1.merge(sketch2);
+ assertEquals(sketch1.getNormalizedRankError(true), rankErrorBeforeMerge);
+
+ assertFalse(sketch1.isEmpty());
+ assertEquals(sketch1.getN(), n);
+ assertEquals(sketch1.getMinItem(), 0);
+ assertEquals(sketch1.getMaxItem(), n - 1);
+ assertEquals(sketch1.getQuantile(0.5), n / 2, n * PMF_EPS_FOR_K_256);
+
+ //merge the other way
+ sketch2.merge(sketch1);
+ assertFalse(sketch1.isEmpty());
+ assertEquals(sketch1.getN(), n);
+ assertEquals(sketch1.getMinItem(), 0);
+ assertEquals(sketch1.getMaxItem(), n - 1);
+ assertEquals(sketch1.getQuantile(0.5), n / 2, n * PMF_EPS_FOR_K_256);
+ }
+
+ @Test
+ public void mergeExactModeLowerK() {
+ final KllLongsSketch sketch1 = KllLongsSketch.newHeapInstance(256);
+ final KllLongsSketch sketch2 = KllLongsSketch.newHeapInstance(128);
+ final int n = 10000;
+ for (int i = 0; i < n; i++) {
+ sketch1.update(i);
+ }
+ sketch2.update(1);
+
+ // rank error should not be affected by a merge with a sketch in exact mode with lower K
+ final double rankErrorBeforeMerge = sketch1.getNormalizedRankError(true);
+ sketch1.merge(sketch2);
+ assertEquals(sketch1.getNormalizedRankError(true), rankErrorBeforeMerge);
+ }
+
+ @Test
+ public void mergeMinMinValueFromOther() {
+ final KllLongsSketch sketch1 = KllLongsSketch.newHeapInstance();
+ final KllLongsSketch sketch2 = KllLongsSketch.newHeapInstance();
+ sketch1.update(1);
+ sketch2.update(2);
+ sketch2.merge(sketch1);
+ assertEquals(sketch2.getMinItem(), 1);
+ }
+
+ @Test
+ public void mergeMinAndMaxFromOther() {
+ final KllLongsSketch sketch1 = KllLongsSketch.newHeapInstance();
+ for (int i = 1; i <= 1_000_000; i++) {
+ sketch1.update(i);
+ }
+ final KllLongsSketch sketch2 = KllLongsSketch.newHeapInstance(10);
+ sketch2.merge(sketch1);
+ assertEquals(sketch2.getMinItem(), 1);
+ assertEquals(sketch2.getMaxItem(), 1_000_000);
+ }
+
+ @Test(expectedExceptions = SketchesArgumentException.class)
+ public void kTooSmall() {
+ KllLongsSketch.newHeapInstance(KllSketch.DEFAULT_M - 1);
+ }
+
+ @Test(expectedExceptions = SketchesArgumentException.class)
+ public void kTooLarge() {
+ KllLongsSketch.newHeapInstance(KllSketch.MAX_K + 1);
+ }
+
+ @Test
+ public void minK() {
+ final KllLongsSketch sketch = KllLongsSketch.newHeapInstance(KllSketch.DEFAULT_M);
+ for (int i = 0; i < 1000; i++) {
+ sketch.update(i);
+ }
+ assertEquals(sketch.getK(), KllSketch.DEFAULT_M);
+ assertEquals(sketch.getQuantile(0.5), 500, 1000 * PMF_EPS_FOR_K_8);
+ }
+
+ @Test
+ public void maxK() {
+ final KllLongsSketch sketch = KllLongsSketch.newHeapInstance(KllSketch.MAX_K);
+ for (int i = 0; i < 1000; i++) {
+ sketch.update(i);
+ }
+ assertEquals(sketch.getK(), KllSketch.MAX_K);
+ assertEquals(sketch.getQuantile(0.5), 500, 1000 * PMF_EPS_FOR_K_256);
+ }
+
+ @Test(expectedExceptions = SketchesArgumentException.class)
+ public void outOfOrderSplitPoints() {
+ final KllLongsSketch sketch = KllLongsSketch.newHeapInstance();
+ sketch.update(0);
+ sketch.getCDF(new long[] {1L, 0L});
+ }
+
+ @Test
+ public void checkReset() {
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance(20);
+ for (int i = 1; i <= 100; i++) { sk.update(i); }
+ long n1 = sk.getN();
+ double min1 = sk.getMinItem();
+ double max1 = sk.getMaxItem();
+ sk.reset();
+ for (int i = 1; i <= 100; i++) { sk.update(i); }
+ long n2 = sk.getN();
+ double min2 = sk.getMinItem();
+ double max2 = sk.getMaxItem();
+ assertEquals(n2, n1);
+ assertEquals(min2, min1);
+ assertEquals(max2, max1);
+ }
+
+ @Test
+ public void checkReadOnlyUpdate() {
+ KllLongsSketch sk1 = KllLongsSketch.newHeapInstance(20);
+ Memory mem = Memory.wrap(sk1.toByteArray());
+ KllLongsSketch sk2 = KllLongsSketch.wrap(mem);
+ try { sk2.update(1); fail(); } catch (SketchesArgumentException e) { }
+ }
+
+ @Test
+ public void checkNewDirectInstanceAndSize() {
+ WritableMemory wmem = WritableMemory.allocate(3000);
+ KllLongsSketch.newDirectInstance(wmem, memReqSvr);
+ try { KllLongsSketch.newDirectInstance(null, memReqSvr); fail(); }
+ catch (NullPointerException e) { }
+ try { KllLongsSketch.newDirectInstance(wmem, null); fail(); }
+ catch (NullPointerException e) { }
+ int updateSize = KllSketch.getMaxSerializedSizeBytes(200, 0, LONGS_SKETCH, true);
+ int compactSize = KllSketch.getMaxSerializedSizeBytes(200, 0, LONGS_SKETCH, false);
+ assertTrue(compactSize < updateSize);
+ }
+
+ @Test
+ public void sortedView() {
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance();
+ sk.update(3);
+ sk.update(1);
+ sk.update(2);
+
+ LongsSortedView view = sk.getSortedView();
+ LongsSortedViewIterator itr = view.iterator();
+ assertEquals(itr.next(), true);
+ assertEquals(itr.getQuantile(), 1);
+ assertEquals(itr.getWeight(), 1);
+ assertEquals(itr.getNaturalRank(EXCLUSIVE), 0);
+ assertEquals(itr.getNaturalRank(INCLUSIVE), 1);
+ assertEquals(itr.next(), true);
+ assertEquals(itr.getQuantile(), 2);
+ assertEquals(itr.getWeight(), 1);
+ assertEquals(itr.getNaturalRank(EXCLUSIVE), 1);
+ assertEquals(itr.getNaturalRank(INCLUSIVE), 2);
+ assertEquals(itr.next(), true);
+ assertEquals(itr.getQuantile(), 3);
+ assertEquals(itr.getWeight(), 1);
+ assertEquals(itr.getNaturalRank(EXCLUSIVE), 2);
+ assertEquals(itr.getNaturalRank(INCLUSIVE), 3);
+ assertEquals(itr.next(), false);
+ }
+
+ @Test //also visual
+ public void checkCDF_PDF() {
+ final double[] cdfI = {.25, .50, .75, 1.0, 1.0 };
+ final double[] cdfE = {0.0, .25, .50, .75, 1.0 };
+ final double[] pmfI = {.25, .25, .25, .25, 0.0 };
+ final double[] pmfE = {0.0, .25, .25, .25, .25 };
+ final double toll = 1E-10;
+ final KllLongsSketch sketch = KllLongsSketch.newHeapInstance();
+ final long[] doublesIn = {10, 20, 30, 40};
+ for (int i = 0; i < doublesIn.length; i++) { sketch.update(doublesIn[i]); }
+ long[] sp = new long[] { 10, 20, 30, 40 };
+ println("SplitPoints:");
+ for (int i = 0; i < sp.length; i++) {
+ printf("%10d", sp[i]);
+ }
+ println("");
+ println("INCLUSIVE:");
+ double[] cdf = sketch.getCDF(sp, INCLUSIVE);
+ double[] pmf = sketch.getPMF(sp, INCLUSIVE);
+ printf("%10s%10s" + LS, "CDF", "PMF");
+ for (int i = 0; i < cdf.length; i++) {
+ printf("%10.2f%10.2f" + LS, cdf[i], pmf[i]);
+ assertEquals(cdf[i], cdfI[i], toll);
+ assertEquals(pmf[i], pmfI[i], toll);
+ }
+ println("EXCLUSIVE");
+ cdf = sketch.getCDF(sp, EXCLUSIVE);
+ pmf = sketch.getPMF(sp, EXCLUSIVE);
+ printf("%10s%10s" + LS, "CDF", "PMF");
+ for (int i = 0; i < cdf.length; i++) {
+ printf("%10.2f%10.2f" + LS, cdf[i], pmf[i]);
+ assertEquals(cdf[i], cdfE[i], toll);
+ assertEquals(pmf[i], pmfE[i], toll);
+ }
+ }
+
+ @Test
+ public void checkWrapCase1Doubles() {
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance(20);
+ for (int i = 1; i <= 21; i++) { sk.update(i); }
+
+ Memory mem = Memory.wrap(sk.toByteArray());
+ KllLongsSketch sk2 = KllLongsSketch.wrap(mem);
+
+ assertTrue(mem.isReadOnly());
+ assertTrue(sk2.isReadOnly());
+ assertFalse(sk2.isDirect());
+ }
+
+ @Test
+ public void checkWritableWrapCase6And2Doubles() {
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance(20);
+ for (int i = 1; i <= 21; i++) { sk.update(i); }
+
+ WritableMemory wmem = WritableMemory.writableWrap(KllHelper.toByteArray(sk, true));
+ KllLongsSketch sk2 = KllLongsSketch.writableWrap(wmem, memReqSvr);
+
+ assertFalse(wmem.isReadOnly());
+ assertFalse(sk2.isReadOnly());
+ assertFalse(sk2.isDirect());
+ }
+
+ @Test
+ public void checkKllSketchCase5Doubles() {
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance(20);
+ for (int i = 1; i <= 21; i++) { sk.update(i); }
+
+ WritableMemory wmem = WritableMemory.writableWrap(sk.toByteArray());
+ KllLongsSketch sk2 = KllLongsSketch.writableWrap(wmem, memReqSvr);
+
+ assertFalse(wmem.isReadOnly());
+ assertTrue(sk2.isReadOnly());
+ assertFalse(sk2.isDirect());
+ }
+
+ @Test
+ public void checkKllSketchCase3Doubles() {
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance(20);
+ for (int i = 1; i <= 21; i++) { sk.update(i); }
+
+ Memory mem = Memory.wrap(KllHelper.toByteArray(sk, true));
+ WritableMemory wmem = (WritableMemory) mem;
+ KllLongsSketch sk2 = KllLongsSketch.writableWrap(wmem, memReqSvr);
+
+ assertTrue(wmem.isReadOnly());
+ assertTrue(sk2.isReadOnly());
+ assertFalse(sk2.isDirect());
+ }
+
+ @Test
+ public void checkKllSketchCase7Doubles() {
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance(20);
+ for (int i = 1; i <= 21; i++) { sk.update(i); }
+
+ Memory mem = Memory.wrap(KllHelper.toByteArray(sk, true));
+ WritableMemory wmem = (WritableMemory) mem;
+ KllLongsSketch sk2 = KllLongsSketch.writableWrap(wmem, memReqSvr);
+
+ assertTrue(wmem.isReadOnly());
+ assertTrue(sk2.isReadOnly());
+ assertFalse(sk2.isDirect());
+ }
+
+ @Test
+ public void checkReadOnlyExceptions() {
+ int[] intArr = new int[0];
+ int intV = 2;
+ int idx = 1;
+ KllLongsSketch sk1 = KllLongsSketch.newHeapInstance(20);
+ Memory mem = Memory.wrap(sk1.toByteArray());
+ KllLongsSketch sk2 = KllLongsSketch.wrap(mem);
+ try { sk2.setLevelsArray(intArr); fail(); } catch (SketchesArgumentException e) { }
+ try { sk2.setLevelsArrayAt(idx,intV); fail(); } catch (SketchesArgumentException e) { }
+ }
+
+ @Test
+ public void checkIsSameResource() {
+ int cap = 128;
+ WritableMemory wmem = WritableMemory.allocate(cap);
+ WritableMemory reg1 = wmem.writableRegion(0, 64);
+ WritableMemory reg2 = wmem.writableRegion(64, 64);
+ assertFalse(reg1 == reg2);
+ assertFalse(reg1.isSameResource(reg2));
+
+ WritableMemory reg3 = wmem.writableRegion(0, 64);
+ assertFalse(reg1 == reg3);
+ assertTrue(reg1.isSameResource(reg3));
+
+ byte[] byteArr1 = KllLongsSketch.newHeapInstance(20).toByteArray();
+ reg1.putByteArray(0, byteArr1, 0, byteArr1.length);
+ KllLongsSketch sk1 = KllLongsSketch.wrap(reg1);
+
+ byte[] byteArr2 = KllLongsSketch.newHeapInstance(20).toByteArray();
+ reg2.putByteArray(0, byteArr2, 0, byteArr2.length);
+ assertFalse(sk1.isSameResource(reg2));
+
+ byte[] byteArr3 = KllLongsSketch.newHeapInstance(20).toByteArray();
+ reg3.putByteArray(0, byteArr3, 0, byteArr3.length);
+ assertTrue(sk1.isSameResource(reg3));
+ }
+
+ @Test
+ public void checkSortedViewAfterReset() {
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance(20);
+ sk.update(1L);
+ LongsSortedView sv = sk.getSortedView();
+ long dsv = sv.getQuantile(1.0, INCLUSIVE);
+ assertEquals(dsv, 1L);
+ sk.reset();
+ try { sk.getSortedView(); fail(); } catch (SketchesArgumentException e) { }
+ }
+
+ @Test
+ public void checkVectorUpdate() {
+ boolean withLevels = false;
+ boolean withLevelsAndItems = true;
+ int k = 20;
+ int n = 108;
+ int maxVsz = 40; //max vector size
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance(k);
+ int j = 1;
+ int rem;
+ while ((rem = n - j + 1) > 0) {
+ int vecSz = min(rem, maxVsz);
+ long[] v = new long[vecSz];
+ for (int i = 0; i < vecSz; i++) { v[i] = j++; }
+ sk.update(v, 0, vecSz);
+ }
+ println(LS + "#<<< END STATE # >>>");
+ println(sk.toString(withLevels, withLevelsAndItems));
+ println("");
+ assertEquals(sk.getN(), 108);
+ assertEquals(sk.getMaxItem(), 108L);
+ assertEquals(sk.getMinItem(), 1L);
+ }
+
+ @Test
+ public void vectorizedUpdates() {
+ final int trials = 1;
+ final int M = 1; //number of vectors
+ final int N = 1000; //vector size
+ final int K = 256;
+ final long[] values = new long[N];
+ long vIn = 1L;
+ long totN = 0;
+ final long startTime = System.nanoTime();
+ for (int t = 0; t < trials; t++) {
+ final KllLongsSketch sketch = KllLongsSketch.newHeapInstance(K);
+ for (int m = 0; m < M; m++) {
+ for (int n = 0; n < N; n++) {
+ values[n] = vIn++; //fill vector
+ }
+ sketch.update(values, 0, N); //vector input
+ }
+ totN = sketch.getN();
+ assertEquals(totN, M * N);
+ assertEquals(sketch.getMinItem(), 1L);
+ assertEquals(sketch.getMaxItem(), totN);
+ assertEquals(sketch.getQuantile(0.5), totN / 2, totN * PMF_EPS_FOR_K_256 * 2.0); //wider tolerance
+ }
+ final long runTime = System.nanoTime() - startTime;
+ println("Vectorized Updates");
+ printf(" Vector size : %,12d" + LS, N);
+ printf(" Num Vectors : %,12d" + LS, M);
+ printf(" Total Input : %,12d" + LS, totN);
+ printf(" Run Time mS : %,12.3f" + LS, runTime / 1e6);
+ final double trialTime = runTime / (1e6 * trials);
+ printf(" mS / Trial : %,12.3f" + LS, trialTime);
+ final double updateTime = runTime / (1.0 * totN * trials);
+ printf(" nS / Update : %,12.3f" + LS, updateTime);
+ }
+
+ @Test
+ public void nonVectorizedUpdates() {
+ final int trials = 1;
+ final int M = 1; //number of vectors
+ final int N = 1000; //vector size
+ final int K = 256;
+ final long[] values = new long[N];
+ long vIn = 1L;
+ long totN = 0;
+ final long startTime = System.nanoTime();
+ for (int t = 0; t < trials; t++) {
+ final KllLongsSketch sketch = KllLongsSketch.newHeapInstance(K);
+ for (int m = 0; m < M; m++) {
+ for (int n = 0; n < N; n++) {
+ values[n] = vIn++; //fill vector
+ }
+ for (int i = 0; i < N; i++) {
+ sketch.update(values[i]); //single item input
+ }
+ }
+ totN = sketch.getN();
+ assertEquals(totN, M * N);
+ assertEquals(sketch.getMinItem(), 1L);
+ assertEquals(sketch.getMaxItem(), totN);
+ assertEquals(sketch.getQuantile(0.5), totN / 2, totN * PMF_EPS_FOR_K_256 * 2.0); //wider tolerance
+ }
+ final long runTime = System.nanoTime() - startTime;
+ println("Vectorized Updates");
+ printf(" Vector size : %,12d" + LS, N);
+ printf(" Num Vectors : %,12d" + LS, M);
+ printf(" Total Input : %,12d" + LS, totN);
+ printf(" Run Time mS : %,12.3f" + LS, runTime / 1e6);
+ final double trialTime = runTime / (1e6 * trials);
+ printf(" mS / Trial : %,12.3f" + LS, trialTime);
+ final double updateTime = runTime / (1.0 * totN * trials);
+ printf(" nS / Update : %,12.3f" + LS, updateTime);
+ }
+
+ private final static boolean enablePrinting = false;
+
+ /**
+ * @param format the format
+ * @param args the args
+ */
+ private static final void printf(final String format, final Object ...args) {
+ if (enablePrinting) { System.out.printf(format, args); }
+ }
+
+ /**
+ * @param o the Object to println
+ */
+ private static final void println(final Object o) {
+ if (enablePrinting) { System.out.println(o.toString()); }
+ }
+}
diff --git a/src/test/java/org/apache/datasketches/kll/KllMiscDirectLongsTest.java b/src/test/java/org/apache/datasketches/kll/KllMiscDirectLongsTest.java
new file mode 100644
index 000000000..366f93f7d
--- /dev/null
+++ b/src/test/java/org/apache/datasketches/kll/KllMiscDirectLongsTest.java
@@ -0,0 +1,459 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.kll;
+
+import static org.apache.datasketches.kll.KllSketch.SketchType.LONGS_SKETCH;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+import org.apache.datasketches.common.SketchesArgumentException;
+import org.apache.datasketches.memory.DefaultMemoryRequestServer;
+import org.apache.datasketches.memory.WritableMemory;
+import org.apache.datasketches.quantilescommon.LongsSortedView;
+import org.apache.datasketches.quantilescommon.LongsSortedViewIterator;
+import org.testng.annotations.Test;
+
+public class KllMiscDirectLongsTest {
+ static final String LS = System.getProperty("line.separator");
+ private static final DefaultMemoryRequestServer memReqSvr = new DefaultMemoryRequestServer();
+
+ @Test
+ public void checkBounds() {
+ final KllLongsSketch kll = getDirectLongsSketch(200, 0);
+ for (int i = 0; i < 1000; i++) {
+ kll.update(i);
+ }
+ final double eps = kll.getNormalizedRankError(false);
+ final long est = kll.getQuantile(0.5);
+ final long ub = kll.getQuantileUpperBound(0.5);
+ final long lb = kll.getQuantileLowerBound(0.5);
+ assertEquals(ub, kll.getQuantile(.5 + eps));
+ assertEquals(lb, kll.getQuantile(0.5 - eps));
+ println("Ext : " + est);
+ println("UB : " + ub);
+ println("LB : " + lb);
+ final double rest = kll.getRank(est);
+ final double restUB = kll.getRankUpperBound(rest);
+ final double restLB = kll.getRankLowerBound(rest);
+ assertTrue(restUB - rest < (2 * eps));
+ assertTrue(rest - restLB < (2 * eps));
+ }
+
+ //@Test //enable static println(..) for visual checking
+ public void visualCheckToString() {
+ final int k = 20;
+ final KllLongsSketch sk = getDirectLongsSketch(k, 0);
+ for (int i = 0; i < 10; i++) { sk.update(i + 1); }
+ println(sk.toString(true, true));
+
+ final KllLongsSketch sk2 = getDirectLongsSketch(k, 0);
+ for (int i = 0; i < 400; i++) { sk2.update(i + 1); }
+ println("\n" + sk2.toString(true, true));
+
+ sk2.merge(sk);
+ final String s2 = sk2.toString(true, true);
+ println(LS + s2);
+ }
+
+ @Test
+ public void viewDirectCompactions() {
+ int k = 20;
+ int u = 108;
+ KllLongsSketch sk = getDirectLongsSketch(k, 0);
+ for (int i = 1; i <= u; i++) {
+ sk.update(i);
+ if (sk.levelsArr[0] == 0) {
+ println(sk.toString(true, true));
+ sk.update(++i);
+ println(sk.toString(true, true));
+ assertEquals(sk.getLongItemsArray()[sk.levelsArr[0]], i);
+ }
+ }
+ }
+
+ @Test
+ public void viewCompactionAndSortedView() {
+ int k = 20;
+ KllLongsSketch sk = getDirectLongsSketch(k, 0);
+ show(sk, 20);
+ LongsSortedView sv = sk.getSortedView();
+ LongsSortedViewIterator itr = sv.iterator();
+ printf("%12s%12s\n", "Value", "CumWeight");
+ while (itr.next()) {
+ long v = itr.getQuantile();
+ long wt = itr.getWeight();
+ printf("%12d%12d\n", v, wt);
+ }
+ }
+
+ private static void show(final KllLongsSketch sk, int limit) {
+ int i = (int) sk.getN();
+ for ( ; i < limit; i++) { sk.update(i + 1); }
+ println(sk.toString(true, true));
+ }
+
+ @Test
+ public void checkSketchInitializeLongHeap() {
+ int k = 20; //don't change this
+ KllLongsSketch sk;
+
+ //println("#### CASE: LONG FULL HEAP");
+ sk = getDirectLongsSketch(k, 0);
+ for (int i = 1; i <= k + 1; i++) { sk.update(i); }
+ //println(sk.toString(true, true));
+ assertEquals(sk.getK(), k);
+ assertEquals(sk.getN(), k + 1);
+ assertEquals(sk.getNumRetained(), 11);
+ assertFalse(sk.isEmpty());
+ assertTrue(sk.isEstimationMode());
+ assertEquals(sk.getMinK(), k);
+ assertEquals(sk.getLongItemsArray().length, 33);
+ assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 3);
+ assertEquals(sk.getMaxItem(), 21L);
+ assertEquals(sk.getMinItem(), 1L);
+ assertEquals(sk.getNumLevels(), 2);
+ assertFalse(sk.isLevelZeroSorted());
+
+ //println("#### CASE: LONG HEAP EMPTY");
+ sk = getDirectLongsSketch(k, 0);
+ //println(sk.toString(true, true));
+ assertEquals(sk.getK(), k);
+ assertEquals(sk.getN(), 0);
+ assertEquals(sk.getNumRetained(), 0);
+ assertTrue(sk.isEmpty());
+ assertFalse(sk.isEstimationMode());
+ assertEquals(sk.getMinK(), k);
+ assertEquals(sk.getLongItemsArray().length, 20);
+ assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 2);
+ try { sk.getMaxItem(); fail(); } catch (SketchesArgumentException e) { }
+ try { sk.getMinItem(); fail(); } catch (SketchesArgumentException e) { }
+ assertEquals(sk.getNumLevels(), 1);
+ assertFalse(sk.isLevelZeroSorted());
+
+ //println("#### CASE: LONG HEAP SINGLE");
+ sk = getDirectLongsSketch(k, 0);
+ sk.update(1);
+ //println(sk.toString(true, true));
+ assertEquals(sk.getK(), k);
+ assertEquals(sk.getN(), 1);
+ assertEquals(sk.getNumRetained(), 1);
+ assertFalse(sk.isEmpty());
+ assertFalse(sk.isEstimationMode());
+ assertEquals(sk.getMinK(), k);
+ assertEquals(sk.getLongItemsArray().length, 20);
+ assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 2);
+ assertEquals(sk.getMaxItem(), 1L);
+ assertEquals(sk.getMinItem(), 1L);
+ assertEquals(sk.getNumLevels(), 1);
+ assertFalse(sk.isLevelZeroSorted());
+ }
+
+ @Test
+ public void checkSketchInitializeLongHeapifyCompactMem() {
+ int k = 20; //don't change this
+ KllLongsSketch sk;
+ KllLongsSketch sk2;
+ byte[] compBytes;
+ WritableMemory wmem;
+
+ //println("#### CASE: LONG FULL HEAPIFIED FROM COMPACT");
+ sk2 = getDirectLongsSketch(k, 0);
+ for (int i = 1; i <= k + 1; i++) { sk2.update(i); }
+ //println(sk.toString(true, true));
+ compBytes = sk2.toByteArray();
+ wmem = WritableMemory.writableWrap(compBytes);
+ //println(KllPreambleUtil.toString(wmem));
+ sk = KllLongsSketch.heapify(wmem);
+ assertEquals(sk.getK(), k);
+ assertEquals(sk.getN(), k + 1);
+ assertEquals(sk.getNumRetained(), 11);
+ assertFalse(sk.isEmpty());
+ assertTrue(sk.isEstimationMode());
+ assertEquals(sk.getMinK(), k);
+ assertEquals(sk.getLongItemsArray().length, 33);
+ assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 3);
+ assertEquals(sk.getMaxItem(), 21L);
+ assertEquals(sk.getMinItem(), 1L);
+ assertEquals(sk.getNumLevels(), 2);
+ assertFalse(sk.isLevelZeroSorted());
+
+ //println("#### CASE: LONG EMPTY HEAPIFIED FROM COMPACT");
+ sk2 = getDirectLongsSketch(k, 0);
+ //println(sk.toString(true, true));
+ compBytes = sk2.toByteArray();
+ wmem = WritableMemory.writableWrap(compBytes);
+ //println(KllPreambleUtil.toString(wmem));
+ sk = KllLongsSketch.heapify(wmem);
+ assertEquals(sk.getK(), k);
+ assertEquals(sk.getN(), 0);
+ assertEquals(sk.getNumRetained(), 0);
+ assertTrue(sk.isEmpty());
+ assertFalse(sk.isEstimationMode());
+ assertEquals(sk.getMinK(), k);
+ assertEquals(sk.getLongItemsArray().length, 20);
+ assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 2);
+ try { sk.getMaxItem(); fail(); } catch (SketchesArgumentException e) { }
+ try { sk.getMinItem(); fail(); } catch (SketchesArgumentException e) { }
+ assertEquals(sk.getNumLevels(), 1);
+ assertFalse(sk.isLevelZeroSorted());
+
+ //println("#### CASE: LONG SINGLE HEAPIFIED FROM COMPACT");
+ sk2 = getDirectLongsSketch(k, 0);
+ sk2.update(1);
+ //println(sk2.toString(true, true));
+ compBytes = sk2.toByteArray();
+ wmem = WritableMemory.writableWrap(compBytes);
+ //println(KllPreambleUtil.toString(wmem));
+ sk = KllLongsSketch.heapify(wmem);
+ assertEquals(sk.getK(), k);
+ assertEquals(sk.getN(), 1);
+ assertEquals(sk.getNumRetained(), 1);
+ assertFalse(sk.isEmpty());
+ assertFalse(sk.isEstimationMode());
+ assertEquals(sk.getMinK(), k);
+ assertEquals(sk.getLongItemsArray().length, 20);
+ assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 2);
+ assertEquals(sk.getMaxItem(), 1L);
+ assertEquals(sk.getMinItem(), 1L);
+ assertEquals(sk.getNumLevels(), 1);
+ assertFalse(sk.isLevelZeroSorted());
+ }
+
+ @Test
+ public void checkSketchInitializeLongHeapifyUpdatableMem() {
+ int k = 20; //don't change this
+ KllLongsSketch sk;
+ KllLongsSketch sk2;
+ byte[] compBytes;
+ WritableMemory wmem;
+
+ //println("#### CASE: LONG FULL HEAPIFIED FROM UPDATABLE");
+ sk2 = getDirectLongsSketch(k, 0);
+ for (int i = 1; i <= k + 1; i++) { sk2.update(i); }
+ //println(sk2.toString(true, true));
+ compBytes = KllHelper.toByteArray(sk2,true);
+ wmem = WritableMemory.writableWrap(compBytes);
+ sk = KllHeapLongsSketch.heapifyImpl(wmem);
+ //println(sk.toString(true, true));
+ assertEquals(sk.getK(), k);
+ assertEquals(sk.getN(), k + 1);
+ assertEquals(sk.getNumRetained(), 11);
+ assertFalse(sk.isEmpty());
+ assertTrue(sk.isEstimationMode());
+ assertEquals(sk.getMinK(), k);
+ assertEquals(sk.getLongItemsArray().length, 33);
+ assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 3);
+ assertEquals(sk.getMaxItem(), 21L);
+ assertEquals(sk.getMinItem(), 1L);
+ assertEquals(sk.getNumLevels(), 2);
+ assertFalse(sk.isLevelZeroSorted());
+
+ // println("#### CASE: LONG EMPTY HEAPIFIED FROM UPDATABLE");
+ sk2 = getDirectLongsSketch(k, 0);
+ //println(sk.toString(true, true));
+ compBytes = KllHelper.toByteArray(sk2, true);
+ wmem = WritableMemory.writableWrap(compBytes);
+ //println(KllPreambleUtil.toString(wmem));
+ sk = KllHeapLongsSketch.heapifyImpl(wmem);
+ assertEquals(sk.getK(), k);
+ assertEquals(sk.getN(), 0);
+ assertEquals(sk.getNumRetained(), 0);
+ assertTrue(sk.isEmpty());
+ assertFalse(sk.isEstimationMode());
+ assertEquals(sk.getMinK(), k);
+ assertEquals(sk.getLongItemsArray().length, 20);
+ assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 2);
+ try { sk.getMaxItem(); fail(); } catch (SketchesArgumentException e) { }
+ try { sk.getMinItem(); fail(); } catch (SketchesArgumentException e) { }
+ assertEquals(sk.getNumLevels(), 1);
+ assertFalse(sk.isLevelZeroSorted());
+
+ //println("#### CASE: LONG SINGLE HEAPIFIED FROM UPDATABLE");
+ sk2 = getDirectLongsSketch(k, 0);
+ sk2.update(1);
+ //println(sk.toString(true, true));
+ compBytes = KllHelper.toByteArray(sk2,true);
+ wmem = WritableMemory.writableWrap(compBytes);
+ //println(KllPreambleUtil.toString(wmem));
+ sk = KllHeapLongsSketch.heapifyImpl(wmem);
+ assertEquals(sk.getK(), k);
+ assertEquals(sk.getN(), 1);
+ assertEquals(sk.getNumRetained(), 1);
+ assertFalse(sk.isEmpty());
+ assertFalse(sk.isEstimationMode());
+ assertEquals(sk.getMinK(), k);
+ assertEquals(sk.getLongItemsArray().length, 20);
+ assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 2);
+ assertEquals(sk.getMaxItem(), 1L);
+ assertEquals(sk.getMinItem(), 1L);
+ assertEquals(sk.getNumLevels(), 1);
+ assertFalse(sk.isLevelZeroSorted());
+ }
+
+ @Test
+ public void checkMemoryToStringLongUpdatable() {
+ int k = 20; //don't change this
+ KllLongsSketch sk;
+ KllLongsSketch sk2;
+ byte[] upBytes;
+ byte[] upBytes2;
+ WritableMemory wmem;
+ String s;
+
+ println("#### CASE: LONG FULL UPDATABLE");
+ sk = getDirectLongsSketch(k, 0);
+ for (int i = 1; i <= k + 1; i++) { sk.update(i); }
+ upBytes = KllHelper.toByteArray(sk, true);
+ wmem = WritableMemory.writableWrap(upBytes);
+ s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true);
+ println("step 1: sketch to byte[]/memory & analyze memory");
+ println(s);
+ sk2 = KllLongsSketch.writableWrap(wmem, memReqSvr);
+ upBytes2 = KllHelper.toByteArray(sk2, true);
+ wmem = WritableMemory.writableWrap(upBytes2);
+ s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true);
+ println("step 2: memory to heap sketch, to byte[]/memory & analyze memory. Should match above");
+ println(s);
+ assertEquals(upBytes, upBytes2);
+
+ println("#### CASE: LONG EMPTY UPDATABLE");
+ sk = getDirectLongsSketch(k, 0);
+ upBytes = KllHelper.toByteArray(sk, true);
+ wmem = WritableMemory.writableWrap(upBytes);
+ s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true);
+ println("step 1: sketch to byte[]/memory & analyze memory");
+ println(s);
+ sk2 = KllLongsSketch.writableWrap(wmem, memReqSvr);
+ upBytes2 = KllHelper.toByteArray(sk2, true);
+ wmem = WritableMemory.writableWrap(upBytes2);
+ s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true);
+ println("step 2: memory to heap sketch, to byte[]/memory & analyze memory. Should match above");
+ println(s);
+ assertEquals(upBytes, upBytes2);
+
+ println("#### CASE: LONG SINGLE UPDATABL");
+ sk = getDirectLongsSketch(k, 0);
+ sk.update(1);
+ upBytes = KllHelper.toByteArray(sk, true);
+ wmem = WritableMemory.writableWrap(upBytes);
+ s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true);
+ println("step 1: sketch to byte[]/memory & analyze memory");
+ println(s);
+ sk2 = KllLongsSketch.writableWrap(wmem, memReqSvr);
+ upBytes2 = KllHelper.toByteArray(sk2, true);
+ wmem = WritableMemory.writableWrap(upBytes2);
+ s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true);
+ println("step 2: memory to heap sketch, to byte[]/memory & analyze memory. Should match above");
+ println(s);
+ assertEquals(upBytes, upBytes2);
+ }
+
+ @Test
+ public void checkSimpleMerge() {
+ int k = 20;
+ int n1 = 21;
+ int n2 = 21;
+ KllLongsSketch sk1 = getDirectLongsSketch(k, 0);
+ KllLongsSketch sk2 = getDirectLongsSketch(k, 0);
+ for (int i = 1; i <= n1; i++) {
+ sk1.update(i);
+ }
+ for (int i = 1; i <= n2; i++) {
+ sk2.update(i + 100);
+ }
+ println(sk1.toString(true, true));
+ println(sk2.toString(true, true));
+ sk1.merge(sk2);
+ println(sk1.toString(true, true));
+ assertEquals(sk1.getMaxItem(), 121L);
+ assertEquals(sk1.getMinItem(), 1L);
+ }
+
+ @Test
+ public void checkSizes() {
+ KllLongsSketch sk = getDirectLongsSketch(20, 0);
+ for (int i = 1; i <= 21; i++) { sk.update(i); }
+ //println(sk.toString(true, true));
+ byte[] byteArr1 = KllHelper.toByteArray(sk, true);
+ int size1 = sk.currentSerializedSizeBytes(true);
+ assertEquals(size1, byteArr1.length);
+ byte[] byteArr2 = sk.toByteArray();
+ int size2 = sk.currentSerializedSizeBytes(false);
+ assertEquals(size2, byteArr2.length);
+ }
+
+ @Test
+ public void checkNewInstance() {
+ int k = 200;
+ WritableMemory dstMem = WritableMemory.allocate(3000);
+ KllLongsSketch sk = KllLongsSketch.newDirectInstance(k, dstMem, memReqSvr);
+ for (int i = 1; i <= 10_000; i++) {sk.update(i); }
+ assertEquals(sk.getMinItem(), 1L);
+ assertEquals(sk.getMaxItem(), 10000L);
+ //println(sk.toString(true, true));
+ }
+
+ @Test
+ public void checkDifferentM() {
+ int k = 20;
+ int m = 4;
+ WritableMemory dstMem = WritableMemory.allocate(1000);
+ KllLongsSketch sk = KllDirectLongsSketch.newDirectUpdatableInstance(k, m, dstMem, memReqSvr);
+ for (int i = 1; i <= 200; i++) {sk.update(i); }
+ assertEquals(sk.getMinItem(), 1L);
+ assertEquals(sk.getMaxItem(), 200L);
+ }
+
+ private static KllLongsSketch getDirectLongsSketch(final int k, final int n) {
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance(k);
+ for (int i = 1; i <= n; i++) { sk.update(i); }
+ byte[] byteArr = KllHelper.toByteArray(sk, true);
+ WritableMemory wmem = WritableMemory.writableWrap(byteArr);
+ KllLongsSketch dfsk = KllLongsSketch.writableWrap(wmem, memReqSvr);
+ return dfsk;
+ }
+
+ @Test
+ public void printlnTest() {
+ String s = "PRINTING: printf in " + this.getClass().getName();
+ println(s);
+ printf("%s\n", s);
+ }
+
+ private final static boolean enablePrinting = false;
+
+ /**
+ * @param format the format
+ * @param args the args
+ */
+ private static final void printf(final String format, final Object ...args) {
+ if (enablePrinting) { System.out.printf(format, args); }
+ }
+
+ /**
+ * @param o the Object to println
+ */
+ private static final void println(final Object o) {
+ if (enablePrinting) { System.out.println(o.toString()); }
+ }
+
+}
diff --git a/src/test/java/org/apache/datasketches/kll/KllMiscDoublesTest.java b/src/test/java/org/apache/datasketches/kll/KllMiscDoublesTest.java
index e58c27419..4ce988d22 100644
--- a/src/test/java/org/apache/datasketches/kll/KllMiscDoublesTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllMiscDoublesTest.java
@@ -100,8 +100,8 @@ public void checkHeapifyExceptions2() {
@Test(expectedExceptions = SketchesArgumentException.class)
public void checkHeapifyExceptions3() {
KllDoublesSketch sk = KllDoublesSketch.newHeapInstance();
- sk.update(1.0f);
- sk.update(2.0f);
+ sk.update(1.0);
+ sk.update(2.0);
WritableMemory wmem = WritableMemory.writableWrap(sk.toByteArray());
wmem.putByte(0, (byte) 1); //corrupt preamble ints, should be 5
KllDoublesSketch.heapify(wmem);
diff --git a/src/test/java/org/apache/datasketches/kll/KllMiscItemsTest.java b/src/test/java/org/apache/datasketches/kll/KllMiscItemsTest.java
index acf3343d9..5f51a7f1a 100644
--- a/src/test/java/org/apache/datasketches/kll/KllMiscItemsTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllMiscItemsTest.java
@@ -293,7 +293,7 @@ public void checkSketchInitializeItemsHeap() {
final int digits = Util.numDigits(n);
KllItemsSketch sk;
- println("#### CASE: FLOAT FULL HEAP");
+ println("#### CASE: ITEM FULL HEAP");
sk = KllItemsSketch.newHeapInstance(k, Comparator.naturalOrder(), serDe);
for (int i = 1; i <= n; i++) { sk.update(Util.longToFixedLengthString(i, digits)); }
println(sk.toString(true, true));
@@ -310,7 +310,7 @@ public void checkSketchInitializeItemsHeap() {
assertEquals(sk.getNumLevels(), 2);
assertFalse(sk.isLevelZeroSorted());
- println("#### CASE: FLOAT HEAP EMPTY");
+ println("#### CASE: ITEM HEAP EMPTY");
sk = KllItemsSketch.newHeapInstance(k, Comparator.naturalOrder(), serDe);
println(sk.toString(true, true));
assertEquals(sk.getK(), k);
@@ -326,7 +326,7 @@ public void checkSketchInitializeItemsHeap() {
assertEquals(sk.getNumLevels(), 1);
assertFalse(sk.isLevelZeroSorted());
- println("#### CASE: FLOAT HEAP SINGLE");
+ println("#### CASE: ITEM HEAP SINGLE");
sk = KllItemsSketch.newHeapInstance(k, Comparator.naturalOrder(), serDe);
sk.update("1");
println(sk.toString(true, true));
@@ -354,7 +354,7 @@ public void checkSketchInitializeItemsHeapifyCompactMem() {
byte[] compBytes;
Memory mem;
- println("#### CASE: FLOAT FULL HEAPIFIED FROM COMPACT");
+ println("#### CASE: ITEM FULL HEAPIFIED FROM COMPACT");
sk2 = KllItemsSketch.newHeapInstance(k, Comparator.naturalOrder(), serDe);
for (int i = 1; i <= n; i++) { sk2.update(Util.longToFixedLengthString(i, digits)); }
println(sk2.toString(true, true));
@@ -375,7 +375,7 @@ public void checkSketchInitializeItemsHeapifyCompactMem() {
assertEquals(sk.getNumLevels(), 2);
assertFalse(sk.isLevelZeroSorted());
- println("#### CASE: FLOAT EMPTY HEAPIFIED FROM COMPACT");
+ println("#### CASE: ITEM EMPTY HEAPIFIED FROM COMPACT");
sk2 = KllItemsSketch.newHeapInstance(k, Comparator.naturalOrder(), serDe);
//println(sk.toString(true, true));
compBytes = sk2.toByteArray();
@@ -395,7 +395,7 @@ public void checkSketchInitializeItemsHeapifyCompactMem() {
assertEquals(sk.getNumLevels(), 1);
assertFalse(sk.isLevelZeroSorted());
- println("#### CASE: FLOAT SINGLE HEAPIFIED FROM COMPACT");
+ println("#### CASE: ITEM SINGLE HEAPIFIED FROM COMPACT");
sk2 = KllItemsSketch.newHeapInstance(k, Comparator.naturalOrder(), serDe);
sk2.update("1");
//println(sk2.toString(true, true));
@@ -417,7 +417,7 @@ public void checkSketchInitializeItemsHeapifyCompactMem() {
assertFalse(sk.isLevelZeroSorted());
}
- //public void checkSketchInitializeFloatHeapifyUpdatableMem() Not Supported
+ //public void checkSketchInitializeItemHeapifyUpdatableMem() Not Supported
@Test //set static enablePrinting = true for visual checking
public void checkMemoryToStringItemsCompact() {
@@ -431,7 +431,7 @@ public void checkMemoryToStringItemsCompact() {
Memory mem;
String s;
- println("#### CASE: FLOAT FULL COMPACT");
+ println("#### CASE: ITEM FULL COMPACT");
sk = KllItemsSketch.newHeapInstance(k, Comparator.naturalOrder(), serDe);
for (int i = 1; i <= n; i++) { sk.update(Util.longToFixedLengthString(i, digits)); }
compBytes = sk.toByteArray();
@@ -447,7 +447,7 @@ public void checkMemoryToStringItemsCompact() {
println(s);
assertEquals(compBytes, compBytes2);
- println("#### CASE: FLOAT EMPTY COMPACT");
+ println("#### CASE: ITEM EMPTY COMPACT");
sk = KllItemsSketch.newHeapInstance(k, Comparator.naturalOrder(), serDe);
compBytes = sk.toByteArray();
mem = Memory.wrap(compBytes);
@@ -462,7 +462,7 @@ public void checkMemoryToStringItemsCompact() {
println(s);
assertEquals(compBytes, compBytes2);
- println("#### CASE: FLOAT SINGLE COMPACT");
+ println("#### CASE: ITEM SINGLE COMPACT");
sk = KllItemsSketch.newHeapInstance(k, Comparator.naturalOrder(), serDe);
sk.update("1");
compBytes = sk.toByteArray();
diff --git a/src/test/java/org/apache/datasketches/kll/KllMiscLongsTest.java b/src/test/java/org/apache/datasketches/kll/KllMiscLongsTest.java
new file mode 100644
index 000000000..018ad91db
--- /dev/null
+++ b/src/test/java/org/apache/datasketches/kll/KllMiscLongsTest.java
@@ -0,0 +1,790 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.kll;
+
+import org.apache.datasketches.common.SketchesArgumentException;
+import org.apache.datasketches.kll.KllDirectLongsSketch.KllDirectCompactLongsSketch;
+import org.apache.datasketches.memory.DefaultMemoryRequestServer;
+import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.memory.MemoryRequestServer;
+import org.apache.datasketches.memory.WritableMemory;
+import org.apache.datasketches.quantilescommon.LongsSortedView;
+import org.apache.datasketches.quantilescommon.LongsSortedViewIterator;
+import org.testng.annotations.Test;
+
+import static org.apache.datasketches.common.Util.LS;
+import static org.apache.datasketches.common.Util.bitAt;
+import static org.apache.datasketches.kll.KllSketch.SketchType.LONGS_SKETCH;
+import static org.apache.datasketches.quantilescommon.QuantileSearchCriteria.INCLUSIVE;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+/**
+ * @author Lee Rhodes
+ */
+public class KllMiscLongsTest {
+ private final MemoryRequestServer memReqSvr = new DefaultMemoryRequestServer();
+
+ @Test
+ public void checkSortedViewConstruction() {
+ final KllLongsSketch kll = KllLongsSketch.newHeapInstance(20);
+ for (int i = 1; i <= 20; i++) { kll.update(i); }
+ LongsSortedView fsv = kll.getSortedView();
+ long[] cumWeights = fsv.getCumulativeWeights();
+ long[] values = fsv.getQuantiles();
+ assertEquals(cumWeights.length, 20);
+ assertEquals(values.length, 20);
+ for (int i = 0; i < 20; i++) {
+ assertEquals(cumWeights[i], i + 1);
+ assertEquals(values[i], i + 1);
+ }
+ }
+
+ @Test //set static enablePrinting = true for visual checking
+ public void checkBounds() {
+ final KllLongsSketch kll = KllLongsSketch.newHeapInstance(); //default k = 200
+ for (int i = 0; i < 1000; i++) {
+ kll.update(i);
+ }
+ final double eps = kll.getNormalizedRankError(false);
+ final long est = kll.getQuantile(0.5);
+ final long ub = kll.getQuantileUpperBound(0.5);
+ final long lb = kll.getQuantileLowerBound(0.5);
+ assertEquals(ub, kll.getQuantile(.5 + eps));
+ assertEquals(lb, kll.getQuantile(0.5 - eps));
+ println("Ext : " + est);
+ println("UB : " + ub);
+ println("LB : " + lb);
+ final double rest = kll.getRank(est);
+ final double restUB = kll.getRankUpperBound(rest);
+ final double restLB = kll.getRankLowerBound(rest);
+ assertTrue(restUB - rest < (2 * eps));
+ assertTrue(rest - restLB < (2 * eps));
+ }
+
+ @Test(expectedExceptions = SketchesArgumentException.class)
+ public void checkHeapifyExceptions1() {
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance();
+ WritableMemory wmem = WritableMemory.writableWrap(sk.toByteArray());
+ wmem.putByte(6, (byte) 3); //corrupt with odd M
+ KllLongsSketch.heapify(wmem);
+ }
+
+ @Test(expectedExceptions = SketchesArgumentException.class)
+ public void checkHeapifyExceptions2() {
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance();
+ WritableMemory wmem = WritableMemory.writableWrap(sk.toByteArray());
+ wmem.putByte(0, (byte) 1); //corrupt preamble ints, should be 2
+ KllLongsSketch.heapify(wmem);
+ }
+
+ @Test(expectedExceptions = SketchesArgumentException.class)
+ public void checkHeapifyExceptions3() {
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance();
+ sk.update(1);
+ sk.update(2);
+ WritableMemory wmem = WritableMemory.writableWrap(sk.toByteArray());
+ wmem.putByte(0, (byte) 1); //corrupt preamble ints, should be 5
+ KllLongsSketch.heapify(wmem);
+ }
+
+ @Test(expectedExceptions = SketchesArgumentException.class)
+ public void checkHeapifyExceptions4() {
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance();
+ WritableMemory wmem = WritableMemory.writableWrap(sk.toByteArray());
+ wmem.putByte(1, (byte) 0); //corrupt SerVer, should be 1 or 2
+ KllLongsSketch.heapify(wmem);
+ }
+
+ @Test(expectedExceptions = SketchesArgumentException.class)
+ public void checkHeapifyExceptions5() {
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance();
+ WritableMemory wmem = WritableMemory.writableWrap(sk.toByteArray());
+ wmem.putByte(2, (byte) 0); //corrupt FamilyID, should be 15
+ KllLongsSketch.heapify(wmem);
+ }
+
+ @Test //set static enablePrinting = true for visual checking
+ public void checkMisc() {
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance(8);
+ try { sk.getMaxItem(); fail(); } catch (SketchesArgumentException e) {} //empty
+ println(sk.toString(true, true));
+ for (int i = 0; i < 20; i++) { sk.update(i); }
+ println(sk.toString(true, true));
+ sk.toByteArray();
+ final long[] items = sk.getLongItemsArray();
+ assertEquals(items.length, 16);
+ final int[] levels = sk.getLevelsArray(sk.sketchStructure);
+ assertEquals(levels.length, 3);
+ assertEquals(sk.getNumLevels(), 2);
+ }
+
+ @Test //set static enablePrinting = true for visual checking
+ public void visualCheckToString() {
+ final KllLongsSketch sk = KllLongsSketch.newHeapInstance(20);
+ int n = 21;
+ for (int i = 1; i <= n; i++) { sk.update(i); }
+ println(sk.toString(true, true));
+ assertEquals(sk.getNumLevels(), 2);
+ assertEquals(sk.getMinItem(), 1);
+ assertEquals(sk.getMaxItem(), 21);
+ assertEquals(sk.getNumRetained(), 11);
+
+ final KllLongsSketch sk2 = KllLongsSketch.newHeapInstance(20);
+ n = 400;
+ for (int i = 101; i <= n + 100; i++) { sk2.update(i); }
+ println(LS + sk2.toString(true, true));
+ assertEquals(sk2.getNumLevels(), 5);
+ assertEquals(sk2.getMinItem(), 101);
+ assertEquals(sk2.getMaxItem(), 500);
+ assertEquals(sk2.getNumRetained(), 52);
+
+ sk2.merge(sk);
+ println(LS + sk2.toString(true, true));
+ assertEquals(sk2.getNumLevels(), 5);
+ assertEquals(sk2.getMinItem(), 1);
+ assertEquals(sk2.getMaxItem(), 500);
+ assertEquals(sk2.getNumRetained(), 56);
+ }
+
+ @Test //set static enablePrinting = true for visual checking
+ public void viewHeapCompactions() {
+ int k = 20;
+ int n = 108;
+ boolean withLevels = false;
+ boolean withLevelsAndItems = true;
+ int compaction = 0;
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance(k);
+ for (int i = 1; i <= n; i++) {
+ sk.update(i);
+ if (sk.levelsArr[0] == 0) {
+ println(LS + "#<<< BEFORE COMPACTION # " + (++compaction) + " >>>");
+ println(sk.toString(withLevels, withLevelsAndItems));
+ sk.update(++i);
+ println(LS + "#<<< AFTER COMPACTION # " + (compaction) + " >>>");
+ println(sk.toString(withLevels, withLevelsAndItems));
+ assertEquals(sk.getLongItemsArray()[sk.levelsArr[0]], i);
+ }
+ }
+ println(LS + "#<<< END STATE # >>>");
+ println(sk.toString(withLevels, withLevelsAndItems));
+ println("");
+ }
+
+ @Test //set static enablePrinting = true for visual checking
+ public void viewDirectCompactions() {
+ int k = 20;
+ int n = 108;
+ boolean withLevels = false;
+ boolean withLevelsAndItems = true;
+ int compaction = 0;
+ int sizeBytes = KllSketch.getMaxSerializedSizeBytes(k, n, LONGS_SKETCH, true);
+ WritableMemory wmem = WritableMemory.allocate(sizeBytes);
+ KllLongsSketch sk = KllLongsSketch.newDirectInstance(k, wmem, memReqSvr);
+ for (int i = 1; i <= n; i++) {
+ sk.update(i);
+ if (sk.levelsArr[0] == 0) {
+ println(LS + "#<<< BEFORE COMPACTION # " + (++compaction) + " >>>");
+ println(sk.toString(withLevels, withLevelsAndItems));
+ sk.update(++i);
+ println(LS + "#<<< AFTER COMPACTION # " + (compaction) + " >>>");
+ println(sk.toString(withLevels, withLevelsAndItems));
+ assertEquals(sk.getLongItemsArray()[sk.levelsArr[0]], i);
+ }
+ }
+ println(LS + "#<<< END STATE # >>>");
+ println(sk.toString(withLevels, withLevelsAndItems));
+ println("");
+ }
+
+ @Test //set static enablePrinting = true for visual checking
+ public void viewCompactionAndSortedView() {
+ int n = 43;
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance(20);
+ for (int i = 1; i <= n; i++) { sk.update(i); }
+ println(sk.toString(true, true));
+ LongsSortedView sv = sk.getSortedView();
+ LongsSortedViewIterator itr = sv.iterator();
+ println("### SORTED VIEW");
+ printf("%6s %12s %12s" + LS, "Idx", "Value", "Weight");
+ int i = 0;
+ while (itr.next()) {
+ long v = itr.getQuantile();
+ long wt = itr.getWeight();
+ printf("%6d %12d %12d" + LS, i, v, wt);
+ i++;
+ }
+ assertEquals(sv.getMinItem(), 1L);
+ assertEquals(sv.getMaxItem(), n);
+ }
+
+ @Test //set static enablePrinting = true for visual checking
+ public void checkWeightedUpdates1() {
+ int k = 20;
+ int weight = 127;
+ long item = 10;
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance(k);
+ println(sk.toString(true, true));
+ sk.update(item, weight);
+ println(sk.toString(true, true));
+ assertEquals(sk.getNumRetained(), 7);
+ assertEquals(sk.getN(), weight);
+ sk.update(item, weight);
+ println(sk.toString(true, true));
+ assertEquals(sk.getNumRetained(), 14);
+ assertEquals(sk.getN(), 254);
+ }
+
+ @Test //set static enablePrinting = true for visual checking
+ public void checkWeightedUpdates2() {
+ int k = 20;
+ int initial = 1000;
+ int weight = 127;
+ long item = 10;
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance(k);
+ for (int i = 1; i <= initial; i++) { sk.update(i + 1000); }
+ println(sk.toString(true, true));
+ sk.update(item, weight);
+ println(sk.toString(true, true));
+ assertEquals(sk.getNumRetained(), 65);
+ assertEquals(sk.getN(), 1127);
+
+ LongsSortedViewIterator itr = sk.getSortedView().iterator();
+ println("### SORTED VIEW");
+ printf("%12s %12s %12s" + LS, "Value", "Weight", "NaturalRank");
+ long cumWt = 0;
+ while (itr.next()) {
+ long v = itr.getQuantile();
+ long wt = itr.getWeight();
+ long natRank = itr.getNaturalRank(INCLUSIVE);
+ cumWt += wt;
+ assertEquals(cumWt, natRank);
+ printf("%12d %12d %12d" + LS, v, wt, natRank);
+ }
+ assertEquals(cumWt, sk.getN());
+ }
+
+ @Test //set static enablePrinting = true for visual checking
+ public void checkCreateItemsArray() { //used with weighted updates
+ long item = 10;
+ int weight = 108;
+ long[] itemsArr = KllLongsHelper.createItemsArray(item, weight);
+ assertEquals(itemsArr.length, 4);
+ for (int i = 0; i < itemsArr.length; i++) { itemsArr[i] = item; }
+ outputItems(itemsArr);
+ }
+
+ private static void outputItems(long[] itemsArr) {
+ String[] hdr2 = {"Index", "Value"};
+ String hdr2fmt = "%6s %15s" + LS;
+ String d2fmt = "%6d %15d" + LS;
+ println("ItemsArr");
+ printf(hdr2fmt, (Object[]) hdr2);
+ for (int i = 0; i < itemsArr.length; i++) {
+ printf(d2fmt, i, itemsArr[i]);
+ }
+ println("");
+ }
+
+ @Test //set static enablePrinting = true for visual checking
+ public void checkCreateLevelsArray() { //used with weighted updates
+ int weight = 108;
+ int[] levelsArr = KllHelper.createLevelsArray(weight);
+ assertEquals(levelsArr.length, 8);
+ int[] correct = {0,0,0,1,2,2,3,4};
+ for (int i = 0; i < levelsArr.length; i++) {
+ assertEquals(levelsArr[i], correct[i]);
+ }
+ outputLevels(weight, levelsArr);
+ }
+
+ private static void outputLevels(int weight, int[] levelsArr) {
+ String[] hdr = {"Lvl", "StartAdr", "BitPattern", "Weight"};
+ String hdrfmt = "%3s %9s %10s %s" + LS;
+ String dfmt = "%3d %9d %10d %d" + LS;
+ String dfmt_2 = "%3d %9d %s" + LS;
+ println("Count = " + weight + " => " + (Integer.toBinaryString(weight)));
+ println("LevelsArr");
+ printf(hdrfmt, (Object[]) hdr);
+ for (int i = 0; i < levelsArr.length; i++) {
+ if (i == levelsArr.length - 1) { printf(dfmt_2, i, levelsArr[i], "ItemsArr.length"); }
+ else {
+ int j = bitAt(weight, i);
+ printf(dfmt, i, levelsArr[i], j, 1 << (i));
+ }
+ }
+ println("");
+ }
+
+ @Test
+ public void viewMemorySketchData() {
+ int k = 20;
+ int n = 109;
+ boolean withLevels = true;
+ boolean withLevelsAndItems = true;
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance(k);
+ for (int i = 1; i <= n; i++) { sk.update(i); }
+ byte[] byteArr = sk.toByteArray();
+ Memory mem = Memory.wrap(byteArr);
+ KllLongsSketch fltSk = KllLongsSketch.wrap(mem);
+ println(fltSk.toString(withLevels, withLevelsAndItems));
+ assertEquals(fltSk.getN(), n);
+ }
+
+ @Test //set static enablePrinting = true for visual checking
+ public void checkIntCapAux() {
+ String[] hdr = {"level", "depth", "wt", "cap", "(end)", "MaxN"};
+ String hdrFmt = "%6s %6s %28s %10s %10s %34s" + LS;
+ String dataFmt = "%6d %6d %,28d %,10d %,10d %,34d" + LS;
+ int k = 1000;
+ int m = 8;
+ int numLevels = 20;
+ println("k=" + k + ", m=" + m + ", numLevels=" + numLevels);
+ printf(hdrFmt, (Object[]) hdr);
+ long maxN = 0;
+ long[] correct = {0,1,1,2,2,3,5,8,12,17,26,39,59,88,132,198,296,444,667,1000};
+ for (int i = 0; i < numLevels; i++) {
+ int depth = numLevels - i - 1;
+ long cap = KllHelper.intCapAux(k, depth);
+ long end = Math.max(m, cap);
+ long wt = 1L << i;
+ maxN += wt * end;
+ printf(dataFmt, i, depth, wt, cap, end, maxN);
+ assertEquals(cap, correct[i]);
+ }
+ }
+
+ @Test //set static enablePrinting = true for visual checking
+ public void checkIntCapAuxAux() {
+ String[] hdr = {"d","twoK","2k*2^d","3^d","tmp=2k*2^d/3^d","(tmp + 1)/2", "(end)"};
+ String hdrFmt = "%6s %10s %20s %20s %15s %12s %10s" + LS;
+ String dataFmt = "%6d %10d %,20d %,20d %15d %12d %10d" + LS;
+ long k = (1L << 16) - 1L;
+ long m = 8;
+ println("k = " + k + ", m = " + m);
+ printf(hdrFmt, (Object[]) hdr);
+ long[] correct =
+ {65535,43690,29127,19418,12945,8630,5753,3836,2557,1705,1136,758,505,337,224,150,100,67,44,30,20,13,9,6,4,3,2,1,1,1,0};
+ for (int i = 0; i < 31; i++) {
+ long twoK = k << 1;
+ long twoKxtwoD = twoK << i;
+ long threeToD = KllHelper.powersOfThree[i];
+ long tmp = twoKxtwoD / threeToD;
+ long result = (tmp + 1L) >>> 1;
+ long end = Math.max(m, result); //performed later
+ printf(dataFmt, i, twoK, twoKxtwoD, threeToD, tmp, result, end);
+ assertEquals(result,correct[i]);
+ assertEquals(result, KllHelper.intCapAuxAux(k, i));
+ }
+ }
+
+ @Test
+ public void checkGrowLevels() {
+ KllLongsSketch sk = KllLongsSketch.newHeapInstance(20);
+ for (int i = 1; i <= 21; i++) { sk.update(i); }
+ assertEquals(sk.getNumLevels(), 2);
+ assertEquals(sk.getLongItemsArray().length, 33);
+ assertEquals(sk.getLevelsArray(sk.sketchStructure)[2], 33);
+ }
+
+ @Test //set static enablePrinting = true for visual checking
+ public void checkSketchInitializeLongHeap() {
+ int k = 20; //don't change this
+ KllLongsSketch sk;
+
+ println("#### CASE: LONG FULL HEAP");
+ sk = KllLongsSketch.newHeapInstance(k);
+ for (int i = 1; i <= k + 1; i++) { sk.update(i); }
+ println(sk.toString(true, true));
+ assertEquals(sk.getK(), k);
+ assertEquals(sk.getN(), k + 1);
+ assertEquals(sk.getNumRetained(), 11);
+ assertFalse(sk.isEmpty());
+ assertTrue(sk.isEstimationMode());
+ assertEquals(sk.getMinK(), k);
+ assertEquals(sk.getLongItemsArray().length, 33);
+ assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 3);
+ assertEquals(sk.getMaxItem(), 21L);
+ assertEquals(sk.getMinItem(), 1L);
+ assertEquals(sk.getNumLevels(), 2);
+ assertFalse(sk.isLevelZeroSorted());
+
+ println("#### CASE: LONG HEAP EMPTY");
+ sk = KllLongsSketch.newHeapInstance(k);
+ println(sk.toString(true, true));
+ assertEquals(sk.getK(), k);
+ assertEquals(sk.getN(), 0);
+ assertEquals(sk.getNumRetained(), 0);
+ assertTrue(sk.isEmpty());
+ assertFalse(sk.isEstimationMode());
+ assertEquals(sk.getMinK(), k);
+ assertEquals(sk.getLongItemsArray().length, 20);
+ assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 2);
+ try { sk.getMaxItem(); fail(); } catch (SketchesArgumentException e) { }
+ try { sk.getMinItem(); fail(); } catch (SketchesArgumentException e) { }
+ assertEquals(sk.getNumLevels(), 1);
+ assertFalse(sk.isLevelZeroSorted());
+
+ println("#### CASE: LONG HEAP SINGLE");
+ sk = KllLongsSketch.newHeapInstance(k);
+ sk.update(1);
+ println(sk.toString(true, true));
+ assertEquals(sk.getK(), k);
+ assertEquals(sk.getN(), 1);
+ assertEquals(sk.getNumRetained(), 1);
+ assertFalse(sk.isEmpty());
+ assertFalse(sk.isEstimationMode());
+ assertEquals(sk.getMinK(), k);
+ assertEquals(sk.getLongItemsArray().length, 20);
+ assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 2);
+ assertEquals(sk.getMaxItem(), 1L);
+ assertEquals(sk.getMinItem(), 1L);
+ assertEquals(sk.getNumLevels(), 1);
+ assertFalse(sk.isLevelZeroSorted());
+ }
+
+ @Test //set static enablePrinting = true for visual checking
+ public void checkSketchInitializeLongHeapifyCompactMem() {
+ int k = 20; //don't change this
+ KllLongsSketch sk;
+ KllLongsSketch sk2;
+ byte[] compBytes;
+ WritableMemory wmem;
+
+ println("#### CASE: LONG FULL HEAPIFIED FROM COMPACT");
+ sk2 = KllLongsSketch.newHeapInstance(k);
+ for (int i = 1; i <= k + 1; i++) { sk2.update(i); }
+ //println(sk2.toString(true, true));
+ compBytes = sk2.toByteArray();
+ wmem = WritableMemory.writableWrap(compBytes);
+ println(KllPreambleUtil.toString(wmem, LONGS_SKETCH, true));
+ sk = KllLongsSketch.heapify(wmem);
+ assertEquals(sk.getK(), k);
+ assertEquals(sk.getN(), k + 1);
+ assertEquals(sk.getNumRetained(), 11);
+ assertFalse(sk.isEmpty());
+ assertTrue(sk.isEstimationMode());
+ assertEquals(sk.getMinK(), k);
+ assertEquals(sk.getLongItemsArray().length, 33);
+ assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 3);
+ assertEquals(sk.getMaxItem(), 21);
+ assertEquals(sk.getMinItem(), 1);
+ assertEquals(sk.getNumLevels(), 2);
+ assertFalse(sk.isLevelZeroSorted());
+
+ println("#### CASE: LONG EMPTY HEAPIFIED FROM COMPACT");
+ sk2 = KllLongsSketch.newHeapInstance(k);
+ //println(sk.toString(true, true));
+ compBytes = sk2.toByteArray();
+ wmem = WritableMemory.writableWrap(compBytes);
+ //println(KllPreambleUtil.toString(wmem, LONGS_SKETCH, true));
+ sk = KllLongsSketch.heapify(wmem);
+ assertEquals(sk.getK(), k);
+ assertEquals(sk.getN(), 0);
+ assertEquals(sk.getNumRetained(), 0);
+ assertTrue(sk.isEmpty());
+ assertFalse(sk.isEstimationMode());
+ assertEquals(sk.getMinK(), k);
+ assertEquals(sk.getLongItemsArray().length, 20);
+ assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 2);
+ try { sk.getMaxItem(); fail(); } catch (SketchesArgumentException e) { }
+ try { sk.getMinItem(); fail(); } catch (SketchesArgumentException e) { }
+ assertEquals(sk.getNumLevels(), 1);
+ assertFalse(sk.isLevelZeroSorted());
+
+ println("#### CASE: LONG SINGLE HEAPIFIED FROM COMPACT");
+ sk2 = KllLongsSketch.newHeapInstance(k);
+ sk2.update(1);
+ //println(sk2.toString(true, true));
+ compBytes = sk2.toByteArray();
+ wmem = WritableMemory.writableWrap(compBytes);
+ //println(KllPreambleUtil.toString(wmem, LONGS_SKETCH, true));
+ sk = KllLongsSketch.heapify(wmem);
+ assertEquals(sk.getK(), k);
+ assertEquals(sk.getN(), 1);
+ assertEquals(sk.getNumRetained(), 1);
+ assertFalse(sk.isEmpty());
+ assertFalse(sk.isEstimationMode());
+ assertEquals(sk.getMinK(), k);
+ assertEquals(sk.getLongItemsArray().length, 20);
+ assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 2);
+ assertEquals(sk.getMaxItem(), 1L);
+ assertEquals(sk.getMinItem(), 1L);
+ assertEquals(sk.getNumLevels(), 1);
+ assertFalse(sk.isLevelZeroSorted());
+ }
+
+ @Test //set static enablePrinting = true for visual checking
+ public void checkSketchInitializeLongHeapifyUpdatableMem() {
+ int k = 20; //don't change this
+ KllLongsSketch sk;
+ KllLongsSketch sk2;
+ byte[] compBytes;
+ WritableMemory wmem;
+
+ println("#### CASE: LONG FULL HEAPIFIED FROM UPDATABLE");
+ sk2 = KllLongsSketch.newHeapInstance(k);
+ for (int i = 1; i <= k + 1; i++) { sk2.update(i); }
+ compBytes = KllHelper.toByteArray(sk2, true);
+ wmem = WritableMemory.writableWrap(compBytes);
+ println(KllPreambleUtil.toString(wmem, LONGS_SKETCH, true));
+ sk = KllHeapLongsSketch.heapifyImpl(wmem);
+ assertEquals(sk.getK(), k);
+ assertEquals(sk.getN(), k + 1);
+ assertEquals(sk.getNumRetained(), 11);
+ assertFalse(sk.isEmpty());
+ assertTrue(sk.isEstimationMode());
+ assertEquals(sk.getMinK(), k);
+ assertEquals(sk.getLongItemsArray().length, 33);
+ assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 3);
+ assertEquals(sk.getMaxItem(), 21);
+ assertEquals(sk.getMinItem(), 1);
+ assertEquals(sk.getNumLevels(), 2);
+ assertFalse(sk.isLevelZeroSorted());
+
+ println("#### CASE: LONG EMPTY HEAPIFIED FROM UPDATABLE");
+ sk2 = KllLongsSketch.newHeapInstance(k);
+ //println(sk.toString(true, true));
+ compBytes = KllHelper.toByteArray(sk2, true);
+ wmem = WritableMemory.writableWrap(compBytes);
+ //println(KllPreambleUtil.toString(wmem, LONGS_SKETCH, true));
+ sk = KllHeapLongsSketch.heapifyImpl(wmem);
+ assertEquals(sk.getK(), k);
+ assertEquals(sk.getN(), 0);
+ assertEquals(sk.getNumRetained(), 0);
+ assertTrue(sk.isEmpty());
+ assertFalse(sk.isEstimationMode());
+ assertEquals(sk.getMinK(), k);
+ assertEquals(sk.getLongItemsArray().length, 20);
+ assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 2);
+ try { sk.getMaxItem(); fail(); } catch (SketchesArgumentException e) { }
+ try { sk.getMinItem(); fail(); } catch (SketchesArgumentException e) { }
+ assertEquals(sk.getNumLevels(), 1);
+ assertFalse(sk.isLevelZeroSorted());
+
+ println("#### CASE: LONG SINGLE HEAPIFIED FROM UPDATABLE");
+ sk2 = KllLongsSketch.newHeapInstance(k);
+ sk2.update(1);
+ //println(sk.toString(true, true));
+ compBytes = KllHelper.toByteArray(sk2, true);
+ wmem = WritableMemory.writableWrap(compBytes);
+ //println(KllPreambleUtil.toString(wmem, LONGS_SKETCH, true));
+ sk = KllHeapLongsSketch.heapifyImpl(wmem);
+ assertEquals(sk.getK(), k);
+ assertEquals(sk.getN(), 1);
+ assertEquals(sk.getNumRetained(), 1);
+ assertFalse(sk.isEmpty());
+ assertFalse(sk.isEstimationMode());
+ assertEquals(sk.getMinK(), k);
+ assertEquals(sk.getLongItemsArray().length, 20);
+ assertEquals(sk.getLevelsArray(sk.sketchStructure).length, 2);
+ assertEquals(sk.getMaxItem(), 1L);
+ assertEquals(sk.getMinItem(), 1L);
+ assertEquals(sk.getNumLevels(), 1);
+ assertFalse(sk.isLevelZeroSorted());
+ }
+
+ @Test //set static enablePrinting = true for visual checking
+ public void checkMemoryToStringLongCompact() {
+ int k = 20; //don't change this
+ KllLongsSketch sk;
+ KllLongsSketch sk2;
+ byte[] compBytes;
+ byte[] compBytes2;
+ WritableMemory wmem;
+ String s;
+
+ println("#### CASE: LONG FULL COMPACT");
+ sk = KllLongsSketch.newHeapInstance(k);
+ for (int i = 1; i <= k + 1; i++) { sk.update(i); }
+ compBytes = sk.toByteArray();
+ wmem = WritableMemory.writableWrap(compBytes);
+ s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true);
+ println("step 1: sketch to byte[]/memory & analyze memory");
+ println(s);
+ sk2 = KllLongsSketch.heapify(wmem);
+ compBytes2 = sk2.toByteArray();
+ wmem = WritableMemory.writableWrap(compBytes2);
+ s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true);
+ println("step 2: memory to heap sketch, to byte[]/memory & analyze memory. Should match above");
+ println(s);
+ assertEquals(compBytes, compBytes2);
+
+ println("#### CASE: LONG EMPTY COMPACT");
+ sk = KllLongsSketch.newHeapInstance(k);
+ compBytes = sk.toByteArray();
+ wmem = WritableMemory.writableWrap(compBytes);
+ s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true);
+ println("step 1: sketch to byte[]/memory & analyze memory");
+ println(s);
+ sk2 = KllLongsSketch.heapify(wmem);
+ compBytes2 = sk2.toByteArray();
+ wmem = WritableMemory.writableWrap(compBytes2);
+ s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true);
+ println("step 2: memory to heap sketch, to byte[]/memory & analyze memory. Should match above");
+ println(s);
+ assertEquals(compBytes, compBytes2);
+
+ println("#### CASE: LONG SINGLE COMPACT");
+ sk = KllLongsSketch.newHeapInstance(k);
+ sk.update(1);
+ compBytes = sk.toByteArray();
+ wmem = WritableMemory.writableWrap(compBytes);
+ s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true);
+ println("step 1: sketch to byte[]/memory & analyze memory");
+ println(s);
+ sk2 = KllLongsSketch.heapify(wmem);
+ compBytes2 = sk2.toByteArray();
+ wmem = WritableMemory.writableWrap(compBytes2);
+ s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true);
+ println("step 2: memory to heap sketch, to byte[]/memory & analyze memory. Should match above");
+ println(s);
+ assertEquals(compBytes, compBytes2);
+ }
+
+ @Test //set static enablePrinting = true for visual checking
+ public void checkMemoryToStringLongUpdatable() {
+ int k = 20; //don't change this
+ KllLongsSketch sk;
+ KllLongsSketch sk2;
+ byte[] upBytes;
+ byte[] upBytes2;
+ WritableMemory wmem;
+ String s;
+
+ println("#### CASE: LONG FULL UPDATABLE");
+ sk = KllLongsSketch.newHeapInstance(20);
+ for (int i = 1; i <= k + 1; i++) { sk.update(i); }
+ upBytes = KllHelper.toByteArray(sk, true);
+ wmem = WritableMemory.writableWrap(upBytes);
+ s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true);
+ println("step 1: sketch to byte[]/memory & analyze memory");
+ println(s);
+ sk2 = KllHeapLongsSketch.heapifyImpl(wmem);
+ upBytes2 = KllHelper.toByteArray(sk2, true);
+ wmem = WritableMemory.writableWrap(upBytes2);
+ s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true);
+ println("step 2: memory to heap sketch, to byte[]/memory & analyze memory. Should match above");
+ println(s); //note: heapify does not copy free space, while toUpdatableByteArray does
+ assertEquals(sk.getN(), sk2.getN());
+ assertEquals(sk.getMinItem(), sk2.getMinItem());
+ assertEquals(sk.getMaxItem(), sk2.getMaxItem());
+ assertEquals(sk.getNumRetained(), sk2.getNumRetained());
+
+ println("#### CASE: LONG EMPTY UPDATABLE");
+ sk = KllLongsSketch.newHeapInstance(k);
+ upBytes = KllHelper.toByteArray(sk, true);
+ wmem = WritableMemory.writableWrap(upBytes);
+ s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true);
+ println("step 1: sketch to byte[]/memory & analyze memory");
+ println(s);
+ sk2 = KllHeapLongsSketch.heapifyImpl(wmem);
+ upBytes2 = KllHelper.toByteArray(sk2, true);
+ wmem = WritableMemory.writableWrap(upBytes2);
+ s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true);
+ println("step 2: memory to heap sketch, to byte[]/memory & analyze memory. Should match above");
+ println(s);
+ assertEquals(upBytes, upBytes2);
+
+ println("#### CASE: LONG SINGLE UPDATABLE");
+ sk = KllLongsSketch.newHeapInstance(k);
+ sk.update(1);
+ upBytes = KllHelper.toByteArray(sk, true);
+ wmem = WritableMemory.writableWrap(upBytes);
+ s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true);
+ println("step 1: sketch to byte[]/memory & analyze memory");
+ println(s);
+ sk2 = KllHeapLongsSketch.heapifyImpl(wmem);
+ upBytes2 = KllHelper.toByteArray(sk2, true);
+ wmem = WritableMemory.writableWrap(upBytes2);
+ s = KllPreambleUtil.toString(wmem, LONGS_SKETCH, true);
+ println("step 2: memory to heap sketch, to byte[]/memory & analyze memory. Should match above");
+ println(s);
+ assertEquals(upBytes, upBytes2);
+ }
+
+ @Test
+ public void checkSimpleMerge() {
+ int k = 20;
+ int m = 8;
+ int n1 = 21;
+ int n2 = 43;
+ WritableMemory wmem = WritableMemory.allocate(3000);
+ WritableMemory wmem2 = WritableMemory.allocate(3000);
+
+ KllLongsSketch sk1 = KllDirectLongsSketch.newDirectUpdatableInstance(k, m, wmem, memReqSvr);
+ KllLongsSketch sk2 = KllDirectLongsSketch.newDirectUpdatableInstance(k, m, wmem2, memReqSvr);
+ for (int i = 1; i <= n1; i++) {
+ sk1.update(i);
+ }
+ for (int i = 1; i <= n2; i++) {
+ sk2.update(i + 100);
+ }
+ sk1.merge(sk2);
+ assertEquals(sk1.getMinItem(), 1L);
+ assertEquals(sk1.getMaxItem(), 143L);
+ }
+
+ @Test
+ public void checkGetSingleItem() {
+ int k = 20;
+ KllLongsSketch skHeap = KllLongsSketch.newHeapInstance(k);
+ skHeap.update(1);
+ assertTrue(skHeap instanceof KllHeapLongsSketch);
+ assertEquals(skHeap.getLongSingleItem(), 1L);
+
+ WritableMemory srcMem = WritableMemory.writableWrap(KllHelper.toByteArray(skHeap, true));
+ KllLongsSketch skDirect = KllLongsSketch.writableWrap(srcMem, memReqSvr);
+ assertTrue(skDirect instanceof KllDirectLongsSketch);
+ assertEquals(skDirect.getLongSingleItem(), 1L);
+
+ Memory srcMem2 = Memory.wrap(skHeap.toByteArray());
+ KllLongsSketch skCompact = KllLongsSketch.wrap(srcMem2);
+ assertTrue(skCompact instanceof KllDirectCompactLongsSketch);
+ assertEquals(skCompact.getLongSingleItem(), 1L);
+ }
+
+ @Test
+ public void printlnTest() {
+ String s = "PRINTING: printf in " + this.getClass().getName();
+ println(s);
+ printf("%s" + LS, s);
+ }
+
+ private final static boolean enablePrinting = false;
+
+ /**
+ * @param format the format
+ * @param args the args
+ */
+ private static final void printf(final String format, final Object ... args) {
+ if (enablePrinting) { System.out.printf(format, args); }
+ }
+
+ /**
+ * @param o the Object to println
+ */
+ private static final void println(final Object o) {
+ if (enablePrinting) { System.out.println(o.toString()); }
+ }
+
+}
diff --git a/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java b/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java
index bbf89f338..2b67f9772 100644
--- a/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java
+++ b/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java
@@ -26,7 +26,10 @@
import java.util.HashSet;
import org.testng.annotations.Test;
-import org.apache.datasketches.memory.WritableHandle;
+
+import java.lang.foreign.Arena;
+import java.nio.ByteOrder;
+
import org.apache.datasketches.memory.WritableMemory;
import org.apache.datasketches.quantilescommon.QuantilesDoublesSketchIterator;
@@ -61,8 +64,8 @@ public void test() {
DoublesSketch.setRandom(1); //make deterministic for test
DoublesUnion dUnion;
DoublesSketch dSketch;
- try ( WritableHandle wdh = WritableMemory.allocateDirect(10_000_000) ) {
- WritableMemory wmem = wdh.getWritable();
+ try (Arena arena = Arena.ofConfined()) {
+ WritableMemory wmem = WritableMemory.allocateDirect(10_000_000, arena);
dUnion = DoublesUnion.builder().setMaxK(8).build(wmem);
for (int s = 0; s < numSketches; s++) { dUnion.union(sketchArr[s]); }
dSketch = dUnion.getResult(); //result is on heap
diff --git a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java
index 5c2882c5e..b61171ca1 100644
--- a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java
+++ b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java
@@ -24,13 +24,13 @@
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertTrue;
+import java.lang.foreign.Arena;
import java.nio.ByteOrder;
import org.apache.datasketches.memory.DefaultMemoryRequestServer;
import org.testng.annotations.Test;
import org.apache.datasketches.memory.Memory;
-import org.apache.datasketches.memory.WritableHandle;
import org.apache.datasketches.memory.WritableMemory;
/**
@@ -44,38 +44,35 @@ public class DirectQuantilesMemoryRequestTest {
public void checkLimitedMemoryScenarios() { //Requesting application
final int k = 128;
final int u = 40 * k;
- final int initBytes = ((2 * k) + 4) << 3; //just the BB
+ final int initBytes = ((2 * k) + 4) << 3; //just the BaseBuffer
//########## Owning Implementation
- // This part would actually be part of the Memory owning implemention so it is faked here
- try (WritableHandle wdh = WritableMemory.allocateDirect(initBytes,
- ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())) {
- final WritableMemory wmem = wdh.getWritable();
- println("Initial mem size: " + wmem.getCapacity());
-
- //########## Receiving Application
- // The receiving application has been given wmem to use for a sketch,
- // but alas, it is not ultimately large enough.
- final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem);
- assertTrue(usk1.isEmpty());
-
- //Load the sketch
- for (int i = 0; i < u; i++) {
- // The sketch uses The MemoryRequest, acquired from wmem, to acquire more memory as
- // needed, and requests via the MemoryRequest to free the old allocations.
- usk1.update(i);
- }
- final double result = usk1.getQuantile(0.5);
- println("Result: " + result);
- assertEquals(result, u / 2.0, 0.05 * u); //Success
-
- //########## Owning Implementation
- //The actual Memory has been re-allocated several times,
- // so the above wmem reference is invalid.
- println("\nFinal mem size: " + wmem.getCapacity());
- } catch (Exception e) {
- throw new RuntimeException(e);
+ // This part would actually be part of the Memory owning implementation so it is faked here
+ WritableMemory wmem = WritableMemory.allocateDirect(initBytes, Arena.ofConfined());
+ WritableMemory wmem2 = wmem;
+ println("Initial mem size: " + wmem.getCapacity());
+
+ //########## Receiving Application
+ // The receiving application has been given wmem to use for a sketch,
+ // but alas, it is not ultimately large enough.
+ final UpdateDoublesSketch usk = DoublesSketch.builder().setK(k).build(wmem);
+ assertTrue(usk.isEmpty());
+
+ //Load the sketch
+ for (int i = 0; i < u; i++) {
+ // The sketch uses The MemoryRequest, acquired from wmem, to acquire more memory as
+ // needed, and requests via the MemoryRequest to free the old allocations.
+ usk.update(i);
}
+ final double result = usk.getQuantile(0.5);
+ println("Result: " + result);
+ assertEquals(result, u / 2.0, 0.05 * u); //Success
+
+ //########## Owning Implementation
+ //The actual Memory has been re-allocated several times,
+ // so the above wmem reference is invalid.
+ println("\nFinal mem size: " + wmem.getCapacity());
+ assertFalse(wmem2.isAlive());
}
@Test
@@ -84,22 +81,17 @@ public void checkGrowBaseBuf() {
final int u = 32; // don't need the BB to fill here
final int initBytes = (4 + (u / 2)) << 3; // not enough to hold everything
- try (WritableHandle memHandler = WritableMemory.allocateDirect(initBytes,
- ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())) {
- //final MemoryManager memMgr = new MemoryManager();
- //final WritableMemory mem1 = memMgr.request(initBytes);
- final WritableMemory mem1 = memHandler.getWritable();
- println("Initial mem size: " + mem1.getCapacity());
- final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(mem1);
- for (int i = 1; i <= u; i++) {
- usk1.update(i);
- }
- final int currentSpace = usk1.getCombinedBufferItemCapacity();
- println("curCombBufItemCap: " + currentSpace);
- assertEquals(currentSpace, 2 * k);
- } catch (final Exception e) {
- throw new RuntimeException(e);
+ WritableMemory wmem = WritableMemory.allocateDirect(initBytes, Arena.ofConfined());
+ WritableMemory wmem2 = wmem;
+ println("Initial mem size: " + wmem.getCapacity());
+ final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem);
+ for (int i = 1; i <= u; i++) {
+ usk1.update(i);
}
+ final int currentSpace = usk1.getCombinedBufferItemCapacity();
+ println("curCombBufItemCap: " + currentSpace);
+ assertEquals(currentSpace, 2 * k);
+ assertFalse(wmem2.isAlive());
}
@Test
@@ -108,58 +100,63 @@ public void checkGrowCombBuf() {
final int u = (2 * k) - 1; //just to fill the BB
final int initBytes = ((2 * k) + 4) << 3; //just room for BB
- try (WritableHandle memHandler = WritableMemory.allocateDirect(initBytes,
- ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())) {
- //final MemoryManager memMgr = new MemoryManager();
- //final WritableMemory mem1 = memMgr.request(initBytes);
- final WritableMemory mem1 = memHandler.getWritable();
- println("Initial mem size: " + mem1.getCapacity());
- final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(mem1);
- for (int i = 1; i <= u; i++) {
- usk1.update(i);
- }
- final int currentSpace = usk1.getCombinedBufferItemCapacity();
- println("curCombBufItemCap: " + currentSpace);
- final double[] newCB = usk1.growCombinedBuffer(currentSpace, 3 * k);
- final int newSpace = usk1.getCombinedBufferItemCapacity();
- println("newCombBurItemCap: " + newSpace);
- assertEquals(newCB.length, 3 * k);
- //memMgr.free(mem1);
- } catch (final Exception e) {
- throw new RuntimeException(e);
+ WritableMemory wmem = WritableMemory.allocateDirect(initBytes, Arena.ofConfined());
+ WritableMemory wmem2 = wmem;
+ println("Initial mem size: " + wmem.getCapacity());
+ final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem);
+ for (int i = 1; i <= u; i++) {
+ usk1.update(i);
}
+ final int currentSpace = usk1.getCombinedBufferItemCapacity();
+ println("curCombBufItemCap: " + currentSpace);
+ final double[] newCB = usk1.growCombinedBuffer(currentSpace, 3 * k);
+ final int newSpace = usk1.getCombinedBufferItemCapacity();
+ println("newCombBurItemCap: " + newSpace);
+ assertEquals(newCB.length, 3 * k);
+ assertFalse(wmem2.isAlive());
}
+ @Test
+ public void checkUpdatableStorageBytes() {
+ final int k = 16;
+ final int initBytes = DoublesSketch.getUpdatableStorageBytes(k, 1);
+ println("Predicted Updatable Storage Bytes: " + initBytes);
+ final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build();
+ usk1.update(1.0);
+ byte[] uarr = usk1.toByteArray();
+ println("Actual Storage Bytes " + uarr.length);
+ assertEquals(initBytes, uarr.length);
+ assertEquals(initBytes, 64);
+ }
+
+
@Test
public void checkGrowFromWrappedEmptySketch() {
final int k = 16;
final int n = 0;
- final int initBytes = DoublesSketch.getUpdatableStorageBytes(k, n); //8 bytes
+ final int initBytes = DoublesSketch.getUpdatableStorageBytes(k, n); //empty: 8 bytes
final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build();
- final Memory origSketchMem = Memory.wrap(usk1.toByteArray());
-
- try (WritableHandle memHandle = WritableMemory.allocateDirect(initBytes,
- ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())) {
- WritableMemory mem = memHandle.getWritable();
- origSketchMem.copyTo(0, mem, 0, initBytes);
- UpdateDoublesSketch usk2 = DirectUpdateDoublesSketch.wrapInstance(mem);
- assertTrue(mem.isSameResource(usk2.getMemory()));
- assertEquals(mem.getCapacity(), initBytes);
- assertTrue(mem.isDirect());
- assertTrue(usk2.isEmpty());
-
- //update the sketch forcing it to grow on-heap
- for (int i = 1; i <= 5; i++) { usk2.update(i); }
- assertEquals(usk2.getN(), 5);
- WritableMemory mem2 = usk2.getMemory();
- assertFalse(mem.isSameResource(mem2));
- assertFalse(mem2.isDirect()); //should now be on-heap
-
- final int expectedSize = COMBINED_BUFFER + ((2 * k) << 3);
- assertEquals(mem2.getCapacity(), expectedSize);
- } catch (final Exception e) {
- throw new RuntimeException(e);
- }
+ final Memory origSketchMem = Memory.wrap(usk1.toByteArray()); //on heap
+
+ WritableMemory wmem = WritableMemory.allocateDirect(initBytes, Arena.ofConfined()); //off heap
+ WritableMemory wmem2 = wmem;
+ origSketchMem.copyTo(0, wmem, 0, initBytes);
+ UpdateDoublesSketch usk2 = DirectUpdateDoublesSketch.wrapInstance(wmem);
+ assertTrue(wmem.isSameResource(usk2.getMemory()));
+ assertEquals(wmem.getCapacity(), initBytes);
+ assertTrue(wmem.isDirect());
+ assertTrue(usk2.isEmpty());
+
+ //update the sketch forcing it to grow on-heap
+ usk2.update(1.0);
+ assertEquals(usk2.getN(), 1);
+ WritableMemory mem2 = usk2.getMemory();
+ assertFalse(wmem.isSameResource(mem2));
+ assertFalse(mem2.isDirect()); //should now be on-heap
+
+ final int expectedSize = COMBINED_BUFFER + ((2 * k) << 3);
+ assertEquals(mem2.getCapacity(), expectedSize);
+ assertFalse(wmem2.isAlive());
}
@Test
diff --git a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java
index 5bc5b4eaa..762cfc057 100644
--- a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java
+++ b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java
@@ -28,13 +28,14 @@
import java.nio.ByteOrder;
import org.apache.datasketches.memory.DefaultMemoryRequestServer;
-import org.apache.datasketches.memory.WritableHandle;
import org.apache.datasketches.memory.WritableMemory;
import org.apache.datasketches.quantilescommon.DoublesSortedView;
import org.apache.datasketches.quantilescommon.DoublesSortedViewIterator;
import org.testng.Assert;
import org.testng.annotations.Test;
+import java.lang.foreign.Arena;
+
public class DoublesSketchTest {
@Test
@@ -140,46 +141,40 @@ public void checkEmptyExceptions() {
@Test
public void directSketchShouldMoveOntoHeapEventually() {
- try (WritableHandle wdh = WritableMemory.allocateDirect(1000,
- ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())) {
- WritableMemory mem = wdh.getWritable();
- UpdateDoublesSketch sketch = DoublesSketch.builder().build(mem);
- Assert.assertTrue(sketch.isSameResource(mem));
- for (int i = 0; i < 1000; i++) {
- sketch.update(i);
- }
- Assert.assertFalse(sketch.isSameResource(mem));
- } catch (final Exception e) {
- throw new RuntimeException(e);
+ WritableMemory wmem = WritableMemory.allocateDirect(1000, Arena.ofConfined());
+ WritableMemory wmem2 = wmem;
+ UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem);
+ Assert.assertTrue(sketch.isSameResource(wmem));
+ for (int i = 0; i < 1000; i++) {
+ sketch.update(i);
}
+ Assert.assertFalse(sketch.isSameResource(wmem));
+ Assert.assertFalse(wmem2.isAlive());
}
@Test
public void directSketchShouldMoveOntoHeapEventually2() {
int i = 0;
- try (WritableHandle wdh =
- WritableMemory.allocateDirect(50, ByteOrder.LITTLE_ENDIAN, new DefaultMemoryRequestServer())) {
- WritableMemory mem = wdh.getWritable();
- UpdateDoublesSketch sketch = DoublesSketch.builder().build(mem);
- Assert.assertTrue(sketch.isSameResource(mem));
- for (; i < 1000; i++) {
- if (sketch.isSameResource(mem)) {
- sketch.update(i);
- } else {
- //println("MOVED OUT at i = " + i);
- break;
- }
+ WritableMemory wmem = WritableMemory.allocateDirect(50, Arena.ofConfined());
+ WritableMemory wmem2 = wmem;
+ UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem);
+ Assert.assertTrue(sketch.isSameResource(wmem));
+ for (; i < 1000; i++) {
+ if (sketch.isSameResource(wmem)) {
+ sketch.update(i);
+ } else {
+ //println("MOVED OUT at i = " + i);
+ break;
}
- } catch (final Exception e) {
- throw new RuntimeException(e);
}
+ Assert.assertFalse(wmem2.isAlive());
}
@Test
public void checkEmptyDirect() {
- try (WritableHandle wdh = WritableMemory.allocateDirect(1000)) {
- WritableMemory mem = wdh.getWritable();
- UpdateDoublesSketch sketch = DoublesSketch.builder().build(mem);
+ try (Arena arena = Arena.ofConfined()) {
+ WritableMemory wmem = WritableMemory.allocateDirect(1000, Arena.ofConfined());
+ UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem);
sketch.toByteArray(); //exercises a specific path
} catch (final Exception e) {
throw new RuntimeException(e);
diff --git a/src/test/java/org/apache/datasketches/quantiles/PreambleUtilTest.java b/src/test/java/org/apache/datasketches/quantiles/PreambleUtilTest.java
index be2f328b2..765d4d037 100644
--- a/src/test/java/org/apache/datasketches/quantiles/PreambleUtilTest.java
+++ b/src/test/java/org/apache/datasketches/quantiles/PreambleUtilTest.java
@@ -37,19 +37,19 @@
import static org.apache.datasketches.quantiles.PreambleUtil.insertSerVer;
import static org.testng.Assert.assertEquals;
-import org.testng.annotations.Test;
+import java.lang.foreign.Arena;
import org.apache.datasketches.memory.Memory;
-import org.apache.datasketches.memory.WritableHandle;
import org.apache.datasketches.memory.WritableMemory;
+import org.testng.annotations.Test;
public class PreambleUtilTest {
@Test
public void checkInsertsAndExtracts() {
final int bytes = 32;
- try (WritableHandle offHeapMemHandler = WritableMemory.allocateDirect(bytes)) {
- final WritableMemory offHeapMem = offHeapMemHandler.getWritable();
+ try (Arena arena = Arena.ofConfined()) {
+ WritableMemory offHeapMem = WritableMemory.allocateDirect(bytes, arena);
final WritableMemory onHeapMem = WritableMemory.writableWrap(new byte[bytes]);
onHeapMem.clear();
diff --git a/src/test/java/org/apache/datasketches/quantiles/QuantilesSketchCrossLanguageTest.java b/src/test/java/org/apache/datasketches/quantiles/QuantilesSketchCrossLanguageTest.java
index 8c80f4399..68347ffb8 100644
--- a/src/test/java/org/apache/datasketches/quantiles/QuantilesSketchCrossLanguageTest.java
+++ b/src/test/java/org/apache/datasketches/quantiles/QuantilesSketchCrossLanguageTest.java
@@ -249,7 +249,7 @@ private static void getAndCheck(String ver, int n, double quantile) {
Assert.assertEquals(q2, quantile, 0.0);
// same thing with compact sketch
- qs2 = CompactDoublesSketch.heapify(srcMem);
+ qs2 = HeapCompactDoublesSketch.heapifyInstance(srcMem);
//Test the quantile
q2 = qs2.getQuantile(nf, EXCLUSIVE);
println("New Median: " + q2);
diff --git a/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleTest.java b/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleTest.java
index db043cff6..e1c6914c7 100644
--- a/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleTest.java
+++ b/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleTest.java
@@ -41,6 +41,8 @@ public void empty() {
assertThrows(SketchesStateException.class, () -> td.getMaxValue());
assertThrows(SketchesStateException.class, () -> td.getRank(0));
assertThrows(SketchesStateException.class, () -> td.getQuantile(0.5));
+ assertThrows(SketchesStateException.class, () -> td.getPMF(new double[]{0}));
+ assertThrows(SketchesStateException.class, () -> td.getCDF(new double[]{0}));
}
@Test
@@ -82,6 +84,14 @@ public void manyValues() {
assertEquals(td.getQuantile(0.9), n * 0.9, n * 0.9 * 0.01);
assertEquals(td.getQuantile(0.95), n * 0.95, n * 0.95 * 0.01);
assertEquals(td.getQuantile(1), n - 1);
+ final double[] pmf = td.getPMF(new double[] {n / 2});
+ assertEquals(pmf.length, 2);
+ assertEquals(pmf[0], 0.5, 0.0001);
+ assertEquals(pmf[1], 0.5, 0.0001);
+ final double[] cdf = td.getCDF(new double[] {n / 2});
+ assertEquals(cdf.length, 2);
+ assertEquals(cdf[0], 0.5, 0.0001);
+ assertEquals(cdf[1], 1.0);
}
@Test
diff --git a/src/test/java/org/apache/datasketches/theta/BitPackingTest.java b/src/test/java/org/apache/datasketches/theta/BitPackingTest.java
index 152bee039..c155bef77 100644
--- a/src/test/java/org/apache/datasketches/theta/BitPackingTest.java
+++ b/src/test/java/org/apache/datasketches/theta/BitPackingTest.java
@@ -31,60 +31,127 @@ public class BitPackingTest {
@Test
public void packUnpackBits() {
- for (int bits = 1; bits <= 63; bits++) {
- final long mask = (1 << bits) - 1;
- long[] input = new long[8];
- final long golden64 = Util.INVERSE_GOLDEN_U64;
- long value = 0xaa55aa55aa55aa55L; // arbitrary starting value
- for (int i = 0; i < 8; ++i) {
- input[i] = value & mask;
- value += golden64;
- }
- byte[] bytes = new byte[8 * Long.BYTES];
- int bitOffset = 0;
- int bufOffset = 0;
- for (int i = 0; i < 8; ++i) {
- BitPacking.packBits(input[i], bits, bytes, bufOffset, bitOffset);
- bufOffset += (bitOffset + bits) >>> 3;
- bitOffset = (bitOffset + bits) & 7;
- }
+ long value = 0xaa55aa55aa55aa55L; // arbitrary starting value
+ for (int n = 0; n < 10000; n++) {
+ for (int bits = 1; bits <= 63; bits++) {
+ final long mask = (1 << bits) - 1;
+ long[] input = new long[8];
+ for (int i = 0; i < 8; ++i) {
+ input[i] = value & mask;
+ value += Util.INVERSE_GOLDEN_U64;
+ }
- long[] output = new long[8];
- bitOffset = 0;
- bufOffset = 0;
- for (int i = 0; i < 8; ++i) {
- BitPacking.unpackBits(output, i, bits, bytes, bufOffset, bitOffset);
- bufOffset += (bitOffset + bits) >>> 3;
- bitOffset = (bitOffset + bits) & 7;
- }
- for (int i = 0; i < 8; ++i) {
- assertEquals(output[i], input[i]);
+ byte[] bytes = new byte[8 * Long.BYTES];
+ int bitOffset = 0;
+ int bufOffset = 0;
+ for (int i = 0; i < 8; ++i) {
+ BitPacking.packBits(input[i], bits, bytes, bufOffset, bitOffset);
+ bufOffset += (bitOffset + bits) >>> 3;
+ bitOffset = (bitOffset + bits) & 7;
+ }
+
+ long[] output = new long[8];
+ bitOffset = 0;
+ bufOffset = 0;
+ for (int i = 0; i < 8; ++i) {
+ BitPacking.unpackBits(output, i, bits, bytes, bufOffset, bitOffset);
+ bufOffset += (bitOffset + bits) >>> 3;
+ bitOffset = (bitOffset + bits) & 7;
+ }
+
+ for (int i = 0; i < 8; ++i) {
+ assertEquals(output[i], input[i]);
+ }
}
}
}
@Test
public void packUnpackBlocks() {
- for (int bits = 1; bits <= 63; bits++) {
- if (enablePrinting) { System.out.println("bits " + bits); }
- final long mask = (1L << bits) - 1;
- long[] input = new long[8];
- final long golden64 = Util.INVERSE_GOLDEN_U64;
- long value = 0xaa55aa55aa55aa55L; // arbitrary starting value
- for (int i = 0; i < 8; ++i) {
- input[i] = value & mask;
- value += golden64;
+ long value = 0xaa55aa55aa55aa55L; // arbitrary starting value
+ for (int n = 0; n < 10000; n++) {
+ for (int bits = 1; bits <= 63; bits++) {
+ if (enablePrinting) { System.out.println("bits " + bits); }
+ final long mask = (1L << bits) - 1;
+ long[] input = new long[8];
+ for (int i = 0; i < 8; ++i) {
+ input[i] = value & mask;
+ value += Util.INVERSE_GOLDEN_U64;
+ }
+
+ byte[] bytes = new byte[8 * Long.BYTES];
+ BitPacking.packBitsBlock8(input, 0, bytes, 0, bits);
+ if (enablePrinting) { hexDump(bytes); }
+
+ long[] output = new long[8];
+ BitPacking.unpackBitsBlock8(output, 0, bytes, 0, bits);
+
+ for (int i = 0; i < 8; ++i) {
+ if (enablePrinting) { System.out.println("checking value " + i); }
+ assertEquals(output[i], input[i]);
+ }
}
- byte[] bytes = new byte[8 * Long.BYTES];
- BitPacking.packBitsBlock8(input, 0, bytes, 0, bits);
- if (enablePrinting) { hexDump(bytes); }
+ }
+ }
+
+ @Test
+ public void packBitsUnpackBlocks() {
+ long value = 0; // arbitrary starting value
+ for (int n = 0; n < 10000; n++) {
+ for (int bits = 1; bits <= 63; bits++) {
+ final long mask = (1 << bits) - 1;
+ long[] input = new long[8];
+ for (int i = 0; i < 8; ++i) {
+ input[i] = value & mask;
+ value += Util.INVERSE_GOLDEN_U64;
+ }
+
+ byte[] bytes = new byte[8 * Long.BYTES];
+ int bitOffset = 0;
+ int bufOffset = 0;
+ for (int i = 0; i < 8; ++i) {
+ BitPacking.packBits(input[i], bits, bytes, bufOffset, bitOffset);
+ bufOffset += (bitOffset + bits) >>> 3;
+ bitOffset = (bitOffset + bits) & 7;
+ }
+
+ long[] output = new long[8];
+ BitPacking.unpackBitsBlock8(output, 0, bytes, 0, bits);
+
+ for (int i = 0; i < 8; ++i) {
+ assertEquals(output[i], input[i]);
+ }
+ }
+ }
+ }
+
+ @Test
+ public void packBlocksUnpackBits() {
+ long value = 123L; // arbitrary starting value
+ for (int n = 0; n < 10000; n++) {
+ for (int bits = 1; bits <= 63; bits++) {
+ final long mask = (1 << bits) - 1;
+ long[] input = new long[8];
+ for (int i = 0; i < 8; ++i) {
+ input[i] = value & mask;
+ value += Util.INVERSE_GOLDEN_U64;
+ }
+
+ byte[] bytes = new byte[8 * Long.BYTES];
+ BitPacking.packBitsBlock8(input, 0, bytes, 0, bits);
- long[] output = new long[8];
- BitPacking.unpackBitsBlock8(output, 0, bytes, 0, bits);
+ long[] output = new long[8];
+ int bitOffset = 0;
+ int bufOffset = 0;
+ for (int i = 0; i < 8; ++i) {
+ BitPacking.unpackBits(output, i, bits, bytes, bufOffset, bitOffset);
+ bufOffset += (bitOffset + bits) >>> 3;
+ bitOffset = (bitOffset + bits) & 7;
+ }
- for (int i = 0; i < 8; ++i) {
- if (enablePrinting) { System.out.println("checking value " + i); }
- assertEquals(output[i], input[i]);
+ for (int i = 0; i < 8; ++i) {
+ assertEquals(output[i], input[i]);
+ }
}
}
}
diff --git a/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java b/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java
index 42b6069da..906c1914b 100644
--- a/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java
+++ b/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java
@@ -29,10 +29,11 @@
import org.apache.datasketches.common.Family;
import org.apache.datasketches.common.SketchesArgumentException;
import org.apache.datasketches.memory.Memory;
-import org.apache.datasketches.memory.WritableHandle;
import org.apache.datasketches.memory.WritableMemory;
import org.testng.annotations.Test;
+import java.lang.foreign.Arena;
+
/**
* @author Lee Rhodes
*/
@@ -78,8 +79,8 @@ public void checkHeapifyWrap(int k, int u, boolean ordered) {
//Prepare Memory for direct
int bytes = usk.getCompactBytes(); //for Compact
- try (WritableHandle wdh = WritableMemory.allocateDirect(bytes)) {
- WritableMemory directMem = wdh.getWritable();
+ try (Arena arena = Arena.ofConfined()) {
+ WritableMemory directMem = WritableMemory.allocateDirect(bytes, arena);
/**Via CompactSketch.compact**/
refSk = usk.compact(ordered, directMem);
@@ -586,6 +587,42 @@ public void serializeDeserializeDirectV4() {
}
}
+ @Test
+ public void serializeWrapBytesV3() {
+ final UpdateSketch sk = Sketches.updateSketchBuilder().build();
+ for (int i = 0; i < 10000; i++) {
+ sk.update(i);
+ }
+ final CompactSketch cs1 = sk.compact();
+ final byte[] bytes = cs1.toByteArray();
+ final CompactSketch cs2 = new WrappedCompactSketch(bytes);
+ assertEquals(cs1.getRetainedEntries(), cs2.getRetainedEntries());
+ final HashIterator it1 = cs1.iterator();
+ final HashIterator it2 = cs2.iterator();
+ while (it1.next() && it2.next()) {
+ assertEquals(it2.get(), it2.get());
+ }
+ assertEquals(bytes, cs2.toByteArray());
+ }
+
+ @Test
+ public void serializeWrapBytesV4() {
+ final UpdateSketch sk = Sketches.updateSketchBuilder().build();
+ for (int i = 0; i < 10000; i++) {
+ sk.update(i);
+ }
+ final CompactSketch cs1 = sk.compact();
+ final byte[] bytes = cs1.toByteArrayCompressed();
+ final CompactSketch cs2 = new WrappedCompactCompressedSketch(bytes);
+ assertEquals(cs1.getRetainedEntries(), cs2.getRetainedEntries());
+ final HashIterator it1 = cs1.iterator();
+ final HashIterator it2 = cs2.iterator();
+ while (it1.next() && it2.next()) {
+ assertEquals(it2.get(), it2.get());
+ }
+ assertEquals(bytes, cs2.toByteArray());
+ }
+
private static class State {
String classType = null;
int count = 0;
diff --git a/src/test/java/org/apache/datasketches/theta/ConcurrentDirectQuickSelectSketchTest.java b/src/test/java/org/apache/datasketches/theta/ConcurrentDirectQuickSelectSketchTest.java
index e4c112281..6d6af7047 100644
--- a/src/test/java/org/apache/datasketches/theta/ConcurrentDirectQuickSelectSketchTest.java
+++ b/src/test/java/org/apache/datasketches/theta/ConcurrentDirectQuickSelectSketchTest.java
@@ -119,7 +119,6 @@ public void checkHeapifyByteArrayExact() {
// That is, this is being run for its side-effect of accessing things.
// If something is wonky, it will generate an exception and fail the test.
local2.toString(true, true, 8, true);
-
}
@Test
diff --git a/src/test/java/org/apache/datasketches/theta/ConcurrentHeapQuickSelectSketchTest.java b/src/test/java/org/apache/datasketches/theta/ConcurrentHeapQuickSelectSketchTest.java
index 2261edc3b..84ddcb80e 100644
--- a/src/test/java/org/apache/datasketches/theta/ConcurrentHeapQuickSelectSketchTest.java
+++ b/src/test/java/org/apache/datasketches/theta/ConcurrentHeapQuickSelectSketchTest.java
@@ -114,7 +114,7 @@ public void checkIllegalSketchID_UpdateSketch() {
WritableMemory mem = WritableMemory.writableWrap(byteArray);
mem.putByte(FAMILY_BYTE, (byte) 0); //corrupt the Sketch ID byte
- //try to heapify the corruped mem
+ //try to heapify the corrupted mem
Sketch.heapify(mem, sl.seed);
}
diff --git a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java
index 5191c7c68..5f608113a 100644
--- a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java
+++ b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java
@@ -38,14 +38,16 @@
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
+import java.lang.foreign.Arena;
+import java.nio.ByteOrder;
import java.util.Arrays;
import org.apache.datasketches.common.Family;
import org.apache.datasketches.common.ResizeFactor;
import org.apache.datasketches.common.SketchesArgumentException;
import org.apache.datasketches.common.SketchesReadOnlyException;
+import org.apache.datasketches.memory.DefaultMemoryRequestServer;
import org.apache.datasketches.memory.Memory;
-import org.apache.datasketches.memory.WritableHandle;
import org.apache.datasketches.memory.WritableMemory;
import org.apache.datasketches.thetacommon.HashOperations;
import org.apache.datasketches.thetacommon.ThetaUtil;
@@ -59,10 +61,10 @@ public class DirectQuickSelectSketchTest {
@Test//(expectedExceptions = SketchesArgumentException.class)
public void checkBadSerVer() {
int k = 512;
- try (WritableHandle h = makeNativeMemory(k)) {
- WritableMemory mem = h.getWritable();
+ try (Arena arena = Arena.ofConfined()) {
+ WritableMemory wmem = makeNativeMemory(k, arena);
- UpdateSketch usk = UpdateSketch.builder().setNominalEntries(k).build(mem);
+ UpdateSketch usk = UpdateSketch.builder().setNominalEntries(k).build(wmem);
DirectQuickSelectSketch sk1 = (DirectQuickSelectSketch)usk; //for internal checks
assertTrue(usk.isEmpty());
@@ -73,33 +75,33 @@ public void checkBadSerVer() {
assertEquals(usk.getEstimate(), k, 0.0);
assertEquals(sk1.getRetainedEntries(false), k);
- mem.putByte(SER_VER_BYTE, (byte) 0); //corrupt the SerVer byte
+ wmem.putByte(SER_VER_BYTE, (byte) 0); //corrupt the SerVer byte
- Sketch.wrap(mem);
+ Sketch.wrap(wmem);
} catch (final Exception e) {
if (e instanceof SketchesArgumentException) {}
else { throw new RuntimeException(e); }
}
}
- @Test//(expectedExceptions = SketchesArgumentException.class)
+ @Test
public void checkConstructorKtooSmall() {
int k = 8;
- try (WritableHandle h = makeNativeMemory(k)) {
- WritableMemory mem = h.getWritable();
- UpdateSketch.builder().setNominalEntries(k).build(mem);
+ try (Arena arena = Arena.ofConfined()) {
+ WritableMemory wmem = makeNativeMemory(k, arena);
+ UpdateSketch.builder().setNominalEntries(k).build(wmem);
} catch (final Exception e) {
if (e instanceof SketchesArgumentException) {}
else { throw new RuntimeException(e); }
}
}
- @Test//(expectedExceptions = SketchesArgumentException.class)
+ @Test
public void checkConstructorMemTooSmall() {
int k = 16;
- try (WritableHandle h = makeNativeMemory(k/2)) {
- WritableMemory mem = h.getWritable();
- UpdateSketch.builder().setNominalEntries(k).build(mem);
+ try (Arena arena = Arena.ofConfined()) {
+ WritableMemory wmem = makeNativeMemory(k/2, arena);
+ UpdateSketch.builder().setNominalEntries(k).build(wmem);
} catch (final Exception e) {
if (e instanceof SketchesArgumentException) {}
else { throw new RuntimeException(e); }
@@ -123,11 +125,9 @@ public void checkHeapifyIllegalFamilyID_heapify() {
public void checkHeapifyMemoryEstimating() {
int k = 512;
int u = 2*k; //thus estimating
-
- try (WritableHandle h = makeNativeMemory(k)) {
- WritableMemory mem = h.getWritable();
-
- UpdateSketch sk1 = UpdateSketch.builder().setNominalEntries(k).build(mem);
+ try (Arena arena = Arena.ofConfined()) {
+ WritableMemory wmem = makeNativeMemory(k, arena);
+ UpdateSketch sk1 = UpdateSketch.builder().setNominalEntries(k).build(wmem);
for (int i=0; i
-
-
-
-
-
+
-
+
+
@@ -57,7 +48,8 @@ under the License.
-
+
+
-
-
-
-
-
-
+
+
-
+
@@ -68,6 +69,7 @@ under the License.
+
-
+
+
+
+
-
-
\ No newline at end of file